#!/usr/bin/env bash # # BridgesLLM Portal — One-Command Installer # ========================================== # Zero questions. Installs everything. Prints a URL. # # Usage: # curl -fsSL https://bridgesllm.ai/install.sh | sudo bash # # The installer handles infrastructure. Everything configurable # happens in the browser-based setup wizard. # # Supports: Ubuntu 22.04/24.04, Debian 12/13 # set -Eeuo pipefail # The in-Portal updater launches this script as a systemd transient unit, and # systemd starts root units with USER set but HOME unset. Under `set -u` the # first bare ${HOME} reference aborts the whole update, so resolve HOME from # the password database once, before anything reads it. A piped shell install # already exports HOME and keeps it untouched. if [[ -z "${HOME:-}" ]]; then HOME="$(getent passwd "$(id -u)" 2>/dev/null | cut -d: -f6 || true)" [[ -n "${HOME}" ]] || HOME="/root" export HOME fi readonly VERSION="4.0.19" # Prisma's CLI spawns a detached telemetry ("checkpoint") process that # outlives the command. Attested database operations prove their recursive # descendant tree empty before publishing a receipt, so any detached # survivor fails the operation. Disable telemetry globally: the installer # must never daemonize tooling side channels, with or without attestation. export CHECKPOINT_DISABLE=1 export DO_NOT_TRACK=1 readonly SCRIPT_NAME="$(basename "$0")" readonly INSTALL_ROOT="/opt/bridgesllm" readonly PORTAL_DIR="${INSTALL_ROOT}/portal" readonly CADDY_CONFIG_HELPER="${PORTAL_DIR}/installer/caddy-managed-config.py" readonly PORTAL_DEPLOY_STAMP="${INSTALL_ROOT}/.last-portal-deploy" readonly UPDATE_STATE_ROOT="/var/lib/bridgesllm-installer" readonly UPDATE_TRANSACTIONS_ROOT="${UPDATE_STATE_ROOT}/transactions" readonly UPDATE_ACTIVE_JOURNAL="${UPDATE_STATE_ROOT}/active-update.json" readonly UPDATE_CUTOVER_JOURNAL="${UPDATE_STATE_ROOT}/cutover-update.json" readonly BACKUP_QUIESCENCE_JOURNAL="/var/lib/bridgesllm/backup-recovery/quiescence.json" readonly RESTORE_ACTIVE_JOURNAL="/var/lib/bridgesllm-restore/active-restore.json" readonly UPDATE_STATE_HELPER="${UPDATE_STATE_ROOT}/update-transaction-state.py" readonly UPDATE_CADDY_RECOVERY_HELPER="${UPDATE_STATE_ROOT}/caddy-managed-config.py" readonly DASHBOARD_UPDATE_PROGRESS_HELPER="${UPDATE_STATE_ROOT}/dashboard-update-progress.py" readonly UPDATE_BACKUP_ROOT="${INSTALL_ROOT}/backups/update-transactions" readonly UPDATE_STAGE_ROOT="${INSTALL_ROOT}/update-staging" readonly UPDATE_BOOT_FENCE_DROPIN_DIR="/etc/systemd/system/bridgesllm-product.service.d" readonly UPDATE_BOOT_FENCE_DROPIN="${UPDATE_BOOT_FENCE_DROPIN_DIR}/20-update-transaction-fence.conf" readonly LEGACY_DOCKER_PRUNE_CRON_PATH="/etc/cron.d/docker-image-prune" readonly LEGACY_DOCKER_PRUNE_QUARANTINE_PATH="/etc/bridgesllm/quarantine/docker-image-prune.legacy" readonly OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_MARKER="/var/lib/bridgesllm/openclaw-gateway-authorization-fence.v1" readonly OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_DROPIN_DIR="/etc/systemd/system/openclaw-gateway.service.d" readonly OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_DROPIN="${OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_DROPIN_DIR}/20-bridgesllm-authorization-fence.conf" readonly OPENCLAW_GATEWAY_ROOT_USER_AUTHORIZATION_FENCE_DROPIN_DIR="/root/.config/systemd/user/openclaw-gateway.service.d" readonly OPENCLAW_GATEWAY_ROOT_USER_AUTHORIZATION_FENCE_DROPIN="${OPENCLAW_GATEWAY_ROOT_USER_AUTHORIZATION_FENCE_DROPIN_DIR}/20-bridgesllm-authorization-fence.conf" readonly RETAINED_INSTALL_MARKER="${INSTALL_ROOT}/.retained-install-v1.json" readonly RETAINED_INSTALL_MANIFEST="${INSTALL_ROOT}/.retained-install-tree-v1.json" readonly UNINSTALL_STATE_ROOT="/var/lib/bridgesllm-installer/uninstall" readonly UNINSTALL_TRANSACTIONS_ROOT="${UNINSTALL_STATE_ROOT}/transactions" readonly UNINSTALL_ACTIVE_JOURNAL="${UNINSTALL_STATE_ROOT}/active-uninstall.json" readonly UNINSTALL_BOOT_FENCE_DROPIN="${UPDATE_BOOT_FENCE_DROPIN_DIR}/30-uninstall-transaction-fence.conf" readonly PORTAL_OPERATION_LOCK_PATH="/run/lock/bridgesllm-portal-installer.lock" readonly UPDATE_CANDIDATE_PORT="4199" readonly LOG_DIR="${INSTALL_ROOT}/logs" readonly TIMESTAMP="$(date +%Y%m%d-%H%M%S)" readonly LOG_FILE="${LOG_DIR}/install-${TIMESTAMP}.log" readonly RELEASE_ORIGIN="${BRIDGESLLM_RELEASE_ORIGIN:-https://bridgesllm.ai}" readonly RELEASE_BASE_URL="${BRIDGESLLM_RELEASE_BASE_URL:-${RELEASE_ORIGIN}/releases/${VERSION}}" readonly RELEASE_URL="${RELEASE_BASE_URL}/portal.tar.gz" readonly RELEASE_MANIFEST_URL="${RELEASE_BASE_URL}/portal-release.manifest" readonly RELEASE_SIGNATURE_URL="${RELEASE_BASE_URL}/portal-release.sig" readonly RELEASE_PUBLIC_KEY_SHA256="72aec2acf2c350dcb4a98104320c3deb522e7fd016c072966327d342897000cc" readonly RELEASE_FALLBACK_DIR="/opt/bridgesllm/portal" readonly MIN_RAM_MB=3500 readonly MIN_DISK_GB=35 # Tested compatibility matrix. Package revisions are deliberately independent: # the CLI/gateway reports the normalized runtime version, while npm retains the # exact stable package revision used to build it. readonly PIN_OPENCLAW_RUNTIME_VERSION="2026.7.1" readonly PIN_OPENCLAW_CORE_PACKAGE_VERSION="2026.7.1-2" readonly PIN_OPENCLAW_CODEX_PLUGIN_VERSION="2026.7.1-1" readonly PIN_BRIDGESLLM_ASK_USER_PLUGIN_VERSION="3.3.0" readonly PIN_CODEX_CLI_VERSION="0.145.0" readonly PIN_CLAUDE_CODE_VERSION="2.1.220" readonly PIN_CLAWHUB_VERSION="0.23.1" readonly PIN_ANTIGRAVITY_VERSION="1.1.7" readonly PIN_GROK_BUILD_VERSION="0.2.112" readonly OLLAMA_INSTALLER_URL="https://ollama.com/install.sh" readonly PIN_NODE_MAJOR="22" readonly PIN_NODE_22_MIN_VERSION="22.22.3" readonly PIN_NODE_24_MIN_VERSION="24.15.0" readonly PIN_NODE_25_MIN_VERSION="25.9.0" readonly OPENCLAW_NODE_ENGINE_RANGE=">=22.22.3 <23 || >=24.15.0 <25 || >=25.9.0" readonly OPENCLAW_PROJECT_SANDBOX_IMAGE_TAG="openclaw-sandbox:bookworm-slim" readonly CODEX_PROJECT_SANDBOX_IMAGE_TAG="bridgesllm-codex-project-runtime:v1" readonly CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_TAG="bridgesllm-claude-code-project-runtime:v1" readonly ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_TAG="bridgesllm-antigravity-project-runtime:v1" readonly OLLAMA_PROJECT_SANDBOX_IMAGE_TAG="bridgesllm-ollama-project-runtime:v1" readonly AGENT_ZERO_PROJECT_SANDBOX_IMAGE_TAG="bridgesllm-agent-zero-project-runtime:v1" readonly PROJECT_EGRESS_PROXY_IMAGE_TAG="bridgesllm-project-egress-proxy:v1" readonly PORTAL_PROJECT_RUNTIME_IMAGE_TAG="bridgesllm-project-runtime:bookworm-node22" readonly PORTAL_PROJECT_RUNTIME_RECIPE_LABEL="com.bridgesllm.portal-project-runtime.recipe-sha256" readonly PROJECT_RUNTIME_APPARMOR_PROFILE_NAME="bridgesllm-project-runtime-v1" readonly PROJECT_RUNTIME_APPARMOR_PROFILE_SOURCE="${PORTAL_DIR}/installer/bridgesllm-project-runtime-v1.apparmor" readonly PROJECT_RUNTIME_APPARMOR_PROFILE_PATH="/etc/apparmor.d/bridgesllm-project-runtime-v1" readonly PROJECT_RUNTIME_APPARMOR_PROFILE_SHA256="6a6f07e3481c678eb6b417532931a22c54200aba3560e1eaa2cb883f99f33164" readonly PROJECT_RUNTIME_SECCOMP_PROFILE_SOURCE="${PORTAL_DIR}/installer/bridgesllm-project-runtime-v1.seccomp.json" readonly PROJECT_RUNTIME_SECCOMP_PROFILE_PATH="/etc/bridgesllm/project-runtime/bridgesllm-project-runtime-v1.seccomp.json" readonly PROJECT_RUNTIME_SECCOMP_PROFILE_SHA256="de1f5327ca42b80be02daba8d39c0d087a530dc3c16f7028170fe068c9d66e61" readonly CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_NAME="bridgesllm-codex-project-runtime-v1" readonly CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_SOURCE="${PORTAL_DIR}/installer/bridgesllm-codex-project-runtime-v1.apparmor" readonly CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_PATH="/etc/apparmor.d/bridgesllm-codex-project-runtime-v1" readonly CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_SHA256="8c4e7db070bb7d6be3ef167e17dcc0949992c8fa6b6c58b47b758e2a939f8b24" readonly CODEX_PROJECT_RUNTIME_SECCOMP_PROFILE_SOURCE="${PORTAL_DIR}/installer/bridgesllm-codex-project-runtime-v1.seccomp.json" readonly CODEX_PROJECT_RUNTIME_SECCOMP_PROFILE_PATH="/etc/bridgesllm/project-runtime/bridgesllm-codex-project-runtime-v1.seccomp.json" readonly CODEX_PROJECT_RUNTIME_SECCOMP_PROFILE_SHA256="e83f93eaf5b476dfd401d0482210217c5ff1484d1655ba9ca77de59435193c02" readonly PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY="apparmor-seccomp-v1" readonly PROJECT_RUNTIME_SECCOMP_ONLY_POLICY="seccomp-only-apparmor-unsupported-v1" # Operator-selected degraded mode (--skip-project-runtimes): the Portal installs # and runs, while every Project runtime admission fails closed with an explicit # disabled reason. This is never selected automatically. readonly PROJECT_RUNTIME_DISABLED_POLICY="project-runtimes-disabled-v1" # Internal cleanup proof emitted only when the durably-created preparation # environment is absent. Because that inode and its parent directory are # fsynced before the first Docker operation, absence proves this transaction # could not have created a candidate tag. readonly PROJECT_RUNTIME_PREPARATION_NOT_STARTED_POLICY="project-runtime-preparation-not-started" readonly OPENCLAW_SANDBOX_RECIPE_LABEL="com.bridgesllm.openclaw-sandbox.recipe-sha256" readonly CODEX_PROJECT_RECIPE_LABEL="com.bridgesllm.codex-project.recipe-sha256" readonly CODEX_PROJECT_CLI_LABEL="com.bridgesllm.codex-project.cli-version" readonly CLAUDE_CODE_PROJECT_RECIPE_LABEL="com.bridgesllm.claude-code-project.recipe-sha256" readonly CLAUDE_CODE_PROJECT_CLI_LABEL="com.bridgesllm.claude-code-project.cli-version" readonly ANTIGRAVITY_PROJECT_RECIPE_LABEL="com.bridgesllm.antigravity-project.recipe-sha256" readonly ANTIGRAVITY_PROJECT_CLI_LABEL="com.bridgesllm.antigravity-project.cli-version" readonly ANTIGRAVITY_PROJECT_BINARY_LABEL="com.bridgesllm.antigravity-project.binary-sha256" readonly OLLAMA_PROJECT_RECIPE_LABEL="com.bridgesllm.ollama-project.recipe-sha256" readonly AGENT_ZERO_PROJECT_RECIPE_LABEL="com.bridgesllm.agent-zero-project.recipe-sha256" readonly AGENT_ZERO_PROJECT_SOURCE_COMMIT_LABEL="com.bridgesllm.agent-zero-project.source-commit" readonly AGENT_ZERO_PROJECT_UPSTREAM_DIGEST_LABEL="com.bridgesllm.agent-zero-project.upstream-digest" readonly AGENT_ZERO_PROJECT_RUNTIME_USER_LABEL="com.bridgesllm.agent-zero-project.runtime-user" readonly AGENT_ZERO_PROJECT_RUNTIME_USER="1000:1000" readonly AGENT_ZERO_PROJECT_SOURCE_COMMIT="d1d48bc9c0e6e253e87c354ce757c518820c6e25" readonly AGENT_ZERO_PROJECT_AMD64_UPSTREAM_DIGEST="sha256:9b48534c1279fb831513b8c970e2d9004e7a2a6708a4d53a91a76d24a4f9f7eb" readonly AGENT_ZERO_PROJECT_ARM64_UPSTREAM_DIGEST="sha256:da107b689828124369d83f017b9664493c0699c60e57809fbd32f647078de49c" readonly PROJECT_EGRESS_RECIPE_LABEL="com.bridgesllm.project-egress.recipe-sha256" readonly PROJECT_EGRESS_ARTIFACTS_LABEL="com.bridgesllm.project-egress.artifacts-sha256" readonly PROJECT_EGRESS_POLICY_LABEL="com.bridgesllm.project-egress.policy" readonly PROJECT_EGRESS_ROLE_LABEL="com.bridgesllm.project-egress.role" readonly PROJECT_EGRESS_IDENTITY_LABEL="com.bridgesllm.project-egress.identity" readonly PROJECT_EGRESS_ACTOR_LABEL="com.bridgesllm.project-egress.actor-id" readonly PROJECT_EGRESS_PROJECT_LABEL="com.bridgesllm.project-egress.project-id" readonly PROJECT_EGRESS_PROVIDER_LABEL="com.bridgesllm.project-egress.provider" readonly PROJECT_EGRESS_CONSUMER_KIND_LABEL="com.bridgesllm.project-egress.consumer-kind" readonly PROJECT_EGRESS_WORKLOAD_LABEL="com.bridgesllm.project-egress.workload-id" readonly PROJECT_EGRESS_FINGERPRINT_LABEL="com.bridgesllm.project-egress.fingerprint" readonly PROJECT_EGRESS_TOKEN_HASH_LABEL="com.bridgesllm.project-egress.token-hash" readonly PROJECT_EGRESS_RUNTIME_FINGERPRINT_LABEL="com.bridgesllm.project-egress.runtime-fingerprint" readonly PROJECT_EGRESS_POLICY_VERSION="portal-project-egress-v1" # Flags DOMAIN="" APP_CONTENT_DOMAIN="" APP_CONTENT_ORIGIN="" APP_CONTENT_DNS_MODE="" APP_CONTENT_SELECTION_EXPLICIT=false # Origin mode: "" (domain/loopback classic behavior) or "tailnet" (private # https://..ts.net via Tailscale Serve; no public ports). ORIGIN_MODE="" ORIGIN_SELECTION_EXPLICIT=false TS_AUTHKEY="" TS_HOSTNAME="bridgesllm-portal" TAILNET_DNS_NAME="" DRY_RUN=false UPDATE_MODE=false UNINSTALL_MODE=false REPAIR_PROJECT_RUNTIME_IMAGE=false RESIDUE_POLICY="" UNINSTALL_RESIDUE_WIPE_BACKUP_DIR="" UNINSTALL_RESIDUE_PLAN_DIGEST="" FORCE_FRESH=false REPAIR_REINSTALL=false RETAINED_RECONNECT_MODE=false SKIP_OLLAMA=false SKIP_OPENCLAW=false SKIP_PROJECT_RUNTIMES=false MAINTAIN_TOOLS=false INSTALL_PROFILE="server" OPENCLAW_PACKAGE_UPDATED=false OPENCLAW_PACKAGE_UPDATE_ATTEMPTED=false OPENCLAW_PACKAGE_PREEXISTED=false OPENCLAW_PREUPDATE_PACKAGE_VERSION="" OPENCLAW_PREUPDATE_RUNTIME_VERSION="" OPENCLAW_STATE_EXISTED_BEFORE_UPDATE=false OPENCLAW_GATEWAY_WAS_ENABLED=false OPENCLAW_GATEWAY_WAS_ACTIVE=false OPENCLAW_BASELINE_PID="" OPENCLAW_BASELINE_RESTARTS="" OPENCLAW_ROLLBACK_PACKAGE_TARBALL="" OPENCLAW_UPGRADE_STATE_MANIFEST="" OPENCLAW_ROLLBACK_IN_PROGRESS=false OPENCLAW_UPGRADE_COMMITTED=false OPENCLAW_RESCUE_MODE=false OPENCLAW_CODEX_PLUGIN_UPDATE_ATTEMPTED=false OPENCLAW_CODEX_PLUGIN_PREEXISTED=false OPENCLAW_CODEX_PLUGIN_PREUPDATE_VERSION="" OPENCLAW_CODEX_PLUGIN_ROLLBACK_TARBALL="" OPENCLAW_CODEX_PLUGIN_BASELINE_CAPTURED=false OPENCLAW_PENDING_INPUT_HOTFIX_APPLIED=false OPENCLAW_PENDING_INPUT_HOTFIX_TARGET="" OPENCLAW_PENDING_INPUT_HOTFIX_BACKUP="" OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED=false OPENCLAW_CLAUDE_ASK_USER_HOTFIX_APPLIED=false OPENCLAW_CLAUDE_ASK_USER_HOTFIX_TARGET="" OPENCLAW_CLAUDE_ASK_USER_HOTFIX_BACKUP="" OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED=false OPENCLAW_ASK_USER_TRANSACTION_ARMED=false OPENCLAW_ASK_USER_TRANSACTION_COMMITTED=false OPENCLAW_ASK_USER_TRANSACTION_DIR="" OPENCLAW_ASK_USER_STATE_DIR="" OPENCLAW_ASK_USER_GATEWAY_WAS_ACTIVE=false OPENCLAW_ASK_USER_BASE_ATTESTED=false OPENCLAW_ASK_USER_LIVE_ATTESTED=false OPENCLAW_COMPAT_HOTFIX_PID="" OPENCLAW_TESTED_PAIR_COMMIT_RECORD="/root/.openclaw/.bridgesllm-tested-pair-commit-v3.json" # Generated during install DB_PASSWORD="" JWT_SECRET="" JWT_REFRESH_SECRET="" PORTAL_UPDATE_PROBE_TOKEN="" PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR="" PROJECT_RUNTIME_REPAIR_ENV_FILE="" PROJECT_RUNTIME_REPAIR_ACTIVE_LOG_FILE="" PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID="" PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_KIND="" PROJECT_RUNTIME_REPAIR_TRANSACTION_ROOT="" PORTAL_PROJECT_RUNTIME_RECIPE_SHA256="" OPENCLAW_TOKEN="" OPENCLAW_PROJECT_SANDBOX_IMAGE_ID="" CODEX_PROJECT_SANDBOX_IMAGE_ID="" CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID="" ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID="" OLLAMA_PROJECT_SANDBOX_IMAGE_ID="" AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID="" PROJECT_EGRESS_PROXY_IMAGE_ID="" PROJECT_EGRESS_TOKEN_SECRET="" PROJECT_RUNTIME_CONFINEMENT_POLICY="" CODEX_PROJECT_RUNTIME_CONFINEMENT_AVAILABLE=false VERIFIED_RELEASE_VERSION="" VERIFIED_RELEASE_ARTIFACT_SHA256="" VERIFIED_RELEASE_MANIFEST_SHA256="" VERIFIED_RELEASE_MANIFEST_SCHEMA="" UNVERIFIED_LOCAL_SOURCE_USED=false SETUP_TOKEN="" SETUP_TOKEN_EXPIRES_AT="" SETUP_TOKEN_USED_AT="" SETUP_SESSION_TOKEN_HASH="" SETUP_SESSION_ORIGIN="" SETUP_SESSION_EXPIRES_AT="" SETUP_HANDOFF_TOKEN_HASH="" SETUP_HANDOFF_ORIGIN="" SETUP_HANDOFF_EXPIRES_AT="" PUBLIC_IP="" TELEMETRY_INSTALL_ID="" # State CURRENT_STEP="startup" TOTAL_STEPS=9 CURRENT_STEP_NUM=0 INSTALL_START_TIME="" PACKAGE_MANAGER_REPAIR_ACTIVE=false PACKAGE_MANAGER_LONG_WAIT_SECONDS=1800 UPDATE_RECOVERY_ARMED=false UPDATE_RECOVERY_CADDY_CHANGED=false UPDATE_RECOVERY_DATABASE_DUMP="" UPDATE_RECOVERY_DATABASE_URL="" UPDATE_RECOVERY_DATABASE_CONTRACT_VARIANT="" UPDATE_RECOVERY_DATABASE_RESTORE_REQUIRED=false UPDATE_RELEASE_STAGE_DIR="" UPDATE_RECOVERY_NODE_MODULES_BACKUP="" UPDATE_RECOVERY_NODE_MODULES_PREEXISTED=false UPDATE_RECOVERY_DEPLOY_STAMP_CAPTURED=false UPDATE_RECOVERY_DEPLOY_STAMP_EXISTED=false UPDATE_RECOVERY_DEPLOY_STAMP_BACKUP="" UPDATE_RECOVERY_IN_PROGRESS=false UPDATE_TRANSACTION_ID="" UPDATE_TRANSACTION_GENERATION="" UPDATE_TRANSACTION_PREVIOUS_VERSION="" UPDATE_TRANSACTION_TARGET_VERSION="" UPDATE_TRANSACTION_BASELINE_PID="" UPDATE_TRANSACTION_BASELINE_BOOT_ID="" UPDATE_TRANSACTION_BASELINE_START_TIME="" UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE=false UPDATE_TRANSACTION_PORTAL_WAS_ENABLED=false UPDATE_TRANSACTION_CANDIDATE_UNIT="" UPDATE_DISK_RESERVE_MANIFEST="" UPDATE_DATABASE_OPERATION_LATCH="" UPDATE_DATABASE_OPERATION_SUPERVISOR_PID="" UPDATE_DATABASE_OPERATION_SETTLEMENT_IN_PROGRESS=false DASHBOARD_UPDATE_PROGRESS_PERCENT=5 DASHBOARD_UPDATE_PROGRESS_PHASE="installer-download" DASHBOARD_UPDATE_PORTAL_COMMITTED=false DASHBOARD_UPDATE_FAILURE_MESSAGE="" DASHBOARD_UPDATE_FAILURE_PHASE="" UNINSTALL_TRANSACTION_ID="" UNINSTALL_RECOVERED_THIS_RUN=false # OS detection OS_ID="" OS_VERSION="" APT_AVAILABLE=false IS_WSL=false node_version_meets_minimum() { command -v node &>/dev/null || return 1 local version major minor patch version="$(node -v 2>/dev/null | sed 's/^v//' || true)" major="${version%%.*}" minor="${version#*.}" minor="${minor%%.*}" patch="${version#*.*.}" [[ "${major}" =~ ^[0-9]+$ && "${minor}" =~ ^[0-9]+$ && "${patch}" =~ ^[0-9]+$ ]] || return 1 # Match the exact engine range published by the tested OpenClaw package. # Node 23 is intentionally unsupported; Node 22/24 also have patch floors. if (( major == 22 )); then (( minor > 22 || (minor == 22 && patch >= 3) )) elif (( major == 23 )); then return 1 elif (( major == 24 )); then (( minor > 15 || (minor == 15 && patch >= 0) )) elif (( major >= 25 )); then (( major > 25 || minor > 9 || (minor == 9 && patch >= 0) )) else return 1 fi } ensure_supported_node_runtime() { if node_version_meets_minimum; then ok "Node.js $(node --version 2>/dev/null || echo compatible) (OpenClaw-compatible; unchanged)" return 0 fi info "Node.js $(node --version 2>/dev/null || echo missing) is outside ${OPENCLAW_NODE_ENGINE_RANGE}; converging the supported Node 22 lane first..." spin "Setting up Node.js ${PIN_NODE_MAJOR} repository" "curl -fsSL https://deb.nodesource.com/setup_${PIN_NODE_MAJOR}.x | bash -" spin "Installing compatible Node.js runtime" "apt-get update -qq && apt-get install -y -qq --allow-downgrades nodejs" node_version_meets_minimum \ || fail "Node.js must satisfy ${OPENCLAW_NODE_ENGINE_RANGE} before Portal or OpenClaw package operations can run." ok "Node.js $(node --version 2>/dev/null || echo compatible) (OpenClaw-compatible)" } # ═══════════════════════════════════════════════════════════════ # Terminal styling # ═══════════════════════════════════════════════════════════════ readonly RED='\033[0;31m' readonly GREEN='\033[0;32m' readonly YELLOW='\033[0;33m' readonly BLUE='\033[0;34m' readonly MAGENTA='\033[0;35m' readonly CYAN='\033[0;36m' readonly WHITE='\033[1;37m' readonly BOLD='\033[1m' readonly DIM='\033[2m' readonly ITALIC='\033[3m' readonly NC='\033[0m' readonly BULLET='•' ok() { echo -e " ${GREEN}✓${NC} $*"; } warn() { echo -e " ${YELLOW}⚠${NC} $*"; } info() { echo -e " ${DIM}→${NC} $*"; } progress() { echo -e " ${BLUE}${BULLET}${NC} $*"; } # The Dashboard updater redirects this installer's stdout into a durable, # root-only outer log that survives the Portal restart. Emit a small, # allowlisted tab-separated protocol only when the fixed systemd wrapper has # supplied a valid operation identity. These records are observability only; # the signed transaction journal remains the recovery authority. dashboard_update_progress() { local status="$1" percent="$2" phase="$3" label="$4" detail="${5:-}" local operation_id="${BRIDGESLLM_DASHBOARD_UPDATE_ID:-}" local helper="${DASHBOARD_UPDATE_PROGRESS_HELPER}" [[ "${UPDATE_MODE:-false}" == "true" \ && "${operation_id}" =~ ^[a-f0-9]{32}$ ]] || return 0 [[ "${status}" =~ ^(running|recovering|rolled_back|updated_with_errors|recovery_required)$ \ && "${percent}" =~ ^[0-9]+$ \ && "${phase}" =~ ^[a-z0-9][a-z0-9-]{0,47}$ ]] || return 0 (( percent <= 99 )) || percent=99 if (( percent < DASHBOARD_UPDATE_PROGRESS_PERCENT )); then percent="${DASHBOARD_UPDATE_PROGRESS_PERCENT}" fi label="${label//$'\t'/ }" label="${label//$'\r'/ }" label="${label//$'\n'/ }" detail="${detail//$'\t'/ }" detail="${detail//$'\r'/ }" detail="${detail//$'\n'/ }" label="${label:0:160}" detail="${detail:0:800}" [[ -n "${label}" ]] || label="Update in progress" DASHBOARD_UPDATE_PROGRESS_PERCENT="${percent}" DASHBOARD_UPDATE_PROGRESS_PHASE="${phase}" if [[ ! -f "${helper}" || -L "${helper}" ]] \ || ! /usr/bin/python3 "${helper}" update \ --operation-id "${operation_id}" \ --status "${status}" \ --percent "${percent}" \ --phase "${phase}" \ --label "${label}" \ --detail "${detail}"; then # Progress is an observer, never recovery authority. A write failure is # visible through the outer unit reconciliation but must not perturb the # signed transaction or trigger rollback by itself. echo " Progress observer could not persist phase ${phase}." >&2 fi } fail() { # A failure inside failure handling must terminate immediately with one # line: no second banner, no second recovery attempt, and never the # unrelated OpenClaw pair rollback that a re-entered fail() used to reach. if [[ "${FAIL_TERMINAL_IN_PROGRESS:-false}" == "true" ]]; then echo -e " ${RED}FATAL during failure handling: $1${NC}" >&2 exit 1 fi FAIL_TERMINAL_IN_PROGRESS=true DASHBOARD_UPDATE_FAILURE_MESSAGE="$1" DASHBOARD_UPDATE_FAILURE_PHASE="${DASHBOARD_UPDATE_PROGRESS_PHASE:-failure}" # fail() never returns; from here on nothing may re-trigger ERR handling. trap - ERR trap '' SIGINT TERM HUP set +e if [[ "${REPAIR_PROJECT_RUNTIME_IMAGE:-false}" == "true" ]]; then echo "" >&2 echo -e " ${RED}${BOLD}Project runtime image repair failed:${NC} $1" >&2 [[ -f "${LOG_FILE}" ]] \ && echo -e " ${DIM}Log: ${LOG_FILE}${NC}" >&2 exit 1 fi if [[ "${DASHBOARD_UPDATE_PORTAL_COMMITTED:-false}" == "true" ]]; then dashboard_update_progress updated_with_errors \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${DASHBOARD_UPDATE_FAILURE_PHASE}" \ "Portal updated; follow-up work failed" "$1" elif [[ "${UPDATE_RECOVERY_ARMED:-false}" == "true" ]]; then dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Update stopped — checking recovery" "$1" else dashboard_update_progress running \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" failure \ "Update stopped before completion" "$1" fi echo "" echo -e " ${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e " ${RED}${BOLD} ERROR${NC} ${CYAN}${CURRENT_STEP}${NC}" echo "" echo -e " ${WHITE} $1${NC}" echo "" echo -e " ${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" if [[ -f "$LOG_FILE" ]]; then echo -e " ${DIM}Log: ${LOG_FILE}${NC}" echo -e " ${DIM}Last 5 lines:${NC}" tail -5 "$LOG_FILE" 2>/dev/null | sed 's/^/ /' fi echo "" if declare -F settle_openclaw_compatibility_hotfix_process >/dev/null 2>&1; then settle_openclaw_compatibility_hotfix_process \ || warn "The OpenClaw compatibility patch process could not be proven stopped before recovery." fi local database_operation_settled=true if declare -F settle_active_update_database_operation >/dev/null 2>&1; then settle_active_update_database_operation || database_operation_settled=false fi if [[ "${database_operation_settled}" != "true" ]]; then warn "The active database operation could not be proven stopped. Recovery was not started; the Portal remains boot-fenced." dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${DASHBOARD_UPDATE_FAILURE_PHASE}" \ "Automatic recovery needs attention" \ "The active database operation could not be proven stopped. Do not start another update." exit 1 fi local update_recovery_attempted=false local update_recovery_succeeded=false if [[ "${UPDATE_RECOVERY_ARMED:-false}" == "true" ]] && declare -F recover_interrupted_update >/dev/null 2>&1; then update_recovery_attempted=true UPDATE_RECOVERY_ARMED=false if ! recover_interrupted_update; then warn "Update recovery did not complete. Portal remains boot-fenced and recovery artifacts were preserved for the next installer run." else update_recovery_succeeded=true fi elif declare -F rollback_openclaw_tested_pair >/dev/null 2>&1; then rollback_openclaw_tested_pair || warn "The OpenClaw core/plugin pair needs manual recovery; see ${LOG_FILE}." fi local preserve_transaction_stage=false if declare -F update_transaction_state_path >/dev/null 2>&1; then local active_receipt="" cutover_receipt="" active_receipt="$( update_transaction_state_path "${UPDATE_ACTIVE_JOURNAL}" 2>/dev/null || true )" cutover_receipt="$( update_transaction_state_path "${UPDATE_CUTOVER_JOURNAL}" 2>/dev/null || true )" if [[ -n "${active_receipt}" \ && ( -e "${active_receipt}" || -L "${active_receipt}" ) ]] \ || [[ -n "${cutover_receipt}" \ && ( -e "${cutover_receipt}" || -L "${cutover_receipt}" ) ]]; then preserve_transaction_stage=true fi fi if [[ "${preserve_transaction_stage}" == "false" \ && "${UPDATE_TRANSACTION_ID:-}" =~ ^[a-f0-9]{32}$ ]] \ && declare -F cleanup_prepared_update_project_runtime_tags \ >/dev/null 2>&1; then local cleanup_policy="" if [[ "${SKIP_PROJECT_RUNTIMES:-false}" == "true" ]]; then cleanup_policy="${PROJECT_RUNTIME_DISABLED_POLICY}" elif [[ -n "${UPDATE_RELEASE_STAGE_DIR:-}" ]] \ && declare -F prepared_update_project_runtime_cleanup_policy_from_stage \ >/dev/null 2>&1; then cleanup_policy="$( prepared_update_project_runtime_cleanup_policy_from_stage \ "${UPDATE_RELEASE_STAGE_DIR}" "${UPDATE_TRANSACTION_ID}" \ 2>/dev/null || true )" fi if ! cleanup_prepared_update_project_runtime_tags \ "${UPDATE_TRANSACTION_ID}" "${cleanup_policy}" >/dev/null 2>&1; then preserve_transaction_stage=true warn "Prepared Project runtime tags could not be proven removed. The private update stage was preserved for exact cleanup on the next installer run." fi fi if [[ -n "${UPDATE_RELEASE_STAGE_DIR:-}" ]] \ && "${preserve_transaction_stage}" == "false" \ && declare -F cleanup_release_stage_dir >/dev/null 2>&1; then cleanup_release_stage_dir "${UPDATE_RELEASE_STAGE_DIR}" >/dev/null 2>&1 || true UPDATE_RELEASE_STAGE_DIR="" fi if [[ "${preserve_transaction_stage}" == "false" ]] \ && declare -F release_update_disk_reserves >/dev/null 2>&1; then release_update_disk_reserves >/dev/null 2>&1 || true fi if [[ "${DASHBOARD_UPDATE_PORTAL_COMMITTED:-false}" == "true" ]]; then dashboard_update_progress updated_with_errors \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${DASHBOARD_UPDATE_FAILURE_PHASE}" \ "Portal updated; follow-up work failed" \ "${DASHBOARD_UPDATE_FAILURE_MESSAGE}" elif [[ "${update_recovery_attempted}" == "true" \ && "${update_recovery_succeeded}" == "true" ]]; then dashboard_update_progress rolled_back \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" rolled-back \ "Previous Portal restored" \ "The update failed, and the previous Portal was restored and verified. ${DASHBOARD_UPDATE_FAILURE_MESSAGE}" elif [[ "${update_recovery_attempted}" == "true" ]]; then dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${DASHBOARD_UPDATE_FAILURE_PHASE}" \ "Automatic recovery needs attention" \ "Recovery did not complete. Do not start another update; review the root-only transaction journal and installer log." fi exit 1 } draw_progress_bar() { local current=$1 local total=$2 local label="${3:-}" local bar_width=24 local filled=$(( (current * bar_width) / total )) local empty=$(( bar_width - filled )) local pct=$(( (current * 100) / total )) local bar="" local i for ((i = 0; i < filled; i++)); do bar+="█"; done for ((i = 0; i < empty; i++)); do bar+="░"; done echo -e " ${CYAN}│${NC} ${CYAN}[${bar}]${NC} ${DIM}${pct}%${NC} ${DIM}·${NC} ${WHITE}${BOLD}${label}${NC} ${DIM}(${current}/${total})${NC}" } step_header() { CURRENT_STEP_NUM=$((CURRENT_STEP_NUM + 1)) local label="$1" echo "" echo -e " ${CYAN}┌────────────────────────────────────────────────────${NC}" draw_progress_bar "$CURRENT_STEP_NUM" "$TOTAL_STEPS" "$label" echo -e " ${CYAN}└────────────────────────────────────────────────────${NC}" } banner() { clear 2>/dev/null || true echo "" echo -e " ${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e " ${WHITE}${BOLD} B R I D G E S L L M Portal${NC}" echo -e " ${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e " ${DIM}Installer v${VERSION}${NC}" } print_kv() { # Print a key-value pair aligned nicely local key="$1" val="$2" color="${3:-$NC}" printf " ${DIM}%-16s${NC} ${color}%s${NC}\n" "${key}" "${val}" } elapsed_since_start() { if [[ -n "$INSTALL_START_TIME" ]]; then local now now=$(date +%s) local diff=$((now - INSTALL_START_TIME)) local mins=$((diff / 60)) local secs=$((diff % 60)) if ((mins > 0)); then echo "${mins}m ${secs}s" else echo "${secs}s" fi fi } # ═══════════════════════════════════════════════════════════════ # Utilities # ═══════════════════════════════════════════════════════════════ command_needs_package_manager() { local cmd="$*" [[ "$cmd" =~ (^|[[:space:]])(apt-get|apt|dpkg)([[:space:]]|$) ]] } package_manager_lock_paths() { cat <<'EOF' /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock EOF } # Active package-manager processes — only ones that actually hold locks. # Excludes passive daemons like unattended-upgrade-shutdown (--wait-for-signal) # and packagekitd (idle), and our own detection processes. package_manager_active_procs() { ps -eo pid=,comm=,args= 2>/dev/null | awk ' # Skip our own detection processes (awk/grep/ps show the pattern text in args) $2 == "awk" || $2 == "grep" || $2 == "ps" || $2 == "bash" || $2 == "sh" { next } # Only match processes actively doing package work /apt-get|aptitude/ { print $2; next } /\/usr\/bin\/dpkg/ { print $2; next } /\/usr\/bin\/unattended-upgrade($| )/ { print "unattended-upgr"; next } /cloud-init.*apt/ { print "cloud-init"; next } ' | sort -u } package_manager_holder_name() { # First: check who actually holds the locks (most accurate) if command -v fuser &>/dev/null; then local lock holders_found="" while IFS= read -r lock; do [[ -e "$lock" ]] || continue local pids pids="$(fuser "$lock" 2>/dev/null | xargs 2>/dev/null || true)" if [[ -n "$pids" ]]; then for pid in $pids; do local name name="$(ps -p "$pid" -o comm= 2>/dev/null || echo "pid:$pid")" holders_found="${holders_found:+$holders_found, }${name}" done fi done < <(package_manager_lock_paths) if [[ -n "$holders_found" ]]; then echo "$holders_found" return fi fi # Fallback: active process names local procs procs="$(package_manager_active_procs | head -3)" if [[ -n "$procs" ]]; then echo "$procs" | paste -sd ', ' - return fi echo "system packages" } cloud_init_pending() { if command -v cloud-init &>/dev/null; then local status status="$(cloud-init status 2>/dev/null || true)" [[ "$status" != *"status: done"* ]] else pgrep -af 'cloud-init' >/dev/null 2>&1 fi } package_manager_is_busy() { # Primary check: are any lock files actually held? if command -v fuser &>/dev/null; then local lock while IFS= read -r lock; do [[ -e "$lock" ]] || continue if fuser "$lock" >/dev/null 2>&1; then return 0 fi done < <(package_manager_lock_paths) fi # Secondary check: are active package processes running? if package_manager_active_procs | grep -q .; then return 0 fi return 1 } safe_package_manager_repair() { $PACKAGE_MANAGER_REPAIR_ACTIVE && return 1 PACKAGE_MANAGER_REPAIR_ACTIVE=true local rc=0 { echo "[$(date -Is)] attempting safe package-manager recovery" >> "$LOG_FILE" DEBIAN_FRONTEND=noninteractive dpkg --configure -a >> "$LOG_FILE" 2>&1 || rc=$? if [[ $rc -eq 0 ]]; then DEBIAN_FRONTEND=noninteractive apt-get -f install -y >> "$LOG_FILE" 2>&1 || rc=$? fi } PACKAGE_MANAGER_REPAIR_ACTIVE=false return $rc } wait_for_package_manager_ready() { local reason="${1:-package manager work}" local timeout_seconds="${2:-$PACKAGE_MANAGER_LONG_WAIT_SECONDS}" local start_ts now elapsed holder last_notice=-1 tick=0 start_ts=$(date +%s) if ! package_manager_is_busy; then return 0 fi echo "" echo -e " ${CYAN}┌────────────────────────────────────────────────────${NC}" if cloud_init_pending; then echo -e " ${CYAN}│${NC} ${YELLOW}⚠${NC} ${WHITE}${BOLD}Fresh VPS — waiting for system updates${NC}" echo -e " ${CYAN}│${NC}" echo -e " ${CYAN}│${NC} ${DIM}Your server is installing security patches. This is${NC}" echo -e " ${CYAN}│${NC} ${DIM}normal on a new VPS and usually takes 2–5 minutes.${NC}" echo -e " ${CYAN}│${NC} ${DIM}The installer will continue automatically.${NC}" else echo -e " ${CYAN}│${NC} ${YELLOW}⚠${NC} ${WHITE}${BOLD}Waiting for package manager${NC}" echo -e " ${CYAN}│${NC}" echo -e " ${CYAN}│${NC} ${DIM}Another package task is running.${NC}" echo -e " ${CYAN}│${NC} ${DIM}The installer will continue automatically.${NC}" fi echo -e " ${CYAN}│${NC}" while package_manager_is_busy; do now=$(date +%s) elapsed=$(( now - start_ts )) holder="$(package_manager_holder_name)" if [[ -t 1 ]]; then # draw_pulse_bar does \r itself, prefix with box line local _pw=24 _pframes=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏') local _pf="${_pframes[$(( tick % ${#_pframes[@]} ))]}" local _pc=$(( _pw * 2 )) _pp=$(( tick % (_pw * 2) )) (( _pp >= _pw )) && _pp=$(( _pc - _pp )) local _pb="" _pi for ((_pi = 0; _pi < _pw; _pi++)); do local _pd=$(( _pi - _pp )); (( _pd < 0 )) && _pd=$(( -_pd )) if (( _pd == 0 )); then _pb+="█"; elif (( _pd == 1 )); then _pb+="▓" elif (( _pd == 2 )); then _pb+="▒"; else _pb+="░"; fi done printf "\r ${CYAN}│${NC} ${CYAN}${_pf}${NC} ${CYAN}[${_pb}]${NC} ${DIM}$(format_elapsed $elapsed) · ${holder}${NC} " elif (( elapsed / 30 != last_notice )); then echo -e " ${CYAN}│${NC} ${DIM}Still waiting ($(format_elapsed $elapsed)) — ${holder}${NC}" last_notice=$(( elapsed / 30 )) fi tick=$(( tick + 1 )) if (( elapsed >= timeout_seconds )); then if [[ -t 1 ]]; then printf "\r%-120s\r" ""; fi echo -e " ${CYAN}│${NC}" echo -e " ${CYAN}└────────────────────────────────────────────────────${NC}" warn "Package manager wait exceeded $((timeout_seconds / 60)) minutes. Checking for a safe recovery..." if package_manager_is_busy; then fail "Package manager is still actively busy (${holder}). First-boot updates may still be running. Wait a few more minutes and rerun: curl -fsSL https://bridgesllm.ai/install.sh | sudo bash" fi if safe_package_manager_repair; then ok "Recovered interrupted package manager state" return 0 fi fail "Package manager appears stuck. Check ${LOG_FILE}, then run: dpkg --configure -a && apt-get -f install" fi sleep 0.2 done if [[ -t 1 ]]; then printf "\r%-120s\r" ""; fi echo -e " ${CYAN}│${NC}" echo -e " ${CYAN}│${NC} ${GREEN}✓${NC} ${WHITE}Package manager ready${NC}" echo -e " ${CYAN}└────────────────────────────────────────────────────${NC}" echo "" } run() { if $DRY_RUN; then echo " [dry-run] $*" >> "$LOG_FILE" 2>&1 else if ! $PACKAGE_MANAGER_REPAIR_ACTIVE && command_needs_package_manager "$*"; then wait_for_package_manager_ready "$CURRENT_STEP" fi bash -c "$*" >> "$LOG_FILE" 2>&1 fi } telemetry_event() { local event="$1" case "${event}" in install_start|install_complete|deps_updated|update_complete) ;; *) return 0 ;; esac local os_name; os_name="$(lsb_release -si 2>/dev/null || echo unknown)" local os_ver; os_ver="$(lsb_release -sr 2>/dev/null || echo unknown)" os_name="$(printf '%s' "${os_name}" | LC_ALL=C tr -cd 'A-Za-z0-9._+ -' | cut -c1-64)" os_ver="$(printf '%s' "${os_ver}" | LC_ALL=C tr -cd 'A-Za-z0-9._+ -' | cut -c1-64)" [[ -n "${os_name}" ]] || os_name="unknown" [[ -n "${os_ver}" ]] || os_ver="unknown" local payload="{\"event\":\"${event}\",\"version\":\"${VERSION}\",\"os\":\"${os_name}\",\"osVersion\":\"${os_ver}\",\"installId\":\"${TELEMETRY_INSTALL_ID}\"}" curl -sf -X POST "https://bridgesllm.ai/api/telemetry/event" \ -H 'Content-Type: application/json' \ -d "${payload}" \ >/dev/null 2>&1 & } load_existing_telemetry_install_id() { TELEMETRY_INSTALL_ID="$(read_env_value "${PORTAL_DIR}/backend/.env.production" "TELEMETRY_INSTALL_ID" 2>/dev/null || echo "")" } ensure_telemetry_install_id() { if [[ ! "${TELEMETRY_INSTALL_ID}" =~ ^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$ ]]; then TELEMETRY_INSTALL_ID="$(cat /proc/sys/kernel/random/uuid 2>/dev/null || uuidgen 2>/dev/null || echo "")" fi [[ "${TELEMETRY_INSTALL_ID}" =~ ^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$ ]] \ || fail "Portal could not create its local telemetry install identifier." } ensure_build_tools() { if ! command -v make &>/dev/null || ! command -v g++ &>/dev/null; then spin "Installing build tools" "apt-get install -y -qq build-essential python3 || { apt-get update -qq && apt-get install -y -qq build-essential python3; }" fi } verify_prisma_client_runtime() { local backend_dir="$1" check_label="${2:-prisma-runtime-check}" [[ -d "${backend_dir}" && ! -L "${backend_dir}" ]] || return 1 ( cd "${backend_dir}" node - "${check_label}" <<'NODE' const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); const label = process.argv[2] || 'prisma-runtime-check'; const fail = (message) => { console.error(`[${label}] ${message}`); process.exit(1); }; for (const name of [ 'PRISMA_CLIENT_ENGINE_TYPE', 'PRISMA_QUERY_ENGINE_BINARY', 'PRISMA_QUERY_ENGINE_LIBRARY', 'PRISMA_CLIENT_GET_TIME', 'NODE_PG_FORCE_NATIVE', 'NODE_TLS_REJECT_UNAUTHORIZED', 'PGUSER', 'PGDATABASE', 'PGPORT', 'PGHOST', 'PGPASSWORD', 'PGBINARY', 'PGOPTIONS', 'PGSSLMODE', 'PGSSLNEGOTIATION', 'PGCLIENT_ENCODING', 'PGREPLICATION', 'PGAPPNAME', 'PGCONNECT_TIMEOUT', ]) { if (Object.prototype.hasOwnProperty.call(process.env, name)) { fail(`installer environment contains forbidden Prisma engine override ${name}`); } } const readPackage = (name) => { const packagePath = path.join( process.cwd(), 'node_modules', ...name.split('/'), 'package.json', ); const stat = fs.lstatSync(packagePath); if ( !stat.isFile() || stat.isSymbolicLink() || stat.uid !== process.geteuid() || (stat.mode & 0o022) !== 0 ) { fail(`unsafe installed package metadata for ${name}`); } const value = JSON.parse(fs.readFileSync(packagePath, 'utf8')); if (value.name !== name) { fail(`installed package identity differs for ${name}`); } return { path: packagePath, value }; }; const assertSecureRegularFile = (filePath, minimumSize, maximumSize) => { const stat = fs.lstatSync(filePath); if ( !stat.isFile() || stat.isSymbolicLink() || stat.uid !== process.geteuid() || (stat.mode & 0o022) !== 0 || stat.size < minimumSize || stat.size > maximumSize ) { fail(`unsafe generated runtime file: ${path.basename(filePath)}`); } return stat; }; const rootPackage = JSON.parse(fs.readFileSync('package.json', 'utf8')); const declared = rootPackage.dependencies || {}; const nodeModulesPath = path.join(process.cwd(), 'node_modules'); const nodeModulesStat = fs.lstatSync(nodeModulesPath); if ( !nodeModulesStat.isDirectory() || nodeModulesStat.isSymbolicLink() || nodeModulesStat.uid !== process.geteuid() || (nodeModulesStat.mode & 0o022) !== 0 ) { fail('node_modules directory is unsafe'); } const client = readPackage('@prisma/client'); const adapter = readPackage('@prisma/adapter-pg'); const prisma = readPackage('prisma'); const pg = readPackage('pg'); for (const [name, installed] of [ ['@prisma/client', client.value.version], ['@prisma/adapter-pg', adapter.value.version], ['prisma', prisma.value.version], ['pg', pg.value.version], ]) { if (!/^\d+\.\d+\.\d+$/.test(declared[name] || '')) { fail(`${name} must have an exact declared version`); } if (declared[name] !== installed) { fail(`${name} declared and installed versions differ`); } } if ( client.value.version !== adapter.value.version || client.value.version !== prisma.value.version ) { fail('Prisma client, adapter, and CLI versions must match exactly'); } const generatedRoot = path.join(process.cwd(), 'node_modules', '.prisma', 'client'); const generatedRootStat = fs.lstatSync(generatedRoot); if ( !generatedRootStat.isDirectory() || generatedRootStat.isSymbolicLink() || generatedRootStat.uid !== process.geteuid() || (generatedRootStat.mode & 0o022) !== 0 ) { fail('generated Prisma client directory is unsafe'); } const generatedSchemaPath = path.join(generatedRoot, 'schema.prisma'); const generatedIndexPath = path.join(generatedRoot, 'index.js'); const generatedWasmPath = path.join(generatedRoot, 'query_compiler_bg.wasm'); const generatedCompilerWrapperPath = path.join( generatedRoot, 'query_compiler_bg.js', ); assertSecureRegularFile(generatedSchemaPath, 100, 8 * 1024 * 1024); assertSecureRegularFile(generatedIndexPath, 1_000, 32 * 1024 * 1024); assertSecureRegularFile(generatedWasmPath, 100_000, 64 * 1024 * 1024); assertSecureRegularFile(generatedCompilerWrapperPath, 1_000, 4 * 1024 * 1024); const generatedSchema = fs.readFileSync(generatedSchemaPath, 'utf8'); const clientGenerator = generatedSchema.match(/generator\s+client\s*\{([\s\S]*?)\}/); if ( !clientGenerator || !/provider\s*=\s*["']prisma-client-js["']/.test(clientGenerator[1]) || !/engineType\s*=\s*["']client["']/.test(clientGenerator[1]) ) { fail('generated Prisma schema is not the Rust-free client engine'); } const generatedIndex = fs.readFileSync(generatedIndexPath, 'utf8'); if (!/["']engineType["']\s*:\s*["']client["']/.test(generatedIndex)) { fail('generated Prisma metadata does not attest engineType=client'); } const forbiddenRuntime = /^(?:libquery_engine-|query-engine-)/; const pendingGeneratedDirectories = [generatedRoot]; let generatedEntryCount = 0; while (pendingGeneratedDirectories.length > 0) { const directory = pendingGeneratedDirectories.pop(); for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { generatedEntryCount += 1; if (generatedEntryCount > 512) { fail('generated Prisma client contains too many runtime entries'); } const entryPath = path.join(directory, entry.name); const entryStat = fs.lstatSync(entryPath); if ( entryStat.isSymbolicLink() || entryStat.uid !== process.geteuid() || (entryStat.mode & 0o022) !== 0 ) { fail(`generated Prisma client contains unsafe entry ${entry.name}`); } if (forbiddenRuntime.test(entry.name)) { fail(`generated Prisma client contains forbidden native runtime ${entry.name}`); } if (entry.isDirectory()) { pendingGeneratedDirectories.push(entryPath); } else if (!entry.isFile()) { fail(`generated Prisma client contains unsupported entry ${entry.name}`); } } } const wasmBytes = fs.readFileSync(generatedWasmPath); if (!wasmBytes.subarray(0, 4).equals(Buffer.from([0x00, 0x61, 0x73, 0x6d]))) { fail('generated Prisma query compiler is not a WebAssembly module'); } const clientRoot = path.dirname(client.path); const compilerSourcePath = path.join( clientRoot, 'runtime', 'query_compiler_bg.postgresql.wasm-base64.js', ); const compilerWrapperSourcePath = path.join( clientRoot, 'runtime', 'query_compiler_bg.postgresql.js', ); assertSecureRegularFile(compilerSourcePath, 100_000, 64 * 1024 * 1024); assertSecureRegularFile(compilerWrapperSourcePath, 1_000, 4 * 1024 * 1024); if ( !fs.readFileSync(generatedCompilerWrapperPath).equals( fs.readFileSync(compilerWrapperSourcePath), ) ) { fail('generated Prisma compiler wrapper differs from the installed client package'); } const compilerSource = require(compilerSourcePath); if (typeof compilerSource.wasm !== 'string') { fail('installed Prisma client is missing the PostgreSQL compiler source'); } const trustedWasmBytes = Buffer.from(compilerSource.wasm, 'base64'); if (!wasmBytes.equals(trustedWasmBytes)) { fail('generated Prisma compiler bytes differ from the installed client package'); } for (const name of ['@prisma/client', '@prisma/adapter-pg', 'pg']) { require(name); } console.log( `[${label}] ok engine=client prisma=${client.value.version} pg=${pg.value.version} wasm_sha256=${crypto.createHash('sha256').update(wasmBytes).digest('hex')}`, ); NODE ) } install_backend_runtime_dependencies() { local backend_dir="${PORTAL_DIR}/backend" [[ -d "${backend_dir}" ]] || fail "Backend directory not found at ${backend_dir}" spin "Installing runtime dependencies" "cd '${backend_dir}' && npm ci --omit=dev --ignore-scripts 2>/dev/null || npm install --omit=dev --ignore-scripts" local repaired_native_modules=() if [[ -d "${backend_dir}/prebuilts/node-pty/prebuilds/linux-x64" ]]; then mkdir -p "${backend_dir}/node_modules/node-pty/prebuilds/linux-x64" cp -f "${backend_dir}/prebuilts/node-pty/prebuilds/linux-x64/pty.node" \ "${backend_dir}/node_modules/node-pty/prebuilds/linux-x64/pty.node" ok "Using prebuilt node-pty binary" else ensure_build_tools spin "Building node-pty from source" "cd '${backend_dir}' && npm rebuild node-pty" fi local runtime_modules=(bcrypt sharp) local module for module in "${runtime_modules[@]}"; do if [[ -d "${backend_dir}/node_modules/${module}" ]]; then repaired_native_modules+=("${module}") fi done if (( ${#repaired_native_modules[@]} > 0 )); then spin "Repairing native runtime modules" "cd '${backend_dir}' && npm rebuild ${repaired_native_modules[*]}" fi progress "Generating database client..." if ! run_without_database_authority \ bash -c 'cd "$1" && exec npx prisma generate' \ bridgesllm-prisma-generate "${backend_dir}" \ >> "${LOG_FILE}" 2>&1; then fail "Database client generation failed — check ${LOG_FILE}." fi if ! (cd "${backend_dir}" && node <<'NODE' >> "$LOG_FILE" 2>&1 const checks = [ ['@prisma/client', () => require('@prisma/client')], ['@prisma/adapter-pg', () => require('@prisma/adapter-pg')], ['pg', () => require('pg')], ['bcrypt', () => require('bcrypt')], ['sharp', () => require('sharp')], ['node-pty', () => require('node-pty')], ]; for (const [name, load] of checks) { try { load(); console.log(`[runtime-check] ok ${name}`); } catch (error) { console.error(`[runtime-check] failed ${name}: ${error && error.stack ? error.stack : error}`); process.exit(1); } } NODE ); then fail "Runtime dependency verification failed — check ${LOG_FILE}" fi if ! verify_prisma_client_runtime "${backend_dir}" runtime-prisma-check \ >> "${LOG_FILE}" 2>&1; then fail "Rust-free database runtime verification failed — check ${LOG_FILE}" fi ok "Runtime dependencies verified" } prepare_staged_backend_runtime_dependencies() { local staged_portal="$1" local backend_dir="${staged_portal}/backend" [[ -d "${backend_dir}" && -f "${backend_dir}/package-lock.json" \ && -f "${backend_dir}/package.json" ]] || return 1 local required_tool for required_tool in npm make g++ python3; do command -v "${required_tool}" >/dev/null 2>&1 || { echo "staged dependency preparation missing required tool: ${required_tool}" \ >> "${LOG_FILE}" return 1 } done info "Preparing candidate dependencies before Portal downtime..." ( cd "${backend_dir}" npm ci --omit=dev --ignore-scripts ) >> "${LOG_FILE}" 2>&1 || return 1 if [[ -f "${backend_dir}/prebuilts/node-pty/prebuilds/linux-x64/pty.node" ]]; then install -D -m 0755 \ "${backend_dir}/prebuilts/node-pty/prebuilds/linux-x64/pty.node" \ "${backend_dir}/node_modules/node-pty/prebuilds/linux-x64/pty.node" \ >> "${LOG_FILE}" 2>&1 || return 1 else return 1 fi ( cd "${backend_dir}" npm rebuild bcrypt sharp run_without_database_authority npx prisma generate node <<'NODE' const checks = [ ['@prisma/client', () => require('@prisma/client')], ['@prisma/adapter-pg', () => require('@prisma/adapter-pg')], ['pg', () => require('pg')], ['bcrypt', () => require('bcrypt')], ['sharp', () => require('sharp')], ['node-pty', () => require('node-pty')], ]; for (const [name, load] of checks) { try { load(); } catch (error) { console.error(`[candidate-runtime-check] failed ${name}: ${error && error.stack ? error.stack : error}`); process.exit(1); } } NODE ) >> "${LOG_FILE}" 2>&1 || return 1 verify_prisma_client_runtime "${backend_dir}" candidate-prisma-check \ >> "${LOG_FILE}" 2>&1 || return 1 ok "Candidate dependencies prepared and verified" } pg_url_component() { local db_url="$1" component="$2" pg_url_uses_supported_prisma_adapter_options "${db_url}" || return 1 printf '%s' "${db_url}" | python3 /dev/fd/3 "${component}" 3<<'PY2' import re import sys from urllib.parse import unquote, urlsplit raw = sys.stdin.read() if (not raw or len(raw.encode("utf-8")) > 128000 or "#" in raw or re.search(r"%(?![0-9A-Fa-f]{2})", raw) or any(ord(char) < 32 or ord(char) == 127 for char in raw)): raise SystemExit(1) try: parsed = urlsplit(raw) _, separator, host_part = parsed.netloc.rpartition("@") host = unquote(parsed.hostname or "", errors="strict") port_number = 5432 if parsed.port is None else parsed.port database = unquote((parsed.path or "").lstrip("/"), errors="strict") user = unquote(parsed.username or "", errors="strict") password = unquote(parsed.password or "", errors="strict") decoded_host_part = unquote(host_part, errors="strict") raw_port_suffix = ( host_part[host_part.find("]") + 1:] if host_part.startswith("[") else f":{host_part.rsplit(':', 1)[1]}" if ":" in host_part else "" ) except (UnicodeDecodeError, ValueError): raise SystemExit(1) identity_or_secret = { "password", "sslpassword", "passfile", "service", "servicefile", "host", "hostaddr", "port", "user", "dbname", "database", } for raw_pair in parsed.query.split("&") if parsed.query else (): if "+" in raw_pair: raise SystemExit(1) raw_key, pair_separator, raw_value = raw_pair.partition("=") if "=" in raw_value: raise SystemExit(1) try: key = unquote(raw_key, errors="strict").lower() value = unquote(raw_value, errors="strict") except UnicodeDecodeError: raise SystemExit(1) if (not pair_separator or not key or key in identity_or_secret or any(ord(char) < 32 or ord(char) == 127 for value_part in (key, value) for char in value_part)): raise SystemExit(1) if (parsed.scheme not in {"postgres", "postgresql"} or parsed.fragment or re.search(r"%(?:23|24|26|2b|2c|2f|3a|3b|3d|3f|40)", parsed.path, re.I) or not parsed.path.startswith("/") or parsed.path.count("/") != 1 or database in {".", ".."} or "/" in database or not separator or parsed.netloc.count("@") != 1 or host_part.startswith("[") or not 1 <= port_number <= 65535 or "," in decoded_host_part or "%" in host or "%" in (parsed.hostname or "") or (raw_port_suffix and not re.fullmatch(r":[1-9][0-9]*", raw_port_suffix)) or re.search(r"[<>\\^|]", host) or host_part != host_part.lower() or not host.isascii() or any(char.isspace() for char in host)): raise SystemExit(1) component = sys.argv[1] values = {"host": host, "port": str(port_number), "database": database, "user": user, "password": password} value = values.get(component, "") if not value or any(ord(char) < 32 or ord(char) == 127 for char in value): raise SystemExit(1) print(value) PY2 } pg_url_uses_public_schema() { local db_url="$1" printf '%s' "${db_url}" | python3 /dev/fd/3 3<<'PY2' import re import sys from urllib.parse import unquote, urlsplit raw = sys.stdin.read() if (not raw or len(raw.encode("utf-8")) > 128000 or "#" in raw or re.search(r"%(?![0-9A-Fa-f]{2})", raw) or any(ord(char) < 32 or ord(char) == 127 for char in raw)): raise SystemExit(1) try: parsed = urlsplit(raw) except ValueError: raise SystemExit(1) if re.search(r"%(?:23|24|26|2b|2c|2f|3a|3b|3d|3f|40)", parsed.path, re.I): raise SystemExit(1) schemas = [] for raw_pair in parsed.query.split("&") if parsed.query else (): if "+" in raw_pair: raise SystemExit(1) raw_key, separator, raw_value = raw_pair.partition("=") if "=" in raw_value: raise SystemExit(1) try: key = unquote(raw_key, errors="strict").lower() value = unquote(raw_value, errors="strict") except UnicodeDecodeError: raise SystemExit(1) if (not separator or any(ord(char) < 32 or ord(char) == 127 for char in value)): raise SystemExit(1) if key == "schema": schemas.append(value) if len(schemas) > 1 or (schemas and schemas[0] != "public"): raise SystemExit(1) PY2 } pg_url_uses_supported_prisma_adapter_options() { local db_url="$1" printf '%s' "${db_url}" | python3 /dev/fd/3 3<<'PY2' import os import re import sys from urllib.parse import unquote, urlsplit raw = sys.stdin.read() if (not raw or len(raw.encode("utf-8")) > 128000 or "#" in raw or re.search(r"%(?![0-9A-Fa-f]{2})", raw) or any(ord(char) < 32 or ord(char) == 127 for char in raw)): raise SystemExit(1) try: parsed = urlsplit(raw) except ValueError: raise SystemExit(1) if re.search(r"%(?:23|24|26|2b|2c|2f|3a|3b|3d|3f|40)", parsed.path, re.I): raise SystemExit(1) security_relevant = { "schema", "connection_limit", "connect_timeout", "pool_timeout", "socket_timeout", "pgbouncer", "statement_cache_size", "max_idle_connection_lifetime", "max_connection_lifetime", "sslmode", "sslcert", "sslkey", "sslrootcert", "sslidentity", "sslpassword", "sslaccept", "channel_binding", "uselibpqcompat", } preserved = { "application_name", "fallback_application_name", "options", "client_encoding", "replication", } allowed = security_relevant | preserved unsupported = { "sslcert", "sslkey", "sslidentity", "sslpassword", "sslaccept", "channel_binding", "uselibpqcompat" } seen = set() values = {} for raw_pair in parsed.query.split("&") if parsed.query else (): if "+" in raw_pair: raise SystemExit(1) raw_key, separator, raw_value = raw_pair.partition("=") if "=" in raw_value: raise SystemExit(1) try: key = unquote(raw_key, errors="strict") value = unquote(raw_value, errors="strict") except UnicodeDecodeError: raise SystemExit(1) if (not separator or any(ord(char) < 32 or ord(char) == 127 for char in value)): raise SystemExit(1) normalized = key.lower() if normalized not in allowed: raise SystemExit(1) if key != normalized or normalized in seen: raise SystemExit(1) seen.add(normalized) if normalized in security_relevant: if normalized in unsupported: raise SystemExit(1) values[normalized] = value if values.get("sslmode") == "prefer": raise SystemExit(1) if "sslmode" in values and values["sslmode"] not in { "disable", "require", "verify-ca", "verify-full" }: raise SystemExit(1) if "sslmode" not in values and (parsed.hostname or "") not in { "localhost", "127.0.0.1", "::1" }: raise SystemExit(1) ssl_root_cert = values.get("sslrootcert") if ssl_root_cert is not None and not os.path.isabs(ssl_root_cert): raise SystemExit(1) if ssl_root_cert is not None and values.get("sslmode") not in { "require", "verify-ca", "verify-full" }: raise SystemExit(1) if values.get("sslmode") in {"verify-ca", "verify-full"} \ and ssl_root_cert is None: raise SystemExit(1) def bounded_integer(name, minimum, maximum, default): value = values.get(name) if value is None: return default if not re.fullmatch(r"[0-9]+", value): raise SystemExit(1) parsed_value = int(value) if not minimum <= parsed_value <= maximum: raise SystemExit(1) return parsed_value bounded_integer("connection_limit", 1, 1000, None) connect_timeout = bounded_integer("connect_timeout", 0, 86400, 5) pool_timeout = bounded_integer("pool_timeout", 0, 86400, 10) bounded_integer("socket_timeout", 0, 86400, None) bounded_integer("max_idle_connection_lifetime", 0, 86400, None) bounded_integer("max_connection_lifetime", 0, 86400, None) bounded_integer("statement_cache_size", 0, 1000000, None) if (("connect_timeout" in values or "pool_timeout" in values) and connect_timeout != pool_timeout): raise SystemExit(1) if values.get("pgbouncer") not in {None, "true", "false"}: raise SystemExit(1) schema = values.get("schema") if schema is not None and (not schema or len(schema.encode("utf-8")) > 63 or any(ord(char) < 32 or ord(char) == 127 for char in schema)): raise SystemExit(1) for tls_path_name in ("sslrootcert",): tls_path = values.get(tls_path_name) if tls_path is not None and (not tls_path or any(ord(char) < 32 or ord(char) == 127 for char in tls_path)): raise SystemExit(1) PY2 } libpq_database_url() { local db_url="$1" pg_url_uses_supported_prisma_adapter_options "${db_url}" || return 1 printf '%s' "${db_url}" | python3 /dev/fd/3 3<<'PY2' import os import re import sys from urllib.parse import unquote, urlsplit, urlunsplit raw = sys.stdin.read() if (not raw or len(raw.encode("utf-8")) > 128000 or "#" in raw or re.search(r"%(?![0-9A-Fa-f]{2})", raw) or any(ord(char) < 32 or ord(char) == 127 for char in raw)): raise SystemExit(1) try: parsed = urlsplit(raw) credentials, separator, host_part = parsed.netloc.rpartition("@") username_raw = credentials.split(":", 1)[0] if separator else "" host = unquote(parsed.hostname or "", errors="strict") user = unquote(parsed.username or "", errors="strict") password = unquote(parsed.password or "", errors="strict") database = unquote((parsed.path or "").lstrip("/"), errors="strict") port = 5432 if parsed.port is None else parsed.port decoded_host_part = unquote(host_part, errors="strict") raw_port_suffix = ( host_part[host_part.find("]") + 1:] if host_part.startswith("[") else f":{host_part.rsplit(':', 1)[1]}" if ":" in host_part else "" ) except (UnicodeDecodeError, ValueError): raise SystemExit(1) if (parsed.scheme not in {"postgres", "postgresql"} or parsed.fragment or re.search(r"%(?:23|24|26|2b|2c|2f|3a|3b|3d|3f|40)", parsed.path, re.I) or not parsed.path.startswith("/") or parsed.path.count("/") != 1 or database in {".", ".."} or "/" in database or not all((separator, username_raw, host, user, password, database)) or parsed.netloc.count("@") != 1 or host_part.startswith("[") or not 1 <= port <= 65535 or "," in decoded_host_part or (raw_port_suffix and not re.fullmatch(r":[1-9][0-9]*", raw_port_suffix)) or "%" in host or "%" in (parsed.hostname or "") or re.search(r"[<>\\^|]", host) or host_part != host_part.lower() or not host.isascii() or any(char.isspace() for char in host) or any(ord(char) < 32 or ord(char) == 127 for value in (host, user, password, database) for char in value)): raise SystemExit(1) # Prisma accepts ORM/pool options that libpq rejects as unknown URI # parameters. Preserve every other query option (notably TLS/connectivity) and # remove only the documented Prisma-only controls for psql/pg_dump postflight. prisma_only = { "schema", "connection_limit", "pool_timeout", "pgbouncer", "statement_cache_size", "socket_timeout", "max_idle_connection_lifetime", "max_connection_lifetime", } preserved = { "connect_timeout", "sslmode", "sslrootcert", "application_name", "fallback_application_name", "options", "client_encoding", "replication", } allowed = prisma_only | preserved identity_or_secret = { "password", "sslpassword", "passfile", "service", "servicefile", "host", "hostaddr", "port", "user", "dbname", "database", } query = [] security = {} seen = set() if parsed.query: for raw_pair in parsed.query.split("&"): if "+" in raw_pair: raise SystemExit(1) raw_key, pair_separator, raw_value = raw_pair.partition("=") if "=" in raw_value: raise SystemExit(1) try: decoded_key = unquote(raw_key, errors="strict") key = decoded_key.lower() decoded_value = unquote(raw_value, errors="strict") except UnicodeDecodeError: raise SystemExit(1) if (not pair_separator or not key or any(ord(char) < 32 or ord(char) == 127 for value_part in (key, decoded_value) for char in value_part)): raise SystemExit(1) if key in identity_or_secret: raise SystemExit(1) if decoded_key != key or key not in allowed or key in seen: raise SystemExit(1) seen.add(key) if key in {"sslmode", "sslrootcert"}: security[key] = decoded_value if key not in prisma_only: query.append(raw_pair) ssl_mode = security.get("sslmode") ssl_root_cert = security.get("sslrootcert") if ssl_mode is not None and ssl_mode not in { "disable", "require", "verify-ca", "verify-full" }: raise SystemExit(1) if ssl_root_cert is not None and ( not ssl_root_cert or not os.path.isabs(ssl_root_cert) or any(ord(char) < 32 or ord(char) == 127 for char in ssl_root_cert) ): raise SystemExit(1) if ssl_root_cert is not None and ssl_mode not in { "require", "verify-ca", "verify-full" }: raise SystemExit(1) if ssl_mode in {"verify-ca", "verify-full"} and ssl_root_cert is None: raise SystemExit(1) # The postflight URI is passed to psql as an explicit --dbname argument, so it # must never carry the password; credentials travel through a 0600 pgpass file # instead. Keep the username (pgpass matching needs it) and the host/port. netloc = f"{username_raw}@{host_part}" print(urlunsplit((parsed.scheme, netloc, parsed.path, "&".join(query), ""))) PY2 } update_pgpass_runner_python() { cat <<'PY2' import os import select import ctypes import re import signal import sys try: parent_pid = int(sys.argv[1]) except (IndexError, ValueError): raise SystemExit(1) libc = ctypes.CDLL(None, use_errno=True) if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: raise SystemExit(1) if os.getppid() != parent_pid: os.kill(os.getpid(), signal.SIGKILL) from urllib.parse import unquote, urlsplit raw_parts = [] raw_size = 0 while True: part = os.read(3, min(65536, 128001 - raw_size)) if not part: break raw_parts.append(part) raw_size += len(part) if raw_size > 128000: raise SystemExit(1) os.close(3) raw_bytes = b"".join(raw_parts) try: raw = raw_bytes.decode("utf-8") except UnicodeDecodeError: raise SystemExit(1) if raw.endswith("\n"): raw = raw[:-1] if (not raw or len(raw.encode("utf-8")) > 128000 or "#" in raw or re.search(r"%(?![0-9A-Fa-f]{2})", raw) or any(ord(char) < 32 or ord(char) == 127 for char in raw)): raise SystemExit(1) try: parsed = urlsplit(raw) _, separator, host_part = parsed.netloc.rpartition("@") host = unquote(parsed.hostname or "", errors="strict") port_number = 5432 if parsed.port is None else parsed.port database = unquote((parsed.path or "").lstrip("/"), errors="strict") user = unquote(parsed.username or "", errors="strict") password = unquote(parsed.password or "", errors="strict") decoded_host_part = unquote(host_part, errors="strict") raw_port_suffix = ( host_part[host_part.find("]") + 1:] if host_part.startswith("[") else f":{host_part.rsplit(':', 1)[1]}" if ":" in host_part else "" ) except (UnicodeDecodeError, ValueError): raise SystemExit(1) identity_or_secret = { "password", "sslpassword", "passfile", "service", "servicefile", "host", "hostaddr", "port", "user", "dbname", "database", } for raw_pair in parsed.query.split("&") if parsed.query else (): if "+" in raw_pair: raise SystemExit(1) raw_key, pair_separator, raw_value = raw_pair.partition("=") if "=" in raw_value: raise SystemExit(1) try: key = unquote(raw_key, errors="strict").lower() value = unquote(raw_value, errors="strict") except UnicodeDecodeError: raise SystemExit(1) if (not pair_separator or not key or key in identity_or_secret or any(ord(char) < 32 or ord(char) == 127 for value_part in (key, value) for char in value_part)): raise SystemExit(1) if (parsed.scheme not in {"postgres", "postgresql"} or parsed.fragment or re.search(r"%(?:23|24|26|2b|2c|2f|3a|3b|3d|3f|40)", parsed.path, re.I) or not parsed.path.startswith("/") or parsed.path.count("/") != 1 or database in {".", ".."} or "/" in database or not separator or parsed.netloc.count("@") != 1 or host_part.startswith("[") or not 1 <= port_number <= 65535 or "," in decoded_host_part or "%" in host or "%" in (parsed.hostname or "") or (raw_port_suffix and not re.fullmatch(r":[1-9][0-9]*", raw_port_suffix)) or re.search(r"[<>\\^|]", host) or host_part != host_part.lower() or not host.isascii() or any(char.isspace() for char in host)): raise SystemExit(1) port = str(port_number) values = (host, port, database, user, password) if not all(values) or any(ord(char) < 32 or ord(char) == 127 for value in values for char in value): raise SystemExit(1) escape = lambda value: value.replace("\\", "\\\\").replace(":", "\\:") try: fd = os.memfd_create("bridgesllm-pgpass", 0) os.fchmod(fd, 0o600) payload = (":".join(escape(value) for value in values) + "\n").encode() if os.write(fd, payload) != len(payload): raise OSError("short pgpass write") os.lseek(fd, 0, os.SEEK_SET) os.set_inheritable(fd, True) except (AttributeError, OSError): raise SystemExit(1) environment = os.environ.copy() environment.pop("DATABASE_URL", None) for key in tuple(environment): if key.startswith("PG") and key != "PGAPPNAME": environment.pop(key, None) environment.pop("BRIDGESLLM_UPDATE_PGPASS_EXEC", None) minimal = environment.pop("BRIDGESLLM_UPDATE_PGPASS_MINIMAL_ENV", "") == "1" if minimal: environment = { "PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C", **({"PGAPPNAME": os.environ["PGAPPNAME"]} if "PGAPPNAME" in os.environ else {}), } environment["PGPASSFILE"] = f"/proc/self/fd/{fd}" os.execvpe(sys.argv[2], sys.argv[2:], environment) PY2 } run_with_update_pgpass() { # Give one libpq operation an anonymous, inherited pgpass inode. There is # never a credential-bearing directory entry to survive SIGKILL: the memfd # exists only while this process tree has it open. local db_url="$1" shift [[ "$#" -gt 0 ]] || return 1 pg_url_uses_supported_prisma_adapter_options "${db_url}" || return 1 local runner_source runner_source="$(update_pgpass_runner_python)" || return 1 local -a clean_environment=( env -u DATABASE_URL -u PGHOST -u PGHOSTADDR -u PGPORT -u PGDATABASE -u PGUSER -u PGPASSWORD -u PGPASSFILE -u PGSERVICE -u PGSERVICEFILE -u PGOPTIONS -u PGSSLMODE -u PGREQUIRESSL -u PGSSLCOMPRESSION -u PGSSLCERT -u PGSSLKEY -u PGSSLROOTCERT -u PGSSLCRL -u PGREQUIREPEER -u PGCHANNELBINDING -u PGTARGETSESSIONATTRS -u PGCONNECT_TIMEOUT -u PGCLIENTENCODING -u PGKRBSRVNAME -u PGGSSLIB -u PGSYSCONFDIR -u PGLOCALEDIR ) if [[ "${BRIDGESLLM_UPDATE_PGPASS_EXEC:-0}" == "1" ]]; then local expected_parent="" current_runner_pid="${BASHPID}" expected_parent="$( awk '/^PPid:[[:space:]]/ { print $2; exit }' \ "/proc/${current_runner_pid}/status" 2>/dev/null )" [[ "${expected_parent}" =~ ^[1-9][0-9]*$ ]] || return 1 exec "${clean_environment[@]}" \ python3 -c "${runner_source}" "${expected_parent}" "$@" \ 3< <(printf '%s' "${db_url}") else "${clean_environment[@]}" \ python3 -c "${runner_source}" "${BASHPID}" "$@" \ 3< <(printf '%s' "${db_url}") fi } run_without_database_authority() { env -u DATABASE_URL -u PGHOST -u PGHOSTADDR -u PGPORT \ -u PGDATABASE -u PGUSER -u PGPASSWORD -u PGPASSFILE \ -u PGSERVICE -u PGSERVICEFILE -u PGOPTIONS -u PGAPPNAME \ -u PGSSLMODE -u PGREQUIRESSL -u PGSSLCOMPRESSION \ -u PGSSLCERT -u PGSSLKEY -u PGSSLROOTCERT -u PGSSLCRL \ -u PGREQUIREPEER -u PGCHANNELBINDING -u PGTARGETSESSIONATTRS \ -u PGCONNECT_TIMEOUT -u PGCLIENTENCODING -u PGKRBSRVNAME \ -u PGGSSLIB -u PGSYSCONFDIR -u PGLOCALEDIR \ "$@" } fsync_update_payload() { # Rollback payloads must survive a crash of the host, not just of the # installer: fsync the file itself and its parent directory entry. local target="$1" python3 - "${target}" <<'PY2' import os import sys path = sys.argv[1] fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) try: os.fsync(fd) finally: os.close(fd) parent = os.path.dirname(path) or "." dfd = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(dfd) finally: os.close(dfd) PY2 } update_database_unsupported_globals_select_sql() { cat <<'SQL' SELECT (SELECT count(*) FROM pg_publication) AS publication_count, ( SELECT count(*) FROM pg_subscription WHERE subdbid = ( SELECT oid FROM pg_database WHERE datname = current_database() ) ) AS subscription_count, (SELECT count(*) FROM pg_cast WHERE oid >= 16384) AS user_cast_count, (SELECT count(*) FROM pg_transform WHERE oid >= 16384) AS user_transform_count, (SELECT count(*) FROM pg_am WHERE oid >= 16384) AS user_access_method_count, (SELECT count(*) FROM pg_event_trigger) AS event_trigger_count, (SELECT count(*) FROM pg_foreign_data_wrapper) AS fdw_count, (SELECT count(*) FROM pg_foreign_server) AS foreign_server_count, ( SELECT count(*) FROM pg_language language WHERE language.lanispl AND NOT ( language.oid < 16384 AND language.lanname = 'plpgsql' AND language.lanpltrusted AND language.laninline <> 0 AND language.lanvalidator <> 0 AND EXISTS ( SELECT 1 FROM pg_depend dependency JOIN pg_extension extension ON extension.oid = dependency.refobjid WHERE dependency.classid = 'pg_language'::regclass AND dependency.objid = language.oid AND dependency.objsubid = 0 AND dependency.refclassid = 'pg_extension'::regclass AND dependency.deptype = 'e' AND extension.extname = 'plpgsql' ) ) ) AS procedural_language_count SQL } update_database_ownership_violations_sql() { cat <<'SQL' SET search_path TO pg_catalog; WITH public_schema AS ( SELECT n.oid, n.nspacl, r.rolname AS owner_name FROM pg_namespace n JOIN pg_roles r ON r.oid = n.nspowner WHERE n.nspname = 'public' ), public_acl AS ( SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee_name, grantor.rolname AS grantor_name, acl.privilege_type, acl.is_grantable FROM public_schema schema CROSS JOIN LATERAL aclexplode(schema.nspacl) acl LEFT JOIN pg_roles grantee ON grantee.oid = acl.grantee JOIN pg_roles grantor ON grantor.oid = acl.grantor ), schema_contract AS ( SELECT CASE WHEN (SELECT count(*) FROM public_schema) = 1 AND (SELECT owner_name FROM public_schema) = current_user AND (SELECT nspacl FROM public_schema) IS NULL THEN 'owner-null' WHEN (SELECT count(*) FROM public_schema) = 1 AND (SELECT owner_name FROM public_schema) = 'pg_database_owner' AND COALESCE(( SELECT array_agg( concat_ws( '|', grantee_name, grantor_name, privilege_type, is_grantable::text ) ORDER BY grantee_name, grantor_name, privilege_type, is_grantable ) FROM public_acl ), ARRAY[]::text[]) = ARRAY[ 'PUBLIC|pg_database_owner|USAGE|false', 'pg_database_owner|pg_database_owner|CREATE|false', 'pg_database_owner|pg_database_owner|USAGE|false' ]::text[] THEN 'pg-database-owner-default' ELSE 'invalid' END AS variant ), unsupported_globals AS ( SQL update_database_unsupported_globals_select_sql cat <<'SQL' ) SELECT ( (SELECT count(*) FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba WHERE d.datname = current_database() AND r.rolname <> current_user) + (SELECT CASE WHEN variant = 'invalid' THEN 1 ELSE 0 END FROM schema_contract) + (SELECT count(*) FROM pg_namespace n WHERE n.nspname <> 'public' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_') + (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace JOIN pg_roles r ON r.oid = c.relowner WHERE n.nspname = 'public' AND (r.rolname <> current_user OR c.relacl IS NOT NULL)) + (SELECT count(*) FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND a.attnum > 0 AND NOT a.attisdropped AND a.attacl IS NOT NULL) + (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace JOIN pg_roles r ON r.oid = p.proowner WHERE n.nspname = 'public' AND (r.rolname <> current_user OR p.proacl IS NOT NULL)) + (SELECT count(*) FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace JOIN pg_roles r ON r.oid = t.typowner WHERE n.nspname = 'public' AND (r.rolname <> current_user OR t.typacl IS NOT NULL)) + (SELECT count(*) FROM pg_collation c JOIN pg_namespace n ON n.oid = c.collnamespace JOIN pg_roles r ON r.oid = c.collowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_conversion c JOIN pg_namespace n ON n.oid = c.connamespace JOIN pg_roles r ON r.oid = c.conowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_operator o JOIN pg_namespace n ON n.oid = o.oprnamespace JOIN pg_roles r ON r.oid = o.oprowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_opclass o JOIN pg_namespace n ON n.oid = o.opcnamespace JOIN pg_roles r ON r.oid = o.opcowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_opfamily o JOIN pg_namespace n ON n.oid = o.opfnamespace JOIN pg_roles r ON r.oid = o.opfowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace JOIN pg_roles r ON r.oid = d.dictowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_ts_config c JOIN pg_namespace n ON n.oid = c.cfgnamespace JOIN pg_roles r ON r.oid = c.cfgowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace JOIN pg_roles r ON r.oid = s.stxowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace JOIN pg_roles r ON r.oid = e.extowner WHERE n.nspname = 'public' AND r.rolname <> current_user) + (SELECT count(*) FROM pg_default_acl d LEFT JOIN pg_namespace n ON n.oid = d.defaclnamespace WHERE d.defaclnamespace = 0 OR n.nspname = 'public') + (SELECT count(*) FROM pg_largeobject_metadata) + (SELECT publication_count + subscription_count + user_cast_count + user_transform_count + user_access_method_count + event_trigger_count + fdw_count + foreign_server_count + procedural_language_count FROM unsupported_globals) )::text || '|' || (SELECT variant FROM schema_contract); SQL } update_database_size_bytes() { local db_url="$1" local connection_url size status=0 connection_url="$(libpq_database_url "${db_url}")" || return 1 size="$(run_with_update_pgpass "${db_url}" \ psql --dbname="${connection_url}" --no-psqlrc \ -v ON_ERROR_STOP=1 -qAt \ -c "SET search_path TO pg_catalog; SELECT pg_catalog.pg_database_size(pg_catalog.current_database())" \ 2>> "${LOG_FILE}")" || status=$? [[ "${status}" -eq 0 ]] || return 1 size="$(tr -d '[:space:]' <<<"${size}")" [[ "${size}" =~ ^[0-9]+$ ]] || return 1 printf '%s\n' "${size}" } update_database_storage_sql() { # Only storage selected by this database is relevant. Cluster-wide # tablespaces can belong to unrelated databases and must not make a Portal # update reserve hundreds of gigabytes on unrelated filesystems. cat <<'SQL' SET search_path TO pg_catalog; WITH current_database_row AS ( SELECT oid, dattablespace FROM pg_database WHERE datname = current_database() ), used_tablespace_oids AS ( SELECT dattablespace AS oid FROM current_database_row UNION SELECT reltablespace FROM pg_class WHERE reltablespace <> 0 UNION SELECT oid FROM pg_tablespace WHERE spcname = NULLIF(current_setting('default_tablespace'), '') ) SELECT json_build_object( 'serverAddress', COALESCE(host(inet_server_addr()), ''), 'serverPort', inet_server_port(), 'backendPid', pg_backend_pid(), -- Settlement identity must come from inside the held session itself. -- application_name is spoofable through URI parameters (application_name= -- or options=-c...), so the settlement probe binds to pid + backend_start. 'backendStart', ( SELECT extract(epoch FROM backend_start)::text FROM pg_stat_activity WHERE pid = pg_backend_pid() ), 'databaseName', current_database(), 'databaseOid', (SELECT oid::bigint FROM current_database_row), -- Storage paths are host authority, not database-role authority. Ordinary -- database owners cannot read data_directory or arbitrary tablespace -- locations on PostgreSQL 16. The canonical proof derives those paths from -- the /proc-attested local postmaster and its pg_tblspc symlinks instead. 'dataDirectory', '', 'walDirectory', '', 'defaultTablespace', current_setting('default_tablespace'), 'defaultTablespaceOid', COALESCE( ( SELECT oid::bigint FROM pg_tablespace WHERE spcname = NULLIF(current_setting('default_tablespace'), '') ), 0 ), 'tempTablespaces', current_setting('temp_tablespaces'), 'tablespaces', '[]'::json, 'tablespaceOids', COALESCE( (SELECT json_agg(oid::bigint ORDER BY oid) FROM used_tablespace_oids), '[]'::json ) )::text; SQL } update_database_storage_json() { # This low-level query is also exercised against real PostgreSQL in CI. The # canonical topology function below keeps the same backend alive long enough # to prove that the SQL session is a child of the observed local postmaster. local db_url="$1" local connection_url result status=0 operation_log="${LOG_FILE}" if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && -n "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" ]]; then operation_log="${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT}/database-topology.log" fi connection_url="$(libpq_database_url "${db_url}")" || return 1 result="$( update_database_storage_sql \ | run_with_update_pgpass "${db_url}" \ psql --dbname="${connection_url}" --no-psqlrc \ -v ON_ERROR_STOP=1 -qAt 2>> "${operation_log}" )" || status=$? [[ "${status}" -eq 0 && -n "${result}" && "${#result}" -le 65536 ]] \ || return 1 printf '%s\n' "${result}" } canonical_update_database_topology() { # Return one canonical JSON record that binds the exact PostgreSQL system, # database, listening endpoint, and every filesystem this database can use. # In production an interactive psql session remains open while /proc is # inspected, proving that the reported session belongs to the attested local # postmaster rather than trusting a loopback address alone. The client is # then released explicitly and its exact backend must disappear. local db_url="$1" local host port database user connection_url work_dir fifo output_file local proof_file app_name remaining="" local psql_pid="" psql_input_fd="" backend_pid="" raw="" proof_output="" local backend_identity="" backend_start="" local status=0 proof_status=0 settlement_status=0 source_only=false host="$(pg_url_component "${db_url}" host)" || return 1 port="$(pg_url_component "${db_url}" port)" || return 1 database="$(pg_url_component "${db_url}" database)" || return 1 user="$(pg_url_component "${db_url}" user)" || return 1 connection_url="$(libpq_database_url "${db_url}")" || return 1 if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && -n "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" ]]; then source_only=true fi work_dir="$(mktemp -d "${TMPDIR:-/tmp}/blp-update-dbtopology.XXXXXX")" \ || return 1 chmod 0700 "${work_dir}" fifo="${work_dir}/topology.input" output_file="${work_dir}/topology.out" proof_file="${work_dir}/topology.proof" app_name="bridgesllm_a009_topology_${BASHPID}_${RANDOM}" if ${source_only}; then raw="$(update_database_storage_json "${db_url}")" || status=$? else mkfifo -m 0600 "${fifo}" || status=$? if [[ "${status}" -eq 0 ]]; then BRIDGESLLM_UPDATE_PGPASS_EXEC=1 \ PGAPPNAME="${app_name}" \ run_with_update_pgpass "${db_url}" stdbuf -oL psql \ --dbname="${connection_url}" --no-psqlrc -q \ -v ON_ERROR_STOP=1 -tA \ < "${fifo}" \ > "${output_file}" 2>> "${LOG_FILE}" & psql_pid=$! exec {psql_input_fd}>"${fifo}" || status=$? fi if [[ "${status}" -eq 0 ]]; then update_database_storage_sql >&"${psql_input_fd}" || status=$? fi if [[ "${status}" -eq 0 ]]; then local attempt for attempt in {1..100}; do raw="$(sed -n '1p' "${output_file}" 2>/dev/null || true)" [[ -n "${raw}" ]] && break kill -0 "${psql_pid}" 2>/dev/null || break sleep 0.05 done [[ -n "${raw}" && "${#raw}" -le 65536 \ && -n "${psql_pid}" && -d "/proc/${psql_pid}" ]] || status=1 if [[ "${status}" -eq 0 ]]; then backend_identity="$( python3 - "${raw}" <<'PY2' import json import re import sys value = json.loads(sys.argv[1]) pid = value.get("backendPid") start = value.get("backendStart") if ( isinstance(pid, bool) or not isinstance(pid, int) or pid <= 1 or not isinstance(start, str) or re.fullmatch(r"[0-9]{1,20}(\.[0-9]{1,9})?", start) is None ): raise SystemExit(1) print(pid) print(start) PY2 )" || status=$? if [[ "${status}" -eq 0 ]]; then backend_pid="${backend_identity%%$'\n'*}" backend_start="${backend_identity#*$'\n'}" [[ -n "${backend_pid}" && -n "${backend_start}" \ && "${backend_pid}" != "${backend_start}" ]] || status=1 fi fi fi fi if [[ "${status}" -eq 0 ]]; then python3 - "${raw}" "${host}" "${port}" "${database}" \ "${source_only}" "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" \ > "${proof_file}" <<'PY2' \ || proof_status=$? import hashlib import ipaddress import json import os import posixpath import re import stat import subprocess import sys import tempfile import ctypes raw, expected_host, expected_port, expected_database, source_only_text, test_root = ( sys.argv[1:7] ) source_only = source_only_text == "true" try: observed = json.loads(raw) except ValueError: raise SystemExit(1) required = { "serverAddress", "serverPort", "backendPid", "backendStart", "databaseName", "databaseOid", "dataDirectory", "walDirectory", "defaultTablespace", "defaultTablespaceOid", "tempTablespaces", "tablespaces", "tablespaceOids", } if ( not isinstance(observed, dict) or set(observed) != required or not isinstance(observed["serverAddress"], str) or isinstance(observed["serverPort"], bool) or not isinstance(observed["serverPort"], int) or isinstance(observed["backendPid"], bool) or not isinstance(observed["backendPid"], int) or not isinstance(observed["backendStart"], str) or re.fullmatch( r"[0-9]{1,20}(\.[0-9]{1,9})?", observed["backendStart"] ) is None or not isinstance(observed["databaseName"], str) or isinstance(observed["databaseOid"], bool) or not isinstance(observed["databaseOid"], int) or not isinstance(observed["dataDirectory"], str) or not isinstance(observed["walDirectory"], str) or not isinstance(observed["defaultTablespace"], str) or isinstance(observed["defaultTablespaceOid"], bool) or not isinstance(observed["defaultTablespaceOid"], int) or not isinstance(observed["tempTablespaces"], str) or not isinstance(observed["tablespaces"], list) or len(observed["tablespaces"]) > 32 or not all(isinstance(path, str) for path in observed["tablespaces"]) or not isinstance(observed["tablespaceOids"], list) or len(observed["tablespaceOids"]) > 32 or not all( not isinstance(oid, bool) and isinstance(oid, int) for oid in observed["tablespaceOids"] ) ): raise SystemExit(1) try: address = ipaddress.ip_address(observed["serverAddress"]) except ValueError: raise SystemExit(1) if ( "/" in observed["serverAddress"] or not 1 <= observed["serverPort"] <= 65535 or observed["databaseName"] != expected_database or not 1 <= observed["databaseOid"] <= 4_294_967_295 or observed["backendPid"] <= 1 or observed["tempTablespaces"].strip() ): raise SystemExit(1) if source_only: host_address = address if ( not address.is_loopback or observed["serverPort"] != int(expected_port) ): raise SystemExit(1) else: try: host_address = ipaddress.ip_address(expected_host) except ValueError: raise SystemExit(1) if not host_address.is_loopback: raise SystemExit(1) if ( observed["dataDirectory"] or observed["walDirectory"] or observed["tablespaces"] ): raise SystemExit(1) tablespace_oids = [] for oid in observed["tablespaceOids"]: if not 1 <= oid <= 4_294_967_295: raise SystemExit(1) tablespace_oids.append(oid) if ( len(tablespace_oids) != len(set(tablespace_oids)) or tablespace_oids != sorted(tablespace_oids) or (not source_only and not tablespace_oids) ): raise SystemExit(1) default_tablespace = observed["defaultTablespace"].strip() default_tablespace_oid = observed["defaultTablespaceOid"] if ( not 0 <= default_tablespace_oid <= 4_294_967_295 or ( not default_tablespace and default_tablespace_oid != 0 ) or ( default_tablespace and ( default_tablespace_oid == 0 or default_tablespace_oid not in tablespace_oids ) ) ): raise SystemExit(1) def safe_directory(path: str) -> str: if ( not path or len(os.fsencode(path)) > 4096 or not os.path.isabs(path) or any(ord(character) < 32 or ord(character) == 127 for character in path) ): raise SystemExit(1) resolved = os.path.realpath(path) if not os.path.isabs(resolved) or not os.path.isdir(resolved): raise SystemExit(1) details = os.stat(resolved) if not stat.S_ISDIR(details.st_mode) or details.st_mode & 0o022: raise SystemExit(1) if source_only: root = os.path.realpath(test_root) if not root or os.path.commonpath((root, resolved)) != root: raise SystemExit(1) elif details.st_uid == 0: raise SystemExit(1) return resolved if source_only: data_directory = safe_directory(observed["dataDirectory"]) wal_directory = safe_directory(observed["walDirectory"]) tablespaces = sorted( {safe_directory(path) for path in observed["tablespaces"]} ) data_details = os.stat(data_directory) system_identifier = os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_DATABASE_SYSTEM_IDENTIFIER", "7612345678901234567", ) else: def process_identity(pid: int) -> dict: process_root = f"/proc/{pid}" details = os.stat(process_root) executable_path = os.path.join(process_root, "exe") executable = os.readlink(executable_path) executable_details = os.stat(executable_path) process_stat = open( os.path.join(process_root, "stat"), "r", encoding="ascii" ).read() suffix = process_stat.rsplit(")", 1) if len(suffix) != 2: raise SystemExit(1) fields = suffix[1].strip().split() if ( len(fields) < 20 or fields[0] in {"Z", "X", "x"} or not fields[1].isdigit() or not fields[19].isdigit() or not os.path.isabs(executable) or executable.endswith(" (deleted)") or any(ord(character) < 32 for character in executable) ): raise SystemExit(1) status = open( os.path.join(process_root, "status"), "r", encoding="ascii" ).read().splitlines() nspid_lines = [line for line in status if line.startswith("NSpid:")] if len(nspid_lines) != 1: raise SystemExit(1) nspids = nspid_lines[0].split()[1:] if not nspids or not all(value.isdigit() for value in nspids): raise SystemExit(1) namespaces = tuple( os.stat(os.path.join(process_root, "ns", name)).st_ino for name in ("pid", "mnt", "net") ) return { "pid": pid, "details": details, "parentPid": int(fields[1]), "startTime": int(fields[19]), "executable": executable, "executableDevice": executable_details.st_dev, "executableInode": executable_details.st_ino, "namespacePids": tuple(int(value) for value in nspids), "namespaces": namespaces, } def same_postgres_process_pair(backend: dict, postmaster: dict) -> bool: return ( backend["parentPid"] == postmaster["pid"] and backend["details"].st_uid == postmaster["details"].st_uid and backend["details"].st_gid == postmaster["details"].st_gid and postmaster["details"].st_uid != 0 and backend["startTime"] > 0 and postmaster["startTime"] > 0 and backend["namespaces"] == postmaster["namespaces"] and os.path.basename(backend["executable"]) in {"postgres", "postmaster"} and os.path.basename(postmaster["executable"]) in {"postgres", "postmaster"} and ( backend["executableDevice"], backend["executableInode"], ) == ( postmaster["executableDevice"], postmaster["executableInode"], ) ) docker_records_cache = {"loaded": False, "records": []} def running_docker_records() -> list: if docker_records_cache["loaded"]: return docker_records_cache["records"] docker = "/usr/bin/docker" details = os.lstat(docker) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != 0 or details.st_mode & 0o022 or not details.st_mode & 0o111 ): raise SystemExit(1) listed = subprocess.run( [docker, "ps", "--quiet", "--no-trunc"], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, text=True, timeout=10, check=False, ) if listed.returncode != 0 or len(listed.stdout) > 16 * 1024: raise SystemExit(1) identifiers = [line.strip() for line in listed.stdout.splitlines()] if ( len(identifiers) > 128 or any(re.fullmatch(r"[a-f0-9]{64}", value) is None for value in identifiers) ): raise SystemExit(1) if not identifiers: docker_records_cache["loaded"] = True return [] inspected = subprocess.run( [docker, "inspect", *identifiers], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, text=True, timeout=20, check=False, ) if inspected.returncode != 0 or len(inspected.stdout) > 8 * 1024 * 1024: raise SystemExit(1) try: records = json.loads(inspected.stdout) except ValueError: raise SystemExit(1) if not isinstance(records, list) or len(records) != len(identifiers): raise SystemExit(1) docker_records_cache["loaded"] = True docker_records_cache["records"] = records return records def container_for_namespaces(namespaces: tuple) -> list: matches = [] for record in running_docker_records(): if not isinstance(record, dict): raise SystemExit(1) state = record.get("State") if not isinstance(state, dict): continue init_pid = state.get("Pid") if ( state.get("Running") is not True or isinstance(init_pid, bool) or not isinstance(init_pid, int) or init_pid <= 1 ): continue try: init_namespaces = tuple( os.stat(f"/proc/{init_pid}/ns/{name}").st_ino for name in ("pid", "mnt", "net") ) except OSError: continue if init_namespaces == namespaces: matches.append(record) return matches self_namespaces = tuple( os.stat(f"/proc/self/ns/{name}").st_ino for name in ("pid", "mnt", "net") ) candidates = [] for entry in os.listdir("/proc"): if not entry.isdigit(): continue try: backend = process_identity(int(entry)) except (OSError, ValueError, SystemExit): continue if backend["namespacePids"][-1] != observed["backendPid"]: continue try: postmaster = process_identity(backend["parentPid"]) except (OSError, ValueError, SystemExit): continue if not same_postgres_process_pair(backend, postmaster): continue container = None if backend["namespaces"] == self_namespaces: if ( not address.is_loopback or address != host_address or observed["serverPort"] != int(expected_port) ): continue else: containers = container_for_namespaces(backend["namespaces"]) if len(containers) != 1: continue container = containers[0] network = container.get("NetworkSettings") ports = network.get("Ports") if isinstance(network, dict) else None bindings = ( ports.get(f'{observed["serverPort"]}/tcp') if isinstance(ports, dict) else None ) if not isinstance(bindings, list) or len(bindings) != 1: continue exact_bindings = [] for binding in bindings: if not isinstance(binding, dict): continue try: binding_address = ipaddress.ip_address( binding.get("HostIp", "") ) binding_port = int(binding.get("HostPort", "")) except (ValueError, TypeError): continue if ( binding_address == host_address and binding_address.is_loopback and binding_port == int(expected_port) ): exact_bindings.append(binding) networks = network.get("Networks") container_addresses = set() if isinstance(networks, dict): for value in networks.values(): if not isinstance(value, dict): continue for field in ("IPAddress", "GlobalIPv6Address"): candidate_address = value.get(field) if not candidate_address: continue try: container_addresses.add( ipaddress.ip_address(candidate_address) ) except ValueError: raise SystemExit(1) if len(exact_bindings) != 1 or address not in container_addresses: continue candidates.append((backend, postmaster, container)) if len(candidates) != 1: raise SystemExit(1) backend, postmaster, container_record = candidates[0] backend_pid = backend["pid"] postmaster_pid = postmaster["pid"] backend_details = backend["details"] postmaster_details = postmaster["details"] backend_binary = backend["executable"] postmaster_binary = postmaster["executable"] backend_start_time = backend["startTime"] postmaster_start_time = postmaster["startTime"] def unsafe_writable_directory(mode: int) -> bool: # Group/other-writable ancestors let a local user replace a path # component, except when the sticky bit forbids renaming or deleting # entries owned by someone else (the official PostgreSQL images ship # /var/lib/postgresql as 1777, exactly like /tmp). return bool(mode & 0o022) and not mode & 0o1000 def clean_namespace_path(path: str) -> str: if ( not isinstance(path, str) or not path or len(os.fsencode(path)) > 4096 or not posixpath.isabs(path) or any(ord(character) < 32 or ord(character) == 127 for character in path) ): raise SystemExit(1) normalized = posixpath.normpath(path) if normalized == "/" or not normalized.startswith("/"): raise SystemExit(1) return normalized namespace_root = f"/proc/{postmaster_pid}/root" def namespace_access_path(path: str) -> str: return namespace_root + clean_namespace_path(path) def resolve_namespace_directory(path: str): requested = clean_namespace_path(path) pending = requested.lstrip("/").split("/") current = "/" # The container root itself is an ancestor: if it is writable by an # unprivileged container user, root-owned top-level entries such as # /var can be replaced after proof. The trailing slash makes lstat # traverse the /proc//root magic link to the directory. root_access_path = namespace_root + "/" root_details = os.lstat(root_access_path) if ( not stat.S_ISDIR(root_details.st_mode) or unsafe_writable_directory(root_details.st_mode) or root_details.st_uid not in {0, postmaster_details.st_uid} ): raise SystemExit(1) chain = [ ( "directory", root_access_path, root_details.st_dev, root_details.st_ino, "", root_details.st_uid, root_details.st_gid, stat.S_IMODE(root_details.st_mode), ) ] followed_links = 0 while pending: component = pending.pop(0) candidate = posixpath.join(current, component) access_path = namespace_access_path(candidate) details = os.lstat(access_path) if stat.S_ISLNK(details.st_mode): if ( details.st_nlink != 1 or details.st_uid not in {0, postmaster_details.st_uid} ): raise SystemExit(1) target = os.readlink(access_path) chain.append( ( "symlink", access_path, details.st_dev, details.st_ino, target, details.st_uid, details.st_gid, stat.S_IMODE(details.st_mode), ) ) followed_links += 1 if followed_links > 8: raise SystemExit(1) unresolved = ( target if posixpath.isabs(target) else posixpath.join(posixpath.dirname(candidate), target) ) if pending: unresolved = posixpath.join(unresolved, *pending) resolved = clean_namespace_path(unresolved) pending = resolved.lstrip("/").split("/") current = "/" continue if ( not stat.S_ISDIR(details.st_mode) or unsafe_writable_directory(details.st_mode) or details.st_uid not in {0, postmaster_details.st_uid} ): raise SystemExit(1) chain.append( ( "directory", access_path, details.st_dev, details.st_ino, "", details.st_uid, details.st_gid, stat.S_IMODE(details.st_mode), ) ) current = candidate if current == "/": raise SystemExit(1) final_details = os.lstat(namespace_access_path(current)) if not stat.S_ISDIR(final_details.st_mode): raise SystemExit(1) return current, final_details, tuple(chain) def recheck_namespace_chain(chain): for kind, access_path, device, inode, target, uid, gid, mode in chain: details = os.lstat(access_path) if ( ( details.st_dev, details.st_ino, details.st_uid, details.st_gid, stat.S_IMODE(details.st_mode), ) != (device, inode, uid, gid, mode) ): raise SystemExit(1) if kind == "symlink": if ( not stat.S_ISLNK(details.st_mode) or details.st_nlink != 1 or os.readlink(access_path) != target ): raise SystemExit(1) elif kind == "directory": if ( not stat.S_ISDIR(details.st_mode) or unsafe_writable_directory(details.st_mode) or details.st_uid not in {0, postmaster_details.st_uid} ): raise SystemExit(1) else: raise SystemExit(1) def capture_host_directory_chain(path: str, anchor: str = ""): if ( not path or len(os.fsencode(path)) > 4096 or not os.path.isabs(path) or any(ord(character) < 32 or ord(character) == 127 for character in path) ): raise SystemExit(1) resolved = os.path.realpath(path) if not os.path.isabs(resolved) or not os.path.isdir(resolved): raise SystemExit(1) if anchor: resolved_anchor = os.path.realpath(anchor) if ( not os.path.isabs(resolved_anchor) or os.path.commonpath((resolved_anchor, resolved)) != resolved_anchor ): raise SystemExit(1) # The anchor only constrains containment. Ancestors are attested # from the filesystem root either way: a writable ancestor of a # Docker mount source lets a local user rename the source directory # out from under the recorded chain after proof. current = os.path.sep paths = [current] for component in resolved.split(os.sep)[1:]: if component in {"", ".", ".."}: raise SystemExit(1) current = os.path.join(current, component) paths.append(current) chain = [] for component_path in paths: details = os.lstat(component_path) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or unsafe_writable_directory(details.st_mode) or details.st_uid not in {0, postmaster_details.st_uid} ): raise SystemExit(1) chain.append( ( component_path, details.st_dev, details.st_ino, details.st_uid, details.st_gid, stat.S_IMODE(details.st_mode), ) ) if chain[-1][3] != postmaster_details.st_uid: raise SystemExit(1) return resolved, tuple(chain) def recheck_host_chain(chain): for path, device, inode, uid, gid, mode in chain: details = os.lstat(path) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or unsafe_writable_directory(details.st_mode) or ( details.st_dev, details.st_ino, details.st_uid, details.st_gid, stat.S_IMODE(details.st_mode), ) != (device, inode, uid, gid, mode) ): raise SystemExit(1) def safe_mount_source(path: str) -> str: if ( not path or len(os.fsencode(path)) > 4096 or not os.path.isabs(path) or any(ord(character) < 32 or ord(character) == 127 for character in path) ): raise SystemExit(1) resolved = os.path.realpath(path) if not os.path.isabs(resolved) or not os.path.isdir(resolved): raise SystemExit(1) details = os.stat(resolved) if ( not stat.S_ISDIR(details.st_mode) or unsafe_writable_directory(details.st_mode) or details.st_uid not in {0, postmaster_details.st_uid} ): raise SystemExit(1) return resolved container_mounts = [] if container_record is not None: raw_mounts = container_record.get("Mounts") if not isinstance(raw_mounts, list) or len(raw_mounts) > 128: raise SystemExit(1) for mount in raw_mounts: if not isinstance(mount, dict): raise SystemExit(1) if ( mount.get("Type") not in {"bind", "volume"} or mount.get("RW") is not True ): continue source = mount.get("Source") destination = mount.get("Destination") if ( not isinstance(source, str) or not os.path.isabs(source) or any(ord(character) < 32 or ord(character) == 127 for character in source) ): raise SystemExit(1) container_mounts.append( (clean_namespace_path(destination), source) ) def persistent_directory(path: str): namespace_path, namespace_details, namespace_chain = ( resolve_namespace_directory(path) ) if container_record is None: host_path, host_chain = capture_host_directory_chain( namespace_path ) else: matches = [] for destination, source in container_mounts: if ( namespace_path == destination or namespace_path.startswith(destination + "/") ): relative = posixpath.relpath(namespace_path, destination) matches.append( ( len(destination), source, relative, ) ) if not matches: raise SystemExit(1) matches.sort(key=lambda value: value[0], reverse=True) if len(matches) > 1 and matches[0][0] == matches[1][0]: raise SystemExit(1) _, selected_source, selected_relative = matches[0] mount_root = safe_mount_source(selected_source) selected_path = ( mount_root if selected_relative == "." else os.path.join( mount_root, *selected_relative.split("/") ) ) host_path, host_chain = capture_host_directory_chain( selected_path, mount_root ) host_details = os.lstat(host_path) if ( host_details.st_dev != namespace_details.st_dev or host_details.st_ino != namespace_details.st_ino ): raise SystemExit(1) return namespace_path, host_path, namespace_chain, host_chain # The backend is held by the open interactive client while this proof # runs. Its exact host # parent is the live local postmaster. The cwd is interpreted inside that # process's mount namespace and, for Docker, mapped only through the exact # inspected persistent bind/volume mount. namespace_data_directory = clean_namespace_path( os.readlink(f"/proc/{postmaster_pid}/cwd") ) data_requested_path = namespace_data_directory ( namespace_data_directory, data_directory, data_chain, data_host_chain, ) = ( persistent_directory(data_requested_path) ) wal_requested_path = posixpath.join( namespace_data_directory, "pg_wal" ) ( namespace_wal_directory, wal_directory, wal_chain, wal_host_chain, ) = ( persistent_directory(wal_requested_path) ) tablespace_root_requested_path = posixpath.join( namespace_data_directory, "pg_tblspc" ) ( namespace_tablespace_root, tablespace_root_details, tablespace_root_chain, ) = ( resolve_namespace_directory( tablespace_root_requested_path ) ) data_details = os.stat(data_directory) if ( data_details.st_uid != postmaster_details.st_uid or os.stat(wal_directory).st_uid != data_details.st_uid or tablespace_root_details.st_uid != data_details.st_uid ): raise SystemExit(1) tablespaces = [] tablespace_links = [] for oid in tablespace_oids: if oid in {1663, 1664}: # pg_default and pg_global continue link_namespace_path = posixpath.join( namespace_tablespace_root, str(oid) ) link_path = namespace_access_path(link_namespace_path) link_details = os.lstat(link_path) if ( not stat.S_ISLNK(link_details.st_mode) or link_details.st_uid != data_details.st_uid or link_details.st_nlink != 1 ): raise SystemExit(1) target = os.readlink(link_path) target_namespace_path = clean_namespace_path( target if posixpath.isabs(target) else posixpath.join( posixpath.dirname(link_namespace_path), target ) ) ( resolved_namespace_path, tablespace, target_chain, target_host_chain, ) = persistent_directory(target_namespace_path) if os.stat(tablespace).st_uid != data_details.st_uid: raise SystemExit(1) tablespaces.append(tablespace) tablespace_links.append( ( link_path, link_details.st_dev, link_details.st_ino, target, target_namespace_path, resolved_namespace_path, tablespace, target_chain, target_host_chain, ) ) tablespaces = sorted(set(tablespaces)) version_path = os.path.join(data_directory, "PG_VERSION") postmaster_path = os.path.join(data_directory, "postmaster.pid") postmaster_pidfile_details = os.lstat(postmaster_path) for required_path in (version_path, postmaster_path): details = os.lstat(required_path) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != data_details.st_uid or details.st_nlink != 1 ): raise SystemExit(1) lines = open(postmaster_path, "r", encoding="ascii").read().splitlines() if ( len(lines) < 4 or not lines[0].isdigit() or int(lines[0]) != postmaster["namespacePids"][-1] or clean_namespace_path(lines[1]) != namespace_data_directory or not lines[3].isdigit() or int(lines[3]) != observed["serverPort"] ): raise SystemExit(1) # Re-read both /proc identities after path resolution. A PID reuse or # process replacement during the proof must fail closed. backend_after = process_identity(backend_pid) postmaster_after = process_identity(postmaster_pid) if ( not same_postgres_process_pair(backend_after, postmaster_after) or backend_after["startTime"] != backend_start_time or postmaster_after["startTime"] != postmaster_start_time or ( backend_after["details"].st_dev, backend_after["details"].st_ino, ) != (backend_details.st_dev, backend_details.st_ino) or ( postmaster_after["details"].st_dev, postmaster_after["details"].st_ino, ) != (postmaster_details.st_dev, postmaster_details.st_ino) or ( backend_after["executableDevice"], backend_after["executableInode"], ) != ( backend["executableDevice"], backend["executableInode"], ) or ( postmaster_after["executableDevice"], postmaster_after["executableInode"], ) != ( postmaster["executableDevice"], postmaster["executableInode"], ) ): raise SystemExit(1) if not os.path.isabs(postmaster_binary): raise SystemExit(1) postgres_bindir = clean_namespace_path( posixpath.dirname(postmaster_binary) ) bindir_access_path = namespace_access_path(postgres_bindir) bindir_details = os.stat(bindir_access_path) if ( not stat.S_ISDIR(bindir_details.st_mode) or bindir_details.st_mode & 0o022 ): raise SystemExit(1) pg_controldata = posixpath.join( postgres_bindir, "pg_controldata" ) tool_access_path = namespace_access_path(pg_controldata) tool_details = os.lstat(tool_access_path) if ( not stat.S_ISREG(tool_details.st_mode) or stat.S_ISLNK(tool_details.st_mode) or tool_details.st_nlink != 1 or tool_details.st_mode & (stat.S_ISUID | stat.S_ISGID) or tool_details.st_mode & 0o022 or not tool_details.st_mode & 0o111 ): raise SystemExit(1) tool_fd = os.open( tool_access_path, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) mount_namespace_fd = -1 container_root_fd = -1 try: opened_tool = os.fstat(tool_fd) if ( opened_tool.st_dev != tool_details.st_dev or opened_tool.st_ino != tool_details.st_ino or not stat.S_ISREG(opened_tool.st_mode) or opened_tool.st_nlink != 1 or opened_tool.st_mode & (stat.S_ISUID | stat.S_ISGID) or opened_tool.st_mode & 0o022 ): raise SystemExit(1) if container_record is not None: mount_namespace_fd = os.open( f"/proc/{postmaster_pid}/ns/mnt", os.O_RDONLY | getattr(os, "O_CLOEXEC", 0), ) container_root_fd = os.open( namespace_root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), ) # pg_controldata is discovered from the exact /proc-attested # postmaster binary, but it is still PostgreSQL-controlled input. For # Docker, enter only the already-pinned mount namespace and chroot to # its exact root so the matching dynamic loader/libraries are visible. # Drop to the postmaster identity before exec in both modes. def exec_verified_tool_as_postmaster(): libc = ctypes.CDLL(None, use_errno=True) if container_record is not None: if libc.setns(mount_namespace_fd, 0) != 0: raise OSError( ctypes.get_errno(), "PostgreSQL mount setns failed" ) os.fchdir(container_root_fd) os.chroot(".") os.chdir("/") if libc.prctl(38, 1, 0, 0, 0) != 0: # PR_SET_NO_NEW_PRIVS raise OSError(ctypes.get_errno(), "PR_SET_NO_NEW_PRIVS failed") os.setgroups([]) os.setgid(postmaster_details.st_gid) os.setuid(postmaster_details.st_uid) os.umask(0o077) if ( os.getuid() != postmaster_details.st_uid or os.geteuid() != postmaster_details.st_uid or os.getgid() != postmaster_details.st_gid or os.getegid() != postmaster_details.st_gid or os.getgroups() ): raise OSError("could not drop to PostgreSQL process identity") argv_values = [ os.fsencode(pg_controldata), os.fsencode(command_data_directory), ] environment_values = [ b"PATH=/usr/bin:/bin", b"LANG=C", b"LC_ALL=C", ] argv = (ctypes.c_char_p * (len(argv_values) + 1))( *argv_values, None ) environment = ( ctypes.c_char_p * (len(environment_values) + 1) )(*environment_values, None) try: execveat = libc.execveat except AttributeError: # musl omits the execveat wrapper but provides fexecve. This # branch is used only when already running in the postmaster's # own PID/mount namespace, where its procfs can resolve the # opened fd without crossing a descendant PID namespace. if container_record is not None: raise OSError("execveat is unavailable for Docker proof") libc.fexecve.argtypes = [ ctypes.c_int, ctypes.POINTER(ctypes.c_char_p), ctypes.POINTER(ctypes.c_char_p), ] libc.fexecve.restype = ctypes.c_int result = libc.fexecve(tool_fd, argv, environment) else: execveat.argtypes = [ ctypes.c_int, ctypes.c_char_p, ctypes.POINTER(ctypes.c_char_p), ctypes.POINTER(ctypes.c_char_p), ctypes.c_int, ] execveat.restype = ctypes.c_int result = execveat( tool_fd, b"", argv, environment, 0x1000, # AT_EMPTY_PATH: execute the verified fd. ) if result != 0: raise OSError( ctypes.get_errno(), "verified pg_controldata exec failed" ) command_data_directory = ( namespace_data_directory if container_record is not None else data_directory ) passed_fds = [tool_fd] if container_record is not None: passed_fds.extend((mount_namespace_fd, container_root_fd)) completed = subprocess.run( [pg_controldata, command_data_directory], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env={ "PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C", }, text=True, timeout=10, check=False, close_fds=True, pass_fds=tuple(passed_fds), preexec_fn=exec_verified_tool_as_postmaster, ) finally: if mount_namespace_fd >= 0: os.close(mount_namespace_fd) if container_root_fd >= 0: os.close(container_root_fd) os.close(tool_fd) if completed.returncode != 0 or len(completed.stdout) > 256 * 1024: raise SystemExit(1) match = re.search( r"^Database system identifier:\s*([0-9]+)\s*$", completed.stdout, flags=re.MULTILINE, ) if match is None: raise SystemExit(1) system_identifier = match.group(1) # Recheck every mutable namespace link and the exact process/pidfile # identities after pg_controldata. Mid-proof storage swaps fail closed. postmaster_final = process_identity(postmaster_pid) backend_final = process_identity(backend_pid) pidfile_final = os.lstat(postmaster_path) if ( not same_postgres_process_pair(backend_final, postmaster_final) or backend_final["startTime"] != backend_start_time or postmaster_final["startTime"] != postmaster_start_time or (pidfile_final.st_dev, pidfile_final.st_ino) != ( postmaster_pidfile_details.st_dev, postmaster_pidfile_details.st_ino, ) ): raise SystemExit(1) ( current_data_namespace, current_data_host, current_data_chain, current_data_host_chain, ) = ( persistent_directory(data_requested_path) ) ( current_wal_namespace, current_wal_host, current_wal_chain, current_wal_host_chain, ) = ( persistent_directory(wal_requested_path) ) ( current_tablespace_root, _current_tablespace_root_details, current_tablespace_root_chain, ) = resolve_namespace_directory(tablespace_root_requested_path) if ( current_data_namespace != namespace_data_directory or current_data_host != data_directory or current_data_chain != data_chain or current_data_host_chain != data_host_chain or current_wal_namespace != namespace_wal_directory or current_wal_host != wal_directory or current_wal_chain != wal_chain or current_wal_host_chain != wal_host_chain or current_tablespace_root != namespace_tablespace_root or current_tablespace_root_chain != tablespace_root_chain ): raise SystemExit(1) recheck_namespace_chain(data_chain) recheck_namespace_chain(wal_chain) recheck_namespace_chain(tablespace_root_chain) recheck_host_chain(data_host_chain) recheck_host_chain(wal_host_chain) for ( link_path, link_device, link_inode, link_target, target_requested_path, namespace_target, host_target, target_chain, target_host_chain, ) in tablespace_links: current_link = os.lstat(link_path) if ( not stat.S_ISLNK(current_link.st_mode) or current_link.st_dev != link_device or current_link.st_ino != link_inode or os.readlink(link_path) != link_target ): raise SystemExit(1) ( current_namespace_target, current_host_target, current_target_chain, current_target_host_chain, ) = persistent_directory(target_requested_path) if ( current_namespace_target != namespace_target or current_host_target != host_target or current_target_chain != target_chain or current_target_host_chain != target_host_chain ): raise SystemExit(1) recheck_namespace_chain(target_chain) recheck_host_chain(target_host_chain) paths = [data_directory, wal_directory, *tablespaces] if any(os.stat(path).st_uid != data_details.st_uid for path in paths): raise SystemExit(1) if re.fullmatch(r"[1-9][0-9]{0,31}", system_identifier) is None: raise SystemExit(1) topology = { "schema": "bridgesllm-update-database-topology-v1", "systemIdentifier": system_identifier, "serverAddress": str(host_address), "serverPort": int(expected_port), "databaseName": observed["databaseName"], "databaseOid": observed["databaseOid"], "dataDirectory": data_directory, "walDirectory": wal_directory, "tablespaces": [ {"path": path, "stDev": os.stat(path).st_dev} for path in tablespaces ], "dataDevice": data_details.st_dev, "walDevice": os.stat(wal_directory).st_dev, } payload = json.dumps( topology, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode("ascii") if len(payload) > 64 * 1024: raise SystemExit(1) print(payload.decode("ascii")) PY2 fi if [[ -n "${psql_input_fd}" ]]; then printf '%s\n' '\q' >&"${psql_input_fd}" 2>/dev/null || true eval "exec ${psql_input_fd}>&-" || true fi if [[ -n "${psql_pid}" ]]; then for _ in {1..20}; do kill -0 "${psql_pid}" 2>/dev/null || break sleep 0.05 done kill -TERM "${psql_pid}" 2>/dev/null || true wait "${psql_pid}" 2>/dev/null || true fi if [[ -n "${backend_pid}" ]]; then [[ -n "${backend_start}" ]] || settlement_status=1 for _ in {1..200}; do [[ "${settlement_status}" -eq 0 ]] || break remaining="$( printf "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE pid = %s AND extract(epoch FROM backend_start)::text = '%s';\n" \ "${backend_pid}" "${backend_start}" \ | run_with_update_pgpass "${db_url}" \ psql --dbname="${connection_url}" --no-psqlrc \ -v ON_ERROR_STOP=1 -qAt 2>>"${LOG_FILE}" )" || { settlement_status=1 break } remaining="$(tr -d '[:space:]' <<<"${remaining}")" [[ "${remaining}" == "0" ]] && break [[ "${remaining}" == "1" ]] || { settlement_status=1 break } sleep 0.05 done [[ "${remaining}" == "0" ]] || settlement_status=1 fi if [[ "${status}" -eq 0 && "${proof_status}" -eq 0 \ && "${settlement_status}" -eq 0 ]]; then proof_output="$(cat "${proof_file}")" || status=$? fi rm -rf -- "${work_dir}" [[ "${status}" -eq 0 && "${proof_status}" -eq 0 \ && "${settlement_status}" -eq 0 && -n "${proof_output}" ]] || return 1 printf '%s\n' "${proof_output}" } release_update_disk_reserves() { # Terminal cleanup is itself journaled. A hard kill after any unlink can # resume from the manifest without guessing at files or touching another # transaction's reserve. local state_root manifest state_root="$(update_transaction_state_path "${UPDATE_STATE_ROOT}")" \ || return 1 manifest="${state_root}/update-disk-reserves.json" UPDATE_DISK_RESERVE_MANIFEST="${manifest}" [[ -e "${manifest}" || -L "${manifest}" ]] || return 0 python3 - "${manifest}" <<'PY2' || return $? import json import os import re import stat import sys import tempfile manifest = sys.argv[1] def fsync_directory(path): descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) def write_manifest(record): directory = os.path.dirname(manifest) payload = ( json.dumps( record, sort_keys=True, separators=(",", ":"), ensure_ascii=True ) + "\n" ).encode("ascii") if len(payload) > 64 * 1024: raise SystemExit(1) descriptor, temporary = tempfile.mkstemp( prefix=".update-disk-reserves.", dir=directory ) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, manifest) temporary = "" fsync_directory(directory) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass source_only = ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and bool(os.environ.get("BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT")) ) crash_after = 0 if source_only and os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_CRASH_AFTER" ): crash_after = int( os.environ["BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_CRASH_AFTER"] ) if crash_after < 1: raise SystemExit(1) crash_step = 0 def maybe_crash(): global crash_step crash_step += 1 if crash_after == crash_step: os._exit(97) try: details = os.lstat(manifest) except OSError: raise SystemExit(1) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or not 1 <= details.st_size <= 64 * 1024 ): raise SystemExit(1) try: record = json.loads(open(manifest, "rb").read()) except (OSError, ValueError): raise SystemExit(1) if ( not isinstance(record, dict) or record.get("schema") != "bridgesllm-update-disk-reserve-v2" or record.get("state") not in { "allocating", "held", "rearming", "recovery-preparing", "recovery", "emergency-ready", "emergency-releasing", "emergency-consumed", "releasing", } or not isinstance(record.get("transactionId"), str) or not re.fullmatch(r"[a-f0-9]{32}", record["transactionId"]) or not isinstance(record.get("tierBytes"), int) or not 1 <= record["tierBytes"] <= 256 * 1024 * 1024 or not isinstance(record.get("entries"), list) or not 2 <= len(record["entries"]) <= 128 or len(record["entries"]) % 2 ): raise SystemExit(1) if record["state"] != "releasing": record["state"] = "releasing" for entry in record["entries"]: if isinstance(entry, dict): entry["removed"] = False write_manifest(record) seen = set() for entry in record["entries"]: tier = entry.get("tier") if isinstance(entry, dict) else None expected_name = ( f".bridgesllm-update-reserve-{tier}-{record['transactionId']}" ) if ( not isinstance(entry, dict) or tier not in {"work", "emergency"} or not isinstance(entry.get("device"), str) or not entry["device"] or not isinstance(entry.get("path"), str) or os.path.basename(entry["path"]) != expected_name or not os.path.isabs(entry["path"]) or os.path.normpath(entry["path"]) != entry["path"] or not isinstance(entry.get("removed"), bool) or (entry["device"], tier) in seen ): raise SystemExit(1) seen.add((entry["device"], tier)) path = entry["path"] parent = os.path.dirname(path) try: parent_details = os.lstat(parent) except OSError: raise SystemExit(1) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_mode & 0o022 ): raise SystemExit(1) try: held = os.lstat(path) except FileNotFoundError: if record["state"] != "releasing": raise SystemExit(1) if not entry["removed"]: entry["removed"] = True write_manifest(record) continue except OSError: raise SystemExit(1) if ( not stat.S_ISREG(held.st_mode) or stat.S_ISLNK(held.st_mode) or held.st_uid != os.geteuid() or held.st_gid != os.getegid() or held.st_nlink != 1 or stat.S_IMODE(held.st_mode) != 0o600 or not 0 <= held.st_size <= record["tierBytes"] or ( entry.get("stDev") is not None and held.st_dev != entry.get("stDev") ) or ( entry.get("stIno") is not None and held.st_ino != entry.get("stIno") ) ): raise SystemExit(1) os.unlink(path) fsync_directory(parent) maybe_crash() entry["removed"] = True write_manifest(record) maybe_crash() os.unlink(manifest) parent = os.path.dirname(manifest) fsync_directory(parent) maybe_crash() PY2 UPDATE_DISK_RESERVE_MANIFEST="" } prepare_update_disk_reserves_for_recovery() { # Recovery consumes only the work tier. The emergency tier remains fully # allocated on every filesystem until the transaction reaches a terminal # receipt. A restarted recovery first tries to re-arm every work tier. If # recovery already consumed that space, it records emergency-ready but this # function never consumes the final tier. The caller must first prove the # sole receipt, rollback artifacts, protected paths, and database topology, # then invoke the explicit emergency consumer immediately before recovery # writes. local state_root manifest state_root="$(update_transaction_state_path "${UPDATE_STATE_ROOT}")" \ || return 1 manifest="${state_root}/update-disk-reserves.json" [[ -f "${manifest}" && ! -L "${manifest}" ]] || return 1 python3 - "${manifest}" <<'PY2' || return $? import errno import json import os import re import stat import sys import tempfile manifest = sys.argv[1] state_root = os.path.dirname(manifest) def fsync_directory(path): descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) def write_manifest(record): payload = ( json.dumps( record, sort_keys=True, separators=(",", ":"), ensure_ascii=True ) + "\n" ).encode("ascii") if len(payload) > 64 * 1024: raise SystemExit(1) descriptor, temporary = tempfile.mkstemp( prefix=".update-disk-reserves.", dir=state_root ) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, manifest) temporary = "" fsync_directory(state_root) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass details = os.lstat(manifest) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or not 1 <= details.st_size <= 64 * 1024 ): raise SystemExit(1) try: record = json.loads(open(manifest, "rb").read()) except (OSError, ValueError): raise SystemExit(1) if ( not isinstance(record, dict) or record.get("schema") != "bridgesllm-update-disk-reserve-v2" or record.get("state") not in { "held", "rearming", "recovery-preparing", "recovery", "emergency-ready", "emergency-releasing", "emergency-consumed", } or not isinstance(record.get("transactionId"), str) or not re.fullmatch(r"[a-f0-9]{32}", record["transactionId"]) or not isinstance(record.get("tierBytes"), int) or not 1 <= record["tierBytes"] <= 256 * 1024 * 1024 or not isinstance(record.get("entries"), list) or not 2 <= len(record["entries"]) <= 128 or len(record["entries"]) % 2 ): raise SystemExit(1) tier_bytes = record["tierBytes"] source_only = ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and bool(os.environ.get("BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT")) ) crash_after = 0 if source_only and os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_RECOVERY_CRASH_AFTER" ): crash_after = int( os.environ[ "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_RECOVERY_CRASH_AFTER" ] ) if crash_after < 1: raise SystemExit(1) crash_step = 0 def maybe_crash(): global crash_step crash_step += 1 if crash_after == crash_step: os._exit(98) entries_by_device = {} for entry in record["entries"]: if not isinstance(entry, dict): raise SystemExit(1) tier = entry.get("tier") device = entry.get("device") path = entry.get("path") expected_name = ( f".bridgesllm-update-reserve-{tier}-{record['transactionId']}" ) if ( tier not in {"work", "emergency"} or not isinstance(device, str) or not device or not isinstance(path, str) or not os.path.isabs(path) or os.path.normpath(path) != path or os.path.basename(path) != expected_name or (device, tier) in entries_by_device ): raise SystemExit(1) parent = os.path.dirname(path) parent_details = os.lstat(parent) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_mode & 0o022 ): raise SystemExit(1) entries_by_device[(device, tier)] = entry devices = {device for device, _ in entries_by_device} if any( (device, "work") not in entries_by_device or (device, "emergency") not in entries_by_device for device in devices ): raise SystemExit(1) def validate_file(entry, *, require_full): details = os.lstat(entry["path"]) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or details.st_dev != entry.get("stDev") or details.st_ino != entry.get("stIno") or not 0 <= details.st_size <= tier_bytes ): raise SystemExit(1) if require_full and ( details.st_size != tier_bytes or details.st_blocks * 512 < tier_bytes ): raise SystemExit(1) return details initial_state = record["state"] def truncate_tier(entry): details = validate_file(entry, require_full=False) descriptor = os.open( entry["path"], os.O_RDWR | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: opened = os.fstat(descriptor) if opened.st_dev != details.st_dev or opened.st_ino != details.st_ino: raise SystemExit(1) os.ftruncate(descriptor, 0) os.fsync(descriptor) finally: os.close(descriptor) fsync_directory(os.path.dirname(entry["path"])) # Emergency states are only validated here. Consumption is a separate, # explicitly ordered operation after the read-only recovery preflight. A # partially releasing state is accepted so SIGKILL can re-run the preflight # before converging its write-ahead truncations. if initial_state in { "emergency-ready", "emergency-releasing", "emergency-consumed" }: for device in sorted(devices): work = entries_by_device[(device, "work")] work_details = validate_file(work, require_full=False) if work_details.st_size != 0 or work.get("released") is not True: raise SystemExit(1) if initial_state == "emergency-consumed": for device in sorted(devices): emergency = entries_by_device[(device, "emergency")] emergency_details = validate_file(emergency, require_full=False) if ( emergency_details.st_size != 0 or emergency.get("released") is not True ): raise SystemExit(1) raise SystemExit(0) for device in sorted(devices): emergency = entries_by_device[(device, "emergency")] details = validate_file(emergency, require_full=False) if initial_state == "emergency-ready": if ( details.st_size != tier_bytes or details.st_blocks * 512 < tier_bytes or emergency.get("released") is not False ): raise SystemExit(1) elif ( details.st_size not in {0, tier_bytes} or ( details.st_size == tier_bytes and details.st_blocks * 512 < tier_bytes ) or emergency.get("released") not in {False, True} or ( emergency.get("released") is True and details.st_size != 0 ) ): raise SystemExit(1) raise SystemExit(0) # Before escalation the emergency tier is immutable and fully allocated. for device in sorted(devices): emergency = entries_by_device[(device, "emergency")] validate_file(emergency, require_full=True) if emergency.get("released") is not False: raise SystemExit(1) forced_rearm_available = None if source_only and os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RECOVERY_AVAILABLE_BYTES" ) is not None: forced_rearm_available = int( os.environ[ "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RECOVERY_AVAILABLE_BYTES" ] ) if forced_rearm_available < 0: raise SystemExit(1) record["state"] = "rearming" write_manifest(record) maybe_crash() rearm_failed = False for device in sorted(devices): entry = entries_by_device[(device, "work")] path = entry["path"] descriptor = -1 try: try: validate_file(entry, require_full=False) descriptor = os.open( path, os.O_RDWR | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) except FileNotFoundError: descriptor = os.open( path, os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) created = os.fstat(descriptor) entry["stDev"] = created.st_dev entry["stIno"] = created.st_ino opened = os.fstat(descriptor) if ( not stat.S_ISREG(opened.st_mode) or opened.st_uid != os.geteuid() or opened.st_gid != os.getegid() or opened.st_nlink != 1 or stat.S_IMODE(opened.st_mode) != 0o600 or opened.st_dev != entry["stDev"] or opened.st_ino != entry["stIno"] ): raise SystemExit(1) if ( forced_rearm_available is not None and forced_rearm_available < tier_bytes ): raise OSError(errno.ENOSPC, "simulated recovery reserve exhaustion") os.ftruncate(descriptor, 0) if hasattr(os, "posix_fallocate"): os.posix_fallocate(descriptor, 0, tier_bytes) else: remaining = tier_bytes block = b"\0" * min(1024 * 1024, tier_bytes) while remaining: written = os.write(descriptor, block[:remaining]) if written <= 0: raise OSError("reserve allocation made no progress") remaining -= written os.fsync(descriptor) held = os.fstat(descriptor) if ( held.st_size != tier_bytes or held.st_blocks * 512 < tier_bytes ): raise OSError("work reserve is sparse or incomplete") except OSError as error: if error.errno not in {errno.ENOSPC, errno.EDQUOT}: raise rearm_failed = True finally: if descriptor >= 0: os.close(descriptor) if rearm_failed: break fsync_directory(os.path.dirname(path)) entry["released"] = False write_manifest(record) maybe_crash() # Whether rearm succeeded or capacity was consumed, return every work inode to # a journaled released state. In the latter case emergency remains untouched # for this recovery attempt and is consumed only on a later invocation. record["state"] = "recovery-preparing" write_manifest(record) maybe_crash() for device in sorted(devices): entry = entries_by_device[(device, "work")] details = validate_file(entry, require_full=False) if details.st_size: truncate_tier(entry) maybe_crash() entry["released"] = True write_manifest(record) maybe_crash() record["state"] = "emergency-ready" if rearm_failed else "recovery" write_manifest(record) maybe_crash() PY2 UPDATE_DISK_RESERVE_MANIFEST="${manifest}" } consume_update_emergency_disk_reserves_for_recovery() { # This is the only non-terminal path allowed to consume the final reserve. # Its caller orders it after the read-only recovery preflight and # immediately before journal/restore writes. Every truncation is protected # by a durable emergency-releasing state and converges after SIGKILL. local state_root manifest state_root="$(update_transaction_state_path "${UPDATE_STATE_ROOT}")" \ || return 1 manifest="${state_root}/update-disk-reserves.json" [[ -f "${manifest}" && ! -L "${manifest}" ]] || return 1 python3 - "${manifest}" <<'PY2' || return $? import json import os import re import stat import sys import tempfile manifest = sys.argv[1] state_root = os.path.dirname(manifest) def fsync_directory(path): descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) def write_manifest(record): payload = ( json.dumps( record, sort_keys=True, separators=(",", ":"), ensure_ascii=True ) + "\n" ).encode("ascii") if len(payload) > 64 * 1024: raise SystemExit(1) descriptor, temporary = tempfile.mkstemp( prefix=".update-disk-reserves.", dir=state_root ) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, manifest) temporary = "" fsync_directory(state_root) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass details = os.lstat(manifest) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or not 1 <= details.st_size <= 64 * 1024 ): raise SystemExit(1) try: record = json.loads(open(manifest, "rb").read()) except (OSError, ValueError): raise SystemExit(1) if ( not isinstance(record, dict) or record.get("schema") != "bridgesllm-update-disk-reserve-v2" or record.get("state") not in { "held", "rearming", "recovery-preparing", "recovery", "emergency-ready", "emergency-releasing", "emergency-consumed", } or not isinstance(record.get("transactionId"), str) or not re.fullmatch(r"[a-f0-9]{32}", record["transactionId"]) or not isinstance(record.get("tierBytes"), int) or not 1 <= record["tierBytes"] <= 256 * 1024 * 1024 or not isinstance(record.get("entries"), list) or not 2 <= len(record["entries"]) <= 128 or len(record["entries"]) % 2 ): raise SystemExit(1) if record["state"] in { "held", "rearming", "recovery-preparing", "recovery" }: raise SystemExit(0) tier_bytes = record["tierBytes"] source_only = ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and bool(os.environ.get("BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT")) ) crash_after = 0 if source_only and os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_RECOVERY_CRASH_AFTER" ): crash_after = int( os.environ[ "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_RECOVERY_CRASH_AFTER" ] ) if crash_after < 1: raise SystemExit(1) crash_step = 0 def maybe_crash(): global crash_step crash_step += 1 if crash_after == crash_step: os._exit(98) entries_by_device = {} for entry in record["entries"]: if not isinstance(entry, dict): raise SystemExit(1) tier = entry.get("tier") device = entry.get("device") path = entry.get("path") expected_name = ( f".bridgesllm-update-reserve-{tier}-{record['transactionId']}" ) if ( tier not in {"work", "emergency"} or not isinstance(device, str) or not device or not isinstance(path, str) or not os.path.isabs(path) or os.path.normpath(path) != path or os.path.basename(path) != expected_name or (device, tier) in entries_by_device ): raise SystemExit(1) parent = os.path.dirname(path) parent_details = os.lstat(parent) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_mode & 0o022 ): raise SystemExit(1) entries_by_device[(device, tier)] = entry devices = {device for device, _ in entries_by_device} if any( (device, "work") not in entries_by_device or (device, "emergency") not in entries_by_device for device in devices ): raise SystemExit(1) def validate_file(entry): details = os.lstat(entry["path"]) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or details.st_dev != entry.get("stDev") or details.st_ino != entry.get("stIno") or details.st_size not in {0, tier_bytes} or ( details.st_size == tier_bytes and details.st_blocks * 512 < tier_bytes ) ): raise SystemExit(1) return details for device in sorted(devices): work = entries_by_device[(device, "work")] if ( validate_file(work).st_size != 0 or work.get("released") is not True ): raise SystemExit(1) if record["state"] == "emergency-consumed": for device in sorted(devices): emergency = entries_by_device[(device, "emergency")] if ( validate_file(emergency).st_size != 0 or emergency.get("released") is not True ): raise SystemExit(1) raise SystemExit(0) if record["state"] == "emergency-ready": for device in sorted(devices): emergency = entries_by_device[(device, "emergency")] if ( validate_file(emergency).st_size != tier_bytes or emergency.get("released") is not False ): raise SystemExit(1) record["state"] = "emergency-releasing" write_manifest(record) maybe_crash() for device in sorted(devices): emergency = entries_by_device[(device, "emergency")] details = validate_file(emergency) if emergency.get("released") is True: if details.st_size != 0: raise SystemExit(1) continue if emergency.get("released") is not False: raise SystemExit(1) if details.st_size == tier_bytes: descriptor = os.open( emergency["path"], os.O_RDWR | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: opened = os.fstat(descriptor) if ( opened.st_dev != details.st_dev or opened.st_ino != details.st_ino ): raise SystemExit(1) os.ftruncate(descriptor, 0) os.fsync(descriptor) finally: os.close(descriptor) fsync_directory(os.path.dirname(emergency["path"])) maybe_crash() emergency["released"] = True write_manifest(record) maybe_crash() record["state"] = "emergency-consumed" write_manifest(record) maybe_crash() PY2 } update_disk_admission_denied() { # Every admission refusal names itself in the log. The caller can then say # what actually happened instead of blaming free space for all of them. printf 'update transaction disk admission denied: %s\n' "$1" \ >> "${LOG_FILE}" 2>/dev/null || true return 1 } update_disk_admission_last_detail() { # Read back the reason this run recorded, for the operator-facing message. # This must never fail: it is expanded inside the operator-facing message of # a refusal, and a non-zero status there would trip the ERR trap and replace # a precise explanation with "Unexpected error" -- the exact substitution # this whole change exists to stop. local detail="" if [[ -r "${LOG_FILE}" ]]; then detail="$( tail -n 200 "${LOG_FILE}" 2>/dev/null \ | grep -E '^update transaction disk admission (denied|failed): ' \ | tail -n 1 \ | sed -E 's/^update transaction disk admission (denied|failed): //' )" || detail="" fi [[ -n "${detail}" ]] || detail="no reason was recorded; see the installer log" printf '%s\n' "${detail}" return 0 } assert_update_transaction_disk_admission() { # The transaction must never begin work it cannot finish or roll back for # want of disk. Before staging, the current Portal is a conservative # candidate estimate. After staging and dependency preparation, the exact # candidate footprint is required and admission runs again before a receipt # or downtime. The journal filesystem gets its own reserve because its # bounded manifests can live on a separately mounted /var. # # Exit codes are load-bearing: 2 means the host genuinely lacks the space, # and anything else non-zero means admission refused for a different, # separately logged reason. Collapsing them into one message hides real # upgrade blockers behind a disk-space claim that is not true. local portal_dir="$1" db_url="$2" staged_portal="${3:-}" local portal_size candidate_size db_size db_topology local backup_root stage_root state_root [[ "${UPDATE_TRANSACTION_ID:-}" =~ ^[a-f0-9]{32}$ ]] \ || update_disk_admission_denied \ "the update transaction identifier is malformed" \ || return 1 portal_size="$(du -sb "${portal_dir}" 2>/dev/null | cut -f1)" \ || update_disk_admission_denied \ "the live Portal tree size could not be measured" \ || return 1 [[ "${portal_size}" =~ ^[0-9]+$ && "${portal_size}" -gt 0 ]] \ || update_disk_admission_denied \ "the live Portal tree reported a non-positive size" \ || return 1 candidate_size="${portal_size}" if [[ -n "${staged_portal}" ]]; then [[ -d "${staged_portal}" && ! -L "${staged_portal}" ]] \ || update_disk_admission_denied \ "the staged candidate tree is missing, is not a directory, or is a symlink" \ || return 1 candidate_size="$(du -sb "${staged_portal}" 2>/dev/null | cut -f1)" \ || update_disk_admission_denied \ "the staged candidate tree size could not be measured" \ || return 1 [[ "${candidate_size}" =~ ^[0-9]+$ && "${candidate_size}" -gt 0 ]] \ || update_disk_admission_denied \ "the staged candidate tree reported a non-positive size" \ || return 1 fi db_size="$(update_database_size_bytes "${db_url}")" \ || update_disk_admission_denied \ "the Portal database size could not be measured" \ || return 1 db_topology="$(canonical_update_database_topology "${db_url}")" \ || update_disk_admission_denied \ "the Portal database storage topology could not be attested" \ || return 1 backup_root="$(update_transaction_state_path "${UPDATE_BACKUP_ROOT}")" \ || update_disk_admission_denied \ "the rollback snapshot root is not a safe update transaction path" \ || return 1 stage_root="$(update_transaction_state_path "${UPDATE_STAGE_ROOT}")" \ || update_disk_admission_denied \ "the staging root is not a safe update transaction path" \ || return 1 state_root="$(update_transaction_state_path "${UPDATE_STATE_ROOT}")" \ || update_disk_admission_denied \ "the installer journal root is not a safe update transaction path" \ || return 1 python3 - "${portal_size}" "${candidate_size}" "${db_size}" \ "${stage_root}" "${backup_root}" "${portal_dir}" "${state_root}" \ "${staged_portal}" "${db_topology}" \ "${UPDATE_TRANSACTION_ID}" <<'PY2' >> "${LOG_FILE}" 2>&1 \ || return $? import hashlib import ipaddress import json import os import re import stat import sys import tempfile RECOVERY_RESERVE_BYTES = 512 * 1024 * 1024 STATE_RESERVE_BYTES = 1024 * 1024 * 1024 # Exit 3 marks a policy refusal that has nothing to do with free space. The # reason is printed into the installer log so the operator is told what the # admission actually objected to. POLICY_DENIED_EXIT = 3 def deny(reason: str) -> SystemExit: print(f"update transaction disk admission denied: {reason}") return SystemExit(POLICY_DENIED_EXIT) portal_size = int(sys.argv[1]) candidate_size = int(sys.argv[2]) db_size = int(sys.argv[3]) stage_root, backup_root, portal_dir, state_root, staged_portal = sys.argv[4:9] topology_raw, transaction_id = sys.argv[9:11] if not re.fullmatch(r"[a-f0-9]{32}", transaction_id): raise deny("the transaction identifier passed to the policy check is malformed") source_only = ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and bool(os.environ.get("BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT")) ) test_root = os.path.realpath( os.environ.get("BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT", "") ) if source_only else "" try: topology = json.loads(topology_raw) except ValueError: raise deny("the database storage topology record is not valid JSON") if ( not isinstance(topology, dict) or set(topology) != { "schema", "systemIdentifier", "serverAddress", "serverPort", "databaseName", "databaseOid", "dataDirectory", "walDirectory", "tablespaces", "dataDevice", "walDevice", } or topology.get("schema") != "bridgesllm-update-database-topology-v1" or not isinstance(topology["systemIdentifier"], str) or not re.fullmatch(r"[1-9][0-9]{0,31}", topology["systemIdentifier"]) or not isinstance(topology["serverAddress"], str) or isinstance(topology["serverPort"], bool) or not isinstance(topology["serverPort"], int) or not 1 <= topology["serverPort"] <= 65535 or not isinstance(topology["databaseName"], str) or isinstance(topology["databaseOid"], bool) or not isinstance(topology["databaseOid"], int) or not isinstance(topology["dataDirectory"], str) or not isinstance(topology["walDirectory"], str) or not isinstance(topology["tablespaces"], list) or len(topology["tablespaces"]) > 32 or not all( isinstance(item, dict) and set(item) == {"path", "stDev"} and isinstance(item["path"], str) and isinstance(item["stDev"], int) and not isinstance(item["stDev"], bool) for item in topology["tablespaces"] ) or isinstance(topology["dataDevice"], bool) or not isinstance(topology["dataDevice"], int) or isinstance(topology["walDevice"], bool) or not isinstance(topology["walDevice"], int) ): raise deny("the database storage topology record failed its schema contract") try: if not ipaddress.ip_address(topology["serverAddress"]).is_loopback: raise deny("the database server address is not a loopback address") except ValueError: raise deny("the database server address is not a valid IP address") def existing_ancestor(path: str) -> str: current = os.path.abspath(path) while not os.path.exists(current): parent = os.path.dirname(current) if parent == current: break current = parent return os.path.realpath(current) def safe_storage_directory(path: str) -> str: if ( not path or len(os.fsencode(path)) > 4096 or not os.path.isabs(path) or any(ord(char) < 32 or ord(char) == 127 for char in path) ): raise deny("a database storage path is empty, relative, over-long, or contains control characters") resolved = os.path.realpath(path) if not os.path.isabs(resolved) or not os.path.isdir(resolved): raise deny("a database storage path does not resolve to an absolute directory") if source_only: if os.path.commonpath((test_root, resolved)) != test_root: raise deny("a database storage path resolves outside the test fixture root") return resolved details = os.stat(resolved) if ( not stat.S_ISDIR(details.st_mode) or details.st_uid == 0 or details.st_mode & 0o022 ): raise deny("a database storage directory is root-owned or group/world writable") return resolved data_directory = safe_storage_directory(topology["dataDirectory"]) wal_directory = safe_storage_directory(topology["walDirectory"]) tablespaces = [ safe_storage_directory(item["path"]) for item in topology["tablespaces"] ] database_paths = list(dict.fromkeys( [data_directory, wal_directory, *tablespaces] )) if ( os.stat(data_directory).st_dev != topology["dataDevice"] or os.stat(wal_directory).st_dev != topology["walDevice"] or any( os.stat(item["path"]).st_dev != item["stDev"] for item in topology["tablespaces"] ) ): raise deny("a database storage directory changed filesystem device during admission") device_map: dict[str, str] = {} if source_only and os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_DEVICE_MAP" ): try: parsed_map = json.loads( os.environ["BRIDGESLLM_UPDATE_TRANSACTION_TEST_DEVICE_MAP"] ) except ValueError: raise deny("the test device map is not valid JSON") if ( not isinstance(parsed_map, dict) or not parsed_map or not all( isinstance(path, str) and isinstance(device, str) and device for path, device in parsed_map.items() ) ): raise deny("the test device map is malformed") device_map = { os.path.realpath(path): device for path, device in parsed_map.items() } def device_for(path: str) -> str: resolved = os.path.realpath(path) matches = [ (len(prefix), device) for prefix, device in device_map.items() if os.path.commonpath((prefix, resolved)) == prefix ] if matches: return max(matches)[1] return f"dev:{os.stat(existing_ancestor(path)).st_dev}" def safe_anchor(path: str) -> str: anchor = existing_ancestor(path) details = os.lstat(anchor) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_mode & 0o022 ): raise deny("a reserve anchor is not a safe directory (missing, a symlink, or group/world writable)") return anchor def database_reserve_anchor(path: str) -> str: # Keep the large reserve outside PGDATA/pg_wal/tablespace contents when a # safe parent is on the same filesystem. A mount rooted at the PostgreSQL # directory necessarily falls back to a hidden root-owned file there. if source_only: return path parent = os.path.realpath(os.path.dirname(path)) parent_details = os.lstat(parent) path_details = os.stat(path) if ( stat.S_ISDIR(parent_details.st_mode) and not stat.S_ISLNK(parent_details.st_mode) and not parent_details.st_mode & 0o022 and parent_details.st_dev == path_details.st_dev ): return parent return path state_parent = os.path.dirname(state_root) state_parent_details = os.lstat(existing_ancestor(state_parent)) if ( not stat.S_ISDIR(state_parent_details.st_mode) or stat.S_ISLNK(state_parent_details.st_mode) or state_parent_details.st_uid != os.geteuid() or state_parent_details.st_mode & 0o022 ): raise deny("the installer journal parent directory is unsafe (wrong owner, a symlink, or group/world writable)") try: os.mkdir(state_root, 0o700) except FileExistsError: pass state_details = os.lstat(state_root) if ( not stat.S_ISDIR(state_details.st_mode) or stat.S_ISLNK(state_details.st_mode) or state_details.st_uid != os.geteuid() or state_details.st_gid != os.getegid() ): raise deny("the installer journal directory is unsafe (wrong owner/group, or a symlink)") if stat.S_IMODE(state_details.st_mode) != 0o700: # Older uninstall runs left this root at 0755 because `install -d` only # applies its mode to the last component. Refusing on that would strand # the host on a permanent, and untrue, disk-space error. A root-owned # directory that was never writable by anyone else can be narrowed to # 0700 safely; one that was group- or world-writable cannot, because its # contents are no longer trustworthy. if state_details.st_mode & 0o022: raise deny("the installer journal directory is group- or world-writable") state_fd = os.open( state_root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: opened = os.fstat(state_fd) if ( opened.st_dev != state_details.st_dev or opened.st_ino != state_details.st_ino ): raise deny("the installer journal directory was replaced during admission") os.fchmod(state_fd, 0o700) state_details = os.fstat(state_fd) finally: os.close(state_fd) if stat.S_IMODE(state_details.st_mode) != 0o700: raise deny("the installer journal directory could not be narrowed to 0700") print( "update transaction disk admission repaired: " f"narrowed {state_root} to 0700" ) requirements: dict[str, int] = {} for path, needed in ( (backup_root, portal_size + db_size), (portal_dir, candidate_size), (state_root, STATE_RESERVE_BYTES), (stage_root, 0 if staged_portal else candidate_size), ): anchor = existing_ancestor(path) device = device_for(anchor) requirements[device] = requirements.get(device, 0) + needed for path in database_paths: device = device_for(path) requirements[device] = requirements.get(device, 0) + db_size paths = [state_root, backup_root, stage_root, os.path.dirname(portal_dir)] if staged_portal: paths.append(staged_portal) paths.extend(database_reserve_anchor(path) for path in database_paths) anchors: dict[str, str] = {} for path in paths: anchor = safe_anchor(path) anchors.setdefault(device_for(anchor), anchor) if set(anchors) != set(requirements): raise deny("the reserve anchors do not cover exactly the devices that carry requirements") reserve_bytes = RECOVERY_RESERVE_BYTES if source_only and os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_BYTES" ): reserve_bytes = int( os.environ["BRIDGESLLM_UPDATE_TRANSACTION_TEST_RESERVE_BYTES"] ) if ( not 2 <= reserve_bytes <= RECOVERY_RESERVE_BYTES or reserve_bytes % 2 ): raise deny("the test reserve size override is out of range") tier_bytes = reserve_bytes // 2 test_available: dict[str, int] = {} if source_only: raw_map = os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_AVAILABLE_BY_DEVICE" ) raw_single = os.environ.get( "BRIDGESLLM_UPDATE_TRANSACTION_TEST_AVAILABLE_BYTES" ) if raw_map: try: parsed_available = json.loads(raw_map) except ValueError: raise deny("the test available-space map is not valid JSON") if ( not isinstance(parsed_available, dict) or not all( isinstance(device, str) and isinstance(value, int) and value >= 0 for device, value in parsed_available.items() ) ): raise deny("the test available-space map is malformed") test_available = parsed_available elif raw_single is not None: value = int(raw_single) if value < 0: raise deny("the test available-space value is negative") test_available = {device: value for device in requirements} manifest_path = os.path.join(state_root, "update-disk-reserves.json") def read_manifest() -> dict[str, object] | None: try: details = os.lstat(manifest_path) except FileNotFoundError: return None if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or not 1 <= details.st_size <= 64 * 1024 ): raise deny("the existing disk reserve manifest failed its ownership, mode, link-count, or size contract") try: record = json.loads(open(manifest_path, "rb").read()) except (OSError, ValueError, SystemExit): raise deny("the existing disk reserve manifest is not readable JSON") return record def write_manifest(record: dict[str, object]) -> None: payload = ( json.dumps( record, sort_keys=True, separators=(",", ":"), ensure_ascii=True ) + "\n" ).encode("ascii") if len(payload) > 64 * 1024: raise deny("the disk reserve manifest payload exceeds its bounded size") descriptor, temporary = tempfile.mkstemp( prefix=".update-disk-reserves.", dir=state_root ) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, manifest_path) temporary = "" directory_fd = os.open( state_root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass expected_entries = [ { "device": device, "tier": tier, "path": os.path.join( anchor, f".bridgesllm-update-reserve-{tier}-{transaction_id}", ), "released": False, } for device, anchor in sorted(anchors.items()) for tier in ("work", "emergency") ] topology_payload = json.dumps( topology, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode("ascii") topology_sha256 = hashlib.sha256(topology_payload).hexdigest() record = read_manifest() reserve_already_held = record is not None if record is not None: if ( record.get("schema") != "bridgesllm-update-disk-reserve-v2" or record.get("state") != "held" or record.get("transactionId") != transaction_id or record.get("tierBytes") != tier_bytes or record.get("databaseSystemIdentifier") != topology["systemIdentifier"] or record.get("databaseTopologySha256") != topology_sha256 or record.get("databaseTopology") != topology or not isinstance(record.get("entries"), list) or [ { "device": entry.get("device"), "tier": entry.get("tier"), "path": entry.get("path"), "released": entry.get("released"), } for entry in record["entries"] ] != expected_entries ): raise deny("the existing disk reserve manifest does not match this transaction, tier size, database identity, or planned entries") for entry in record["entries"]: held = os.lstat(entry["path"]) if ( not stat.S_ISREG(held.st_mode) or stat.S_ISLNK(held.st_mode) or held.st_uid != os.geteuid() or held.st_gid != os.getegid() or held.st_nlink != 1 or stat.S_IMODE(held.st_mode) != 0o600 or held.st_size != tier_bytes or held.st_dev != entry.get("stDev") or held.st_ino != entry.get("stIno") or held.st_blocks * 512 < tier_bytes ): raise deny("a held reserve file no longer matches its manifest entry (owner, mode, size, inode, or allocated blocks)") for device, anchor in anchors.items(): details = os.statvfs(anchor) available = ( test_available[device] if device in test_available else details.f_bavail * details.f_frsize ) required = requirements[device] if not reserve_already_held: required += reserve_bytes if available < required: print( "update transaction disk admission failed: " f"device {device} at {anchor} has {available} bytes available, " f"requires {required} " f"(short by {required - available} bytes)" ) raise SystemExit(2) if not reserve_already_held: record = { "schema": "bridgesllm-update-disk-reserve-v2", "state": "allocating", "transactionId": transaction_id, "tierBytes": tier_bytes, "databaseSystemIdentifier": topology["systemIdentifier"], "databaseTopologySha256": topology_sha256, "databaseTopology": topology, "entries": expected_entries, } write_manifest(record) held_entries: list[dict[str, object]] = [] for planned in expected_entries: flags = ( os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) ) descriptor = os.open(planned["path"], flags, 0o600) try: os.fchmod(descriptor, 0o600) if hasattr(os, "posix_fallocate"): os.posix_fallocate(descriptor, 0, tier_bytes) else: remaining = tier_bytes block = b"\0" * min(1024 * 1024, tier_bytes) while remaining: written = os.write(descriptor, block[:remaining]) if written <= 0: raise OSError("reserve allocation made no progress") remaining -= written os.fsync(descriptor) held = os.fstat(descriptor) if ( held.st_size != tier_bytes or held.st_blocks * 512 < tier_bytes ): raise OSError("reserve file is sparse or incomplete") held_entries.append({ **planned, "stDev": held.st_dev, "stIno": held.st_ino, }) finally: os.close(descriptor) parent_fd = os.open( os.path.dirname(planned["path"]), os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(parent_fd) finally: os.close(parent_fd) record["state"] = "held" record["entries"] = held_entries write_manifest(record) print( "update transaction disk admission passed: " f"portal={portal_size} candidate={candidate_size} database={db_size} " f"devices={len(requirements)} reserve_per_device={reserve_bytes} " f"reserve_tier={tier_bytes}" ) PY2 UPDATE_DISK_RESERVE_MANIFEST="${state_root}/update-disk-reserves.json" } read_update_disk_reserve_database_topology() { local expected_transaction_id="$1" state_root manifest state_root="$(update_transaction_state_path "${UPDATE_STATE_ROOT}")" \ || return 1 manifest="${state_root}/update-disk-reserves.json" python3 - "${manifest}" "${expected_transaction_id}" <<'PY2' import hashlib import json import os import re import stat import sys manifest, expected_transaction_id = sys.argv[1:3] if re.fullmatch(r"[a-f0-9]{32}", expected_transaction_id) is None: raise SystemExit(1) details = os.lstat(manifest) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or not 1 <= details.st_size <= 64 * 1024 ): raise SystemExit(1) try: record = json.loads(open(manifest, "rb").read()) except (OSError, ValueError): raise SystemExit(1) if ( not isinstance(record, dict) or record.get("schema") != "bridgesllm-update-disk-reserve-v2" or record.get("state") not in { "held", "rearming", "recovery-preparing", "recovery", "emergency-ready", "emergency-releasing", "emergency-consumed", } or record.get("transactionId") != expected_transaction_id or not isinstance(record.get("databaseTopology"), dict) or not isinstance(record.get("databaseSystemIdentifier"), str) or re.fullmatch( r"[1-9][0-9]{0,31}", record["databaseSystemIdentifier"] ) is None or not isinstance(record.get("databaseTopologySha256"), str) or re.fullmatch( r"[a-f0-9]{64}", record["databaseTopologySha256"] ) is None ): raise SystemExit(1) payload = json.dumps( record["databaseTopology"], sort_keys=True, separators=(",", ":"), ensure_ascii=True, ).encode("ascii") digest = hashlib.sha256(payload).hexdigest() if ( digest != record["databaseTopologySha256"] or record["databaseTopology"].get("systemIdentifier") != record["databaseSystemIdentifier"] ): raise SystemExit(1) print(record["databaseSystemIdentifier"]) print(digest) print(payload.decode("ascii")) PY2 } assert_update_database_topology_unchanged() { local db_url="$1" context="${2:-database-operation}" local target transaction_id receipt_system_identifier receipt_digest local current_topology local -a pinned=() target="$(current_update_receipt_target)" || return 1 transaction_id="$( read_update_transaction_field "${target}" transaction_id )" || return 1 receipt_system_identifier="$( read_update_transaction_field "${target}" database_system_identifier )" || return 1 receipt_digest="$( read_update_transaction_field "${target}" database_topology_sha256 )" || return 1 mapfile -t pinned < <( read_update_disk_reserve_database_topology "${transaction_id}" ) || return 1 [[ "${#pinned[@]}" -eq 3 \ && "${pinned[0]}" == "${receipt_system_identifier}" \ && "${pinned[1]}" == "${receipt_digest}" ]] || return 1 current_topology="$(canonical_update_database_topology "${db_url}")" \ || return 1 python3 - "${pinned[2]}" "${current_topology}" \ "${receipt_system_identifier}" "${receipt_digest}" "${context}" <<'PY2' import hashlib import json import sys pinned_raw, current_raw, expected_system_identifier, expected_digest, context = ( sys.argv[1:6] ) try: pinned = json.loads(pinned_raw) current = json.loads(current_raw) except ValueError: raise SystemExit(1) if not isinstance(pinned, dict) or not isinstance(current, dict): raise SystemExit(1) pinned_payload = json.dumps( pinned, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode("ascii") current_payload = json.dumps( current, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode("ascii") if ( hashlib.sha256(pinned_payload).hexdigest() != expected_digest or pinned.get("systemIdentifier") != expected_system_identifier or current.get("systemIdentifier") != expected_system_identifier or current_payload != pinned_payload ): print( f"database topology changed before {context}; refusing operation", file=sys.stderr, ) raise SystemExit(1) PY2 } attest_update_database_guard_identity() { local observed_json="$1" pinned_json="$2" db_url="$3" local current_topology current_topology="$(canonical_update_database_topology "${db_url}")" \ || return 1 python3 - "${observed_json}" "${pinned_json}" \ "${current_topology}" <<'PY2' import hashlib import ipaddress import json import os import ctypes import posixpath import re import stat import subprocess import sys try: observed = json.loads(sys.argv[1]) pinned = json.loads(sys.argv[2]) current = json.loads(sys.argv[3]) except ValueError: raise SystemExit(1) if ( not isinstance(observed, dict) or not isinstance(pinned, dict) or not isinstance(current, dict) ): raise SystemExit(1) current_payload = json.dumps( current, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode("ascii") pinned_payload = json.dumps( pinned, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode("ascii") if ( current_payload != pinned_payload or current.get("systemIdentifier") != pinned.get("systemIdentifier") ): raise SystemExit(1) expected_observed = { "serverAddress", "serverPort", "backendPid", "backendStart", "databaseName", "databaseOid", "dataDirectory", "walDirectory", "defaultTablespace", "defaultTablespaceOid", "tempTablespaces", "tablespaces", "tablespaceOids", } if ( set(observed) != expected_observed or observed.get("tempTablespaces", "").strip() or observed.get("dataDirectory") != "" or observed.get("walDirectory") != "" or observed.get("tablespaces") != [] or observed.get("databaseName") != pinned.get("databaseName") or observed.get("databaseOid") != pinned.get("databaseOid") or not isinstance(observed.get("tablespaceOids"), list) or not isinstance(observed.get("backendStart"), str) or isinstance(observed.get("defaultTablespaceOid"), bool) or not isinstance(observed.get("defaultTablespaceOid"), int) ): raise SystemExit(1) default_tablespace = observed.get("defaultTablespace", "").strip() default_tablespace_oid = observed["defaultTablespaceOid"] tablespace_oids = observed["tablespaceOids"] if ( not all( not isinstance(oid, bool) and isinstance(oid, int) and 1 <= oid <= 4_294_967_295 for oid in tablespace_oids ) or tablespace_oids != sorted(set(tablespace_oids)) or not 0 <= default_tablespace_oid <= 4_294_967_295 or (not default_tablespace and default_tablespace_oid != 0) or ( default_tablespace and ( default_tablespace_oid == 0 or default_tablespace_oid not in tablespace_oids ) ) ): raise SystemExit(1) database_backend_pid = observed.get("backendPid") server_port = observed.get("serverPort") try: observed_address = ipaddress.ip_address(observed.get("serverAddress", "")) expected_address = ipaddress.ip_address(pinned.get("serverAddress", "")) except ValueError: raise SystemExit(1) if ( isinstance(database_backend_pid, bool) or not isinstance(database_backend_pid, int) or database_backend_pid <= 1 or isinstance(server_port, bool) or not isinstance(server_port, int) or not 1 <= server_port <= 65535 or not expected_address.is_loopback or isinstance(pinned.get("serverPort"), bool) or not isinstance(pinned.get("serverPort"), int) ): raise SystemExit(1) def process_identity(pid): root = f"/proc/{pid}" details = os.stat(root) executable_path = os.path.join(root, "exe") executable = os.readlink(executable_path) executable_details = os.stat(executable_path) process_stat = open( os.path.join(root, "stat"), "r", encoding="ascii" ).read() split = process_stat.rsplit(")", 1) if len(split) != 2: raise SystemExit(1) fields = split[1].strip().split() if ( len(fields) < 20 or fields[0] in {"Z", "X", "x"} or not fields[1].isdigit() or not fields[19].isdigit() or not os.path.isabs(executable) or executable.endswith(" (deleted)") ): raise SystemExit(1) status = open( os.path.join(root, "status"), "r", encoding="ascii" ).read().splitlines() nspid = [line for line in status if line.startswith("NSpid:")] if len(nspid) != 1: raise SystemExit(1) namespace_pids = nspid[0].split()[1:] if not namespace_pids or not all(value.isdigit() for value in namespace_pids): raise SystemExit(1) return { "pid": pid, "uid": details.st_uid, "gid": details.st_gid, "procDevice": details.st_dev, "procInode": details.st_ino, "parentPid": int(fields[1]), "startTime": int(fields[19]), "executable": executable, "executableDevice": executable_details.st_dev, "executableInode": executable_details.st_ino, "namespacePids": tuple(int(value) for value in namespace_pids), "namespaces": tuple( os.stat(os.path.join(root, "ns", name)).st_ino for name in ("pid", "mnt", "net") ), } def postgres_pair(backend, postmaster): return ( backend["parentPid"] == postmaster["pid"] and backend["uid"] == postmaster["uid"] and backend["gid"] == postmaster["gid"] and backend["uid"] != 0 and backend["namespaces"] == postmaster["namespaces"] and os.path.basename(backend["executable"]) in {"postgres", "postmaster"} and os.path.basename(postmaster["executable"]) in {"postgres", "postmaster"} and ( backend["executableDevice"], backend["executableInode"] ) == ( postmaster["executableDevice"], postmaster["executableInode"] ) ) def docker_records(): docker = "/usr/bin/docker" details = os.lstat(docker) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != 0 or details.st_mode & 0o022 ): raise SystemExit(1) listed = subprocess.run( [docker, "ps", "--quiet", "--no-trunc"], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, text=True, timeout=10, check=False, ) identifiers = [line.strip() for line in listed.stdout.splitlines()] if ( listed.returncode != 0 or len(listed.stdout) > 16 * 1024 or len(identifiers) > 128 or any(re.fullmatch(r"[a-f0-9]{64}", value) is None for value in identifiers) ): raise SystemExit(1) if not identifiers: return [] inspected = subprocess.run( [docker, "inspect", *identifiers], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, text=True, timeout=20, check=False, ) if inspected.returncode != 0 or len(inspected.stdout) > 8 * 1024 * 1024: raise SystemExit(1) try: records = json.loads(inspected.stdout) except ValueError: raise SystemExit(1) if not isinstance(records, list): raise SystemExit(1) return records records = None self_namespaces = tuple( os.stat(f"/proc/self/ns/{name}").st_ino for name in ("pid", "mnt", "net") ) candidates = [] for entry in os.listdir("/proc"): if not entry.isdigit(): continue try: backend_candidate = process_identity(int(entry)) except (OSError, ValueError, SystemExit): continue if backend_candidate["namespacePids"][-1] != database_backend_pid: continue try: postmaster_candidate = process_identity( backend_candidate["parentPid"] ) except (OSError, ValueError, SystemExit): continue if not postgres_pair(backend_candidate, postmaster_candidate): continue container = None if backend_candidate["namespaces"] == self_namespaces: if ( not observed_address.is_loopback or observed_address != expected_address or server_port != pinned["serverPort"] ): continue else: if records is None: records = docker_records() namespace_matches = [] for record in records: if not isinstance(record, dict): raise SystemExit(1) state = record.get("State") init_pid = state.get("Pid") if isinstance(state, dict) else None if ( not isinstance(state, dict) or state.get("Running") is not True or isinstance(init_pid, bool) or not isinstance(init_pid, int) or init_pid <= 1 ): continue try: namespaces = tuple( os.stat(f"/proc/{init_pid}/ns/{name}").st_ino for name in ("pid", "mnt", "net") ) except OSError: continue if namespaces == backend_candidate["namespaces"]: namespace_matches.append(record) if len(namespace_matches) != 1: continue container = namespace_matches[0] network = container.get("NetworkSettings") ports = network.get("Ports") if isinstance(network, dict) else None bindings = ( ports.get(f"{server_port}/tcp") if isinstance(ports, dict) else None ) exact_bindings = [] if isinstance(bindings, list) and len(bindings) == 1: for binding in bindings: if not isinstance(binding, dict): continue try: bind_address = ipaddress.ip_address( binding.get("HostIp", "") ) bind_port = int(binding.get("HostPort", "")) except (ValueError, TypeError): continue if ( bind_address == expected_address and bind_address.is_loopback and bind_port == pinned["serverPort"] ): exact_bindings.append(binding) container_addresses = set() networks = network.get("Networks") if isinstance(network, dict) else None if isinstance(networks, dict): for value in networks.values(): if not isinstance(value, dict): continue for field in ("IPAddress", "GlobalIPv6Address"): raw_address = value.get(field) if raw_address: container_addresses.add( ipaddress.ip_address(raw_address) ) if ( len(exact_bindings) != 1 or observed_address not in container_addresses ): continue candidates.append((backend_candidate, postmaster_candidate, container)) if len(candidates) != 1: raise SystemExit(1) backend, postmaster, container = candidates[0] backend_pid = backend["pid"] postmaster_pid = postmaster["pid"] namespace_data_directory = posixpath.normpath( os.readlink(f"/proc/{postmaster_pid}/cwd") ) if ( not posixpath.isabs(namespace_data_directory) or namespace_data_directory == "/" ): raise SystemExit(1) data_directory = os.path.realpath(pinned["dataDirectory"]) if container is None: candidate_data_directory = os.path.realpath(namespace_data_directory) else: mounts = container.get("Mounts") if not isinstance(mounts, list): raise SystemExit(1) mapped = [] for mount in mounts: if ( not isinstance(mount, dict) or mount.get("Type") not in {"bind", "volume"} or mount.get("RW") is not True ): continue source = mount.get("Source") destination = mount.get("Destination") if not isinstance(source, str) or not isinstance(destination, str): raise SystemExit(1) destination = posixpath.normpath(destination) if ( namespace_data_directory == destination or namespace_data_directory.startswith(destination + "/") ): relative = posixpath.relpath( namespace_data_directory, destination ) mapped.append( ( len(destination), source if relative == "." else os.path.join( source, *relative.split("/") ), ) ) if not mapped: raise SystemExit(1) mapped.sort(reverse=True) candidate_data_directory = os.path.realpath(mapped[0][1]) namespace_details = os.stat( f"/proc/{postmaster_pid}/root{namespace_data_directory}" ) host_details = os.stat(candidate_data_directory) if ( namespace_details.st_dev != host_details.st_dev or namespace_details.st_ino != host_details.st_ino ): raise SystemExit(1) if candidate_data_directory != data_directory: raise SystemExit(1) postmaster_path = os.path.join(data_directory, "postmaster.pid") postmaster_details = os.lstat(postmaster_path) if ( not stat.S_ISREG(postmaster_details.st_mode) or stat.S_ISLNK(postmaster_details.st_mode) or postmaster_details.st_nlink != 1 or postmaster_details.st_uid != postmaster["uid"] or postmaster_details.st_gid != postmaster["gid"] or postmaster_details.st_mode & 0o022 or postmaster["uid"] != os.stat(data_directory).st_uid ): raise SystemExit(1) lines = open(postmaster_path, "r", encoding="ascii").read().splitlines() if ( len(lines) < 4 or not lines[0].isdigit() or int(lines[0]) != postmaster["namespacePids"][-1] or posixpath.normpath(lines[1]) != namespace_data_directory or not lines[3].isdigit() or int(lines[3]) != server_port ): raise SystemExit(1) # A separate full canonical proof above has already re-read pg_controldata # from this exact persistent data directory. The held guard backend is bound # here to the same postmaster, namespace, endpoint, database OID, and mapped # data inode; its host PID can now be watched with pidfd for the operation. boot_id = open( "/proc/sys/kernel/random/boot_id", "r", encoding="ascii" ).read().strip() token = { "bootId": boot_id, "dataDirectory": data_directory, "postmasterPid": postmaster_pid, "postmasterStartTime": postmaster["startTime"], "postmasterExecutableDevice": postmaster["executableDevice"], "postmasterExecutableInode": postmaster["executableInode"], "postmasterPidFileDevice": postmaster_details.st_dev, "postmasterPidFileInode": postmaster_details.st_ino, "postmasterPidFileUid": postmaster_details.st_uid, "postmasterPidFileGid": postmaster_details.st_gid, "postmasterPidFileMode": stat.S_IMODE(postmaster_details.st_mode), "databaseBackendPid": database_backend_pid, "backendPid": backend_pid, "backendStartTime": backend["startTime"], "backendExecutableDevice": backend["executableDevice"], "backendExecutableInode": backend["executableInode"], } print(json.dumps(token, sort_keys=True, separators=(",", ":"))) PY2 } verify_update_database_guard_identity() { local expected_token="$1" python3 - "${expected_token}" <<'PY2' import json import os import re import secrets import stat import sys import time try: expected = json.loads(sys.argv[1]) except ValueError: raise SystemExit(1) required = { "bootId", "dataDirectory", "postmasterPid", "postmasterStartTime", "postmasterExecutableDevice", "postmasterExecutableInode", "postmasterPidFileDevice", "postmasterPidFileInode", "postmasterPidFileUid", "postmasterPidFileGid", "postmasterPidFileMode", "databaseBackendPid", "backendPid", "backendStartTime", "backendExecutableDevice", "backendExecutableInode", } if not isinstance(expected, dict) or set(expected) != required: raise SystemExit(1) if ( isinstance(expected["databaseBackendPid"], bool) or not isinstance(expected["databaseBackendPid"], int) or not 1 < expected["databaseBackendPid"] <= 2_147_483_647 ): raise SystemExit(1) if ( open("/proc/sys/kernel/random/boot_id", "r", encoding="ascii").read().strip() != expected["bootId"] ): raise SystemExit(1) def current_process(pid): root = f"/proc/{pid}" executable_details = os.stat(os.path.join(root, "exe")) process_stat = open( os.path.join(root, "stat"), "r", encoding="ascii" ).read() split = process_stat.rsplit(")", 1) if len(split) != 2: raise SystemExit(1) fields = split[1].strip().split() if ( len(fields) < 20 or fields[0] in {"Z", "X", "x"} or not fields[1].isdigit() or not fields[19].isdigit() ): raise SystemExit(1) return ( int(fields[1]), int(fields[19]), executable_details.st_dev, executable_details.st_ino, ) postmaster = current_process(expected["postmasterPid"]) backend = current_process(expected["backendPid"]) if ( postmaster[1:] != ( expected["postmasterStartTime"], expected["postmasterExecutableDevice"], expected["postmasterExecutableInode"], ) or backend != ( expected["postmasterPid"], expected["backendStartTime"], expected["backendExecutableDevice"], expected["backendExecutableInode"], ) ): raise SystemExit(1) postmaster_path = os.path.join(expected["dataDirectory"], "postmaster.pid") details = os.lstat(postmaster_path) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_nlink != 1 or details.st_uid != expected["postmasterPidFileUid"] or details.st_gid != expected["postmasterPidFileGid"] or stat.S_IMODE(details.st_mode) != expected["postmasterPidFileMode"] or details.st_mode & 0o022 or details.st_dev != expected["postmasterPidFileDevice"] or details.st_ino != expected["postmasterPidFileInode"] ): raise SystemExit(1) PY2 } start_update_database_guard_pidfd_watchdog() { local expected_token="$1" local ready_path="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/watchdog.ready" local watchdog_expected_parent="${BASHPID}" rm -f -- "${ready_path}" python3 - "${watchdog_expected_parent}" \ "${expected_token}" "${ready_path}" <<'PY2' & import json import ctypes import os import select import signal import sys import time try: parent_pid = int(sys.argv[1]) except ValueError: raise SystemExit(92) expected = json.loads(sys.argv[2]) ready_path = sys.argv[3] libc = ctypes.CDLL(None, use_errno=True) if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: raise SystemExit(92) if os.getppid() != parent_pid: os.kill(os.getpid(), signal.SIGKILL) postmaster_fd = os.pidfd_open(expected["postmasterPid"], 0) backend_fd = os.pidfd_open(expected["backendPid"], 0) poller = select.poll() poller.register(postmaster_fd, select.POLLIN | select.POLLERR | select.POLLHUP) poller.register(backend_fd, select.POLLIN | select.POLLERR | select.POLLHUP) ready_fd = os.open( ready_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) os.write(ready_fd, b"BRIDGESLLM_A009_WATCHDOG_READY\n") os.close(ready_fd) signal.signal(signal.SIGTERM, lambda _signal, _frame: raise_exit()) def raise_exit(): raise SystemExit(0) try: while True: if poller.poll(100): raise SystemExit(92) time.sleep(0.05) finally: os.close(postmaster_fd) os.close(backend_fd) PY2 UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID=$! local attempt marker="" for attempt in {1..100}; do marker="$(cat "${ready_path}" 2>/dev/null || true)" [[ "${marker}" == "BRIDGESLLM_A009_WATCHDOG_READY" ]] && break update_database_guard_process_is_live \ "${UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID}" 2>/dev/null || break sleep 0.05 done rm -f -- "${ready_path}" [[ "${marker}" == "BRIDGESLLM_A009_WATCHDOG_READY" ]] \ && update_database_guard_process_is_live \ "${UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID}" } cleanup_update_database_operation_guard() { if [[ -n "${UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD:-}" ]]; then printf '%s\n' '\q' >&"${UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD}" \ 2>/dev/null || true eval "exec ${UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD}>&-" || true fi if [[ -n "${UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID:-}" ]]; then kill -TERM "${UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID}" 2>/dev/null \ || true wait "${UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID}" 2>/dev/null || true fi if [[ -n "${UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID:-}" ]]; then kill -TERM "${UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID}" 2>/dev/null \ || true wait "${UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID}" 2>/dev/null || true fi if [[ -n "${UPDATE_DATABASE_OPERATION_GUARD_DIR:-}" \ && "${UPDATE_DATABASE_OPERATION_GUARD_DIR}" == \ "${TMPDIR:-/tmp}"/blp-update-dbguard.* ]]; then rm -rf -- "${UPDATE_DATABASE_OPERATION_GUARD_DIR}" fi UPDATE_DATABASE_OPERATION_GUARD_DIR="" UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID="" UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID="" UPDATE_DATABASE_OPERATION_GUARD_BACKEND_PID="" UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD="" UPDATE_DATABASE_OPERATION_GUARD_IDENTITY="" UPDATE_DATABASE_OPERATION_GUARD_INITIAL_TOPOLOGY="" } start_update_database_operation_guard() { local db_url="$1" context="$2" connection_url fifo output local target transaction_id marker="" observed="" identity="" local -a pinned=() assert_update_database_topology_unchanged \ "${db_url}" "${context}-guard-start" || return 1 target="$(current_update_receipt_target)" || return 1 transaction_id="$( read_update_transaction_field "${target}" transaction_id )" || return 1 mapfile -t pinned < <( read_update_disk_reserve_database_topology "${transaction_id}" ) || return 1 [[ "${#pinned[@]}" -eq 3 ]] || return 1 connection_url="$(libpq_database_url "${db_url}")" || return 1 UPDATE_DATABASE_OPERATION_GUARD_DIR="$( mktemp -d "${TMPDIR:-/tmp}/blp-update-dbguard.XXXXXX" )" || return 1 chmod 0700 "${UPDATE_DATABASE_OPERATION_GUARD_DIR}" fifo="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/input" output="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/output" mkfifo -m 0600 "${fifo}" \ || { cleanup_update_database_operation_guard; return 1; } : >"${output}" chmod 0600 "${output}" BRIDGESLLM_UPDATE_PGPASS_EXEC=1 \ BRIDGESLLM_UPDATE_PGPASS_MINIMAL_ENV=1 \ PGAPPNAME=bridgesllm-update-guard \ run_with_update_pgpass "${db_url}" \ stdbuf -oL psql --dbname="${connection_url}" --no-psqlrc -qAt \ -v ON_ERROR_STOP=1 <"${fifo}" >"${output}" 2>>"${LOG_FILE}" & UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID=$! exec {UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD}>"${fifo}" \ || { cleanup_update_database_operation_guard; return 1; } printf '%s\n' \ "SELECT 'BRIDGESLLM_A009_GUARD_READY';" \ >&"${UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD}" update_database_storage_sql >&"${UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD}" local attempt for attempt in {1..100}; do marker="$(sed -n '1p' "${output}" 2>/dev/null || true)" observed="$(sed -n '2p' "${output}" 2>/dev/null || true)" [[ "${marker}" == "BRIDGESLLM_A009_GUARD_READY" && -n "${observed}" ]] \ && break update_database_guard_process_is_live \ "${UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID}" 2>/dev/null \ || break sleep 0.05 done [[ "${marker}" == "BRIDGESLLM_A009_GUARD_READY" \ && -n "${observed}" && "${#observed}" -le 65536 ]] \ || { cleanup_update_database_operation_guard; return 1; } identity="$(attest_update_database_guard_identity \ "${observed}" "${pinned[2]}" "${db_url}")" \ || { cleanup_update_database_operation_guard; return 1; } UPDATE_DATABASE_OPERATION_GUARD_IDENTITY="${identity}" UPDATE_DATABASE_OPERATION_GUARD_INITIAL_TOPOLOGY="${observed}" UPDATE_DATABASE_OPERATION_GUARD_BACKEND_PID="$( python3 - "${identity}" <<'PY2' import json import sys print(json.loads(sys.argv[1])["databaseBackendPid"]) PY2 )" || { cleanup_update_database_operation_guard; return 1; } start_update_database_guard_pidfd_watchdog "${identity}" \ || { cleanup_update_database_operation_guard; return 1; } } finish_update_database_operation_guard() { local db_url="$1" context="$2" output marker="" observed="" identity="" output="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/output" verify_update_database_guard_identity \ "${UPDATE_DATABASE_OPERATION_GUARD_IDENTITY}" || return 1 printf '%s\n' \ "SELECT 'BRIDGESLLM_A009_GUARD_FINAL';" \ >&"${UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD}" || return 1 update_database_storage_sql >&"${UPDATE_DATABASE_OPERATION_GUARD_INPUT_FD}" \ || return 1 local attempt for attempt in {1..100}; do marker="$(sed -n '3p' "${output}" 2>/dev/null || true)" observed="$(sed -n '4p' "${output}" 2>/dev/null || true)" [[ "${marker}" == "BRIDGESLLM_A009_GUARD_FINAL" && -n "${observed}" ]] \ && break kill -0 "${UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID}" 2>/dev/null \ || break sleep 0.05 done [[ "${marker}" == "BRIDGESLLM_A009_GUARD_FINAL" \ && -n "${observed}" \ && "${observed}" == "${UPDATE_DATABASE_OPERATION_GUARD_INITIAL_TOPOLOGY}" ]] \ || return 1 local target transaction_id local -a pinned=() target="$(current_update_receipt_target)" || return 1 transaction_id="$( read_update_transaction_field "${target}" transaction_id )" || return 1 mapfile -t pinned < <( read_update_disk_reserve_database_topology "${transaction_id}" ) || return 1 [[ "${#pinned[@]}" -eq 3 ]] || return 1 identity="$(attest_update_database_guard_identity \ "${observed}" "${pinned[2]}" "${db_url}")" || return 1 [[ "${identity}" == "${UPDATE_DATABASE_OPERATION_GUARD_IDENTITY}" ]] \ || return 1 assert_update_database_topology_unchanged \ "${db_url}" "${context}-guard-complete" } update_database_operation_supervisor_python() { cat <<'PY2' import ctypes import os import re import signal import sys try: original_parent = int(sys.argv[1]) except (IndexError, ValueError): raise SystemExit(92) libc = ctypes.CDLL(None, use_errno=True) if libc.prctl(1, signal.SIGTERM, 0, 0, 0) != 0: raise SystemExit(92) if os.getppid() != original_parent: os.kill(os.getpid(), signal.SIGKILL) context, ready_fifo, start_fifo, completion_path = sys.argv[2:6] command = sys.argv[6:] if not command: raise SystemExit(92) def host_process_start_time(pid): text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() split = text.rsplit(")", 1) if len(split) != 2: raise SystemExit(92) fields = split[1].strip().split() if len(fields) < 20 or fields[0] in {"Z", "X", "x"}: raise SystemExit(92) return int(fields[19]) def write_completion(supervisor_pid, supervisor_start, status): payload = ( "BRIDGESLLM_A009_OPERATION_COMPLETE_V1 " f"{supervisor_pid} {supervisor_start} {status}\n" ).encode("ascii") flags = ( os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) ) descriptor = os.open(completion_path, flags, 0o600) try: os.fchmod(descriptor, 0o600) if os.write(descriptor, payload) != len(payload): raise OSError("short completion write") os.fsync(descriptor) finally: os.close(descriptor) directory = os.open( os.path.dirname(completion_path), os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), ) try: os.fsync(directory) finally: os.close(directory) # Put the destructive command in a private PID namespace whose init process is # the descendant supervisor below. Linux kills every remaining process in a # PID namespace if its init dies, including setsid()/double-fork escape # attempts. The small outer monitor is the exact process tracked by the shell; # its death also SIGKILLs the namespace init through PDEATHSIG. outer_supervisor_pid = os.getpid() outer_supervisor_start = host_process_start_time(outer_supervisor_pid) CLONE_NEWPID = 0x20000000 if libc.unshare(CLONE_NEWPID) != 0: raise SystemExit(92) namespace_init_host_pid = os.fork() if namespace_init_host_pid != 0: os.close(3) while True: try: _waited, namespace_status = os.waitpid(namespace_init_host_pid, 0) break except InterruptedError: continue if os.WIFEXITED(namespace_status): outer_status = os.WEXITSTATUS(namespace_status) elif os.WIFSIGNALED(namespace_status): outer_status = 128 + os.WTERMSIG(namespace_status) else: outer_status = 92 # waitpid above reaped PID-namespace init. The kernel guarantees every # other member was killed before namespace init could disappear. write_completion( outer_supervisor_pid, outer_supervisor_start, outer_status ) raise SystemExit(outer_status) if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: raise SystemExit(92) host_stat = open("/proc/self/stat", "r", encoding="ascii").read() host_fields = host_stat.rsplit(")", 1)[1].strip().split() if ( len(host_fields) < 20 or host_fields[0] in {"Z", "X", "x"} or int(host_fields[1]) != outer_supervisor_pid ): os.kill(os.getpid(), signal.SIGKILL) operation_root_pid = int(host_stat.split("(", 1)[0].strip()) if libc.prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER raise SystemExit(92) import json import time from urllib.parse import unquote, urlsplit raw_parts = [] raw_size = 0 while True: part = os.read(3, min(65536, 128001 - raw_size)) if not part: break raw_parts.append(part) raw_size += len(part) if raw_size > 128000: raise SystemExit(92) os.close(3) raw_bytes = b"".join(raw_parts) try: database_url = raw_bytes.decode("utf-8") except UnicodeDecodeError: raise SystemExit(92) if database_url.endswith("\n"): database_url = database_url[:-1] if not database_url or "#" in database_url or re.search( r"%(?![0-9A-Fa-f]{2})", database_url ) or any( ord(character) < 32 or ord(character) == 127 for character in database_url ): raise SystemExit(92) def pgpass_payload(raw): try: parsed = urlsplit(raw) _, separator, host_part = parsed.netloc.rpartition("@") host = unquote(parsed.hostname or "", errors="strict") port_number = 5432 if parsed.port is None else parsed.port database = unquote((parsed.path or "").lstrip("/"), errors="strict") user = unquote(parsed.username or "", errors="strict") password = unquote(parsed.password or "", errors="strict") decoded_host_part = unquote(host_part, errors="strict") raw_port_suffix = ( host_part[host_part.find("]") + 1:] if host_part.startswith("[") else f":{host_part.rsplit(':', 1)[1]}" if ":" in host_part else "" ) except (UnicodeDecodeError, ValueError): raise SystemExit(92) forbidden = { "password", "sslpassword", "passfile", "service", "servicefile", "host", "hostaddr", "port", "user", "dbname", "database", } for raw_pair in parsed.query.split("&") if parsed.query else (): if "+" in raw_pair: raise SystemExit(92) raw_key, pair_separator, raw_value = raw_pair.partition("=") if "=" in raw_value: raise SystemExit(92) try: key = unquote(raw_key, errors="strict").lower() value = unquote(raw_value, errors="strict") except UnicodeDecodeError: raise SystemExit(92) if ( not pair_separator or not key or key in forbidden or any( ord(character) < 32 or ord(character) == 127 for value_part in (key, value) for character in value_part ) ): raise SystemExit(92) if ( parsed.scheme not in {"postgres", "postgresql"} or parsed.fragment or re.search( r"%(?:23|24|26|2b|2c|2f|3a|3b|3d|3f|40)", parsed.path, re.I, ) or not parsed.path.startswith("/") or parsed.path.count("/") != 1 or database in {".", ".."} or "/" in database or not separator or parsed.netloc.count("@") != 1 or host_part.startswith("[") or not 1 <= port_number <= 65535 or "," in decoded_host_part or (raw_port_suffix and not re.fullmatch(r":[1-9][0-9]*", raw_port_suffix)) or "%" in host or "%" in (parsed.hostname or "") or re.search(r"[<>\\^|]", host) or host_part != host_part.lower() or not host.isascii() or any(character.isspace() for character in host) ): raise SystemExit(92) values = (host, str(port_number), database, user, password) if not all(values) or any( ord(character) < 32 or ord(character) == 127 for value in values for character in value ): raise SystemExit(92) escape = lambda value: value.replace("\\", "\\\\").replace(":", "\\:") return (":".join(escape(value) for value in values) + "\n").encode() def process_start_time(pid): text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() split = text.rsplit(")", 1) if len(split) != 2: raise SystemExit(92) fields = split[1].strip().split() if len(fields) < 20 or fields[0] in {"Z", "X", "x"}: raise SystemExit(92) return int(fields[19]) os.setsid() supervisor_pid = outer_supervisor_pid supervisor_start = process_start_time(supervisor_pid) stop_requested = 0 def request_stop(signal_number, _frame): global stop_requested stop_requested = signal_number for handled_signal in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): signal.signal(handled_signal, request_stop) child_pid = os.fork() if child_pid == 0: os.setpgid(0, 0) if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: os._exit(92) child_stat = open("/proc/self/stat", "r", encoding="ascii").read() child_fields = child_stat.rsplit(")", 1)[1].strip().split() if ( len(child_fields) < 20 or child_fields[0] in {"Z", "X", "x"} or int(child_fields[1]) != operation_root_pid ): os.kill(os.getpid(), signal.SIGKILL) child_pid = int(child_stat.split("(", 1)[0].strip()) metadata = { "schema": "bridgesllm-update-operation-ready-v1", "supervisorPid": supervisor_pid, "supervisorStartTime": supervisor_start, "childPid": child_pid, "childStartTime": int(child_fields[19]), "childPgid": int(child_fields[2]), } try: descriptor = os.open( ready_fifo, os.O_WRONLY | getattr(os, "O_CLOEXEC", 0) ) os.write( descriptor, ( json.dumps(metadata, sort_keys=True, separators=(",", ":")) + "\n" ).encode("ascii"), ) os.close(descriptor) descriptor = os.open( start_fifo, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) ) token = b"" while not token.endswith(b"\n") and len(token) <= 16: part = os.read(descriptor, 16) if not part: break token += part os.close(descriptor) if token != b"START\n": os._exit(92) environment = os.environ.copy() environment.pop("DATABASE_URL", None) for key in tuple(environment): if key.startswith("PG") and key != "PGAPPNAME": environment.pop(key, None) if context in {"prisma-migrate", "prisma-schema-diff"}: environment["DATABASE_URL"] = database_url else: payload = pgpass_payload(database_url) credential_fd = os.memfd_create("bridgesllm-pgpass", 0) os.fchmod(credential_fd, 0o600) if os.write(credential_fd, payload) != len(payload): os._exit(92) os.lseek(credential_fd, 0, os.SEEK_SET) os.set_inheritable(credential_fd, True) environment["PGPASSFILE"] = f"/proc/self/fd/{credential_fd}" os.execvpe(command[0], command, environment) except BaseException: os._exit(92) def process_identity(pid): text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() split = text.rsplit(")", 1) if len(split) != 2: raise ProcessLookupError fields = split[1].strip().split() if len(fields) < 20 or fields[0] in {"Z", "X", "x"}: raise ProcessLookupError return ( int(fields[1]), int(fields[2]), int(fields[3]), int(fields[19]), ) def owned_members(): identities = {} for entry in os.scandir("/proc"): if not entry.name.isdigit(): continue try: pid = int(entry.name) if pid == operation_root_pid: continue identities[pid] = process_identity(pid) except ( FileNotFoundError, PermissionError, ProcessLookupError, ValueError, ): continue # The supervisor is a child subreaper. A recursive PPID closure covers # ordinary children, setsid()/setpgid() escape attempts, and double-forked # descendants once they are reparented to this exact supervisor. parents = {operation_root_pid} members = {} changed = True while changed: changed = False for pid, (ppid, _pgid, _sid, start_time) in identities.items(): if pid not in members and ppid in parents: members[pid] = start_time parents.add(pid) changed = True return sorted(members.items()) def group_exists(): return bool(owned_members()) def reap_nonblocking(): result = None while True: try: waited_pid, status = os.waitpid(-1, os.WNOHANG) except ChildProcessError: return result if waited_pid == 0: return result if waited_pid == child_pid: result = status def settle_descendants(force): signal_number = signal.SIGKILL if force else signal.SIGTERM members = dict(owned_members()) for pid, expected_start in sorted(members.items(), reverse=True): descriptor = None try: descriptor = os.pidfd_open(pid, 0) if dict(owned_members()).get(pid) != expected_start: continue signal.pidfd_send_signal(descriptor, signal_number) except (FileNotFoundError, PermissionError, ProcessLookupError, OSError): continue finally: if descriptor is not None: os.close(descriptor) def drain_descendants(): settle_descendants(False) deadline = time.monotonic() + 3.0 while time.monotonic() < deadline: reap_nonblocking() if not group_exists(): return True settle_descendants(False) time.sleep(0.05) settle_descendants(True) deadline = time.monotonic() + 3.0 while time.monotonic() < deadline: reap_nonblocking() if not group_exists(): return True settle_descendants(True) time.sleep(0.05) reap_nonblocking() return not group_exists() child_status = None while child_status is None and not stop_requested: try: waited_pid, status = os.waitpid(child_pid, os.WNOHANG) except ChildProcessError: break if waited_pid == child_pid: child_status = status break time.sleep(0.05) unexpected_tree = group_exists() daemonized = child_status is not None and unexpected_tree if (stop_requested or unexpected_tree) and not drain_descendants(): raise SystemExit(92) reap_nonblocking() if group_exists(): raise SystemExit(92) if stop_requested: final_status = 128 + stop_requested elif daemonized: final_status = 92 elif child_status is None: final_status = 92 elif os.WIFEXITED(child_status): final_status = os.WEXITSTATUS(child_status) elif os.WIFSIGNALED(child_status): final_status = 128 + os.WTERMSIG(child_status) else: final_status = 92 raise SystemExit(final_status) PY2 } update_database_process_matches() { local pid="$1" start_time="$2" expected_pgid="${3:-}" expected_sid="${4:-}" python3 - "${pid}" "${start_time}" "${expected_pgid}" "${expected_sid}" <<'PY2' import os import sys pid_text, expected_start, expected_pgid, expected_sid = sys.argv[1:] if not pid_text.isdigit() or not expected_start.isdigit(): raise SystemExit(1) pid = int(pid_text) if pid <= 1: raise SystemExit(1) text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() split = text.rsplit(")", 1) if len(split) != 2: raise SystemExit(1) fields = split[1].strip().split() if ( len(fields) < 20 or fields[0] in {"Z", "X", "x"} or fields[19] != expected_start ): raise SystemExit(1) if expected_pgid and os.getpgid(pid) != int(expected_pgid): raise SystemExit(1) if expected_sid and os.getsid(pid) != int(expected_sid): raise SystemExit(1) PY2 } signal_update_database_operation_descendants() { local supervisor_pid="$1" supervisor_start="$2" signal_number="$3" python3 - "${supervisor_pid}" "${supervisor_start}" \ "${signal_number}" <<'PY2' import os import signal import sys if ( not all(value.isdigit() and int(value) > 1 for value in sys.argv[1:3]) or sys.argv[3] not in {"9", "15"} or not hasattr(os, "pidfd_open") or not hasattr(signal, "pidfd_send_signal") ): raise SystemExit(1) supervisor_pid, supervisor_start, signal_number = map(int, sys.argv[1:]) def identity(pid): text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() split = text.rsplit(")", 1) if len(split) != 2: raise ProcessLookupError fields = split[1].strip().split() if len(fields) < 20 or fields[0] in {"Z", "X", "x"}: raise ProcessLookupError return int(fields[1]), int(fields[19]) supervisor_descriptor = None try: supervisor_descriptor = os.pidfd_open(supervisor_pid, 0) _supervisor_parent, observed_start = identity(supervisor_pid) if observed_start != supervisor_start: raise SystemExit(2) except (FileNotFoundError, PermissionError, ProcessLookupError, OSError): raise SystemExit(2) identities = {} for entry in os.scandir("/proc"): if not entry.name.isdigit(): continue pid = int(entry.name) if pid == supervisor_pid: continue try: identities[pid] = identity(pid) except (FileNotFoundError, PermissionError, ProcessLookupError, ValueError): continue parents = {supervisor_pid} members = {} changed = True while changed: changed = False for pid, (ppid, start_time) in identities.items(): if pid not in members and ppid in parents: members[pid] = start_time parents.add(pid) changed = True for pid, expected_start in sorted(members.items(), reverse=True): descriptor = None try: descriptor = os.pidfd_open(pid, 0) _ppid, start_time = identity(pid) _supervisor_parent, live_supervisor_start = identity(supervisor_pid) if ( start_time != expected_start or live_supervisor_start != supervisor_start ): continue signal.pidfd_send_signal(descriptor, signal_number) except (FileNotFoundError, PermissionError, ProcessLookupError, OSError): continue finally: if descriptor is not None: os.close(descriptor) if supervisor_descriptor is not None: os.close(supervisor_descriptor) print(len(members)) PY2 } signal_update_database_operation_process_exact() { local pid="$1" start_time="$2" signal_number="$3" python3 - "${pid}" "${start_time}" "${signal_number}" <<'PY2' import os import signal import sys pid_text, start_text, signal_text = sys.argv[1:] if ( not pid_text.isdigit() or int(pid_text) <= 1 or not start_text.isdigit() or signal_text not in {"9", "15"} or not hasattr(os, "pidfd_open") or not hasattr(signal, "pidfd_send_signal") ): raise SystemExit(1) pid = int(pid_text) descriptor = os.pidfd_open(pid, 0) try: text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() fields = text.rsplit(")", 1)[1].strip().split() if ( len(fields) < 20 or fields[0] in {"Z", "X", "x"} or fields[19] != start_text ): raise SystemExit(2) signal.pidfd_send_signal(descriptor, int(signal_text)) finally: os.close(descriptor) PY2 } read_update_database_operation_completion() { local supervisor_pid="$1" supervisor_start="$2" local completion_path="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/operation.complete" python3 - "${completion_path}" "${supervisor_pid}" \ "${supervisor_start}" <<'PY2' import os import stat import sys path, expected_pid, expected_start = sys.argv[1:] flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) descriptor = os.open(path, flags) try: metadata = os.fstat(descriptor) if ( not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_uid != os.geteuid() or metadata.st_gid != os.getegid() or metadata.st_size > 128 ): raise SystemExit(1) raw = os.read(descriptor, 129) finally: os.close(descriptor) parts = raw.decode("ascii", errors="strict").rstrip("\n").split(" ") if ( len(parts) != 4 or parts[0] != "BRIDGESLLM_A009_OPERATION_COMPLETE_V1" or parts[1] != expected_pid or parts[2] != expected_start or not parts[3].isdigit() or not 0 <= int(parts[3]) <= 255 ): raise SystemExit(1) print(parts[3]) PY2 } launch_update_database_operation() { local db_url="$1" context="$2" shift 2 local ready_fifo="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/operation.ready" local start_fifo="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/operation.start" local completion_path="${UPDATE_DATABASE_OPERATION_GUARD_DIR}/operation.complete" local ready_fd start_fd supervisor_pid supervisor_start="" local metadata="" parsed="" child_pid="" child_start="" child_pgid="" local runner_source supervisor_expected_parent="${BASHPID}" runner_source="$(update_database_operation_supervisor_python)" || return 1 [[ ! -e "${completion_path}" && ! -L "${completion_path}" ]] || return 1 mkfifo -m 0600 "${ready_fifo}" "${start_fifo}" || return 1 exec {ready_fd}<>"${ready_fifo}" || return 1 exec {start_fd}<>"${start_fifo}" || { eval "exec ${ready_fd}>&-" return 1 } env -u DATABASE_URL -u PGHOST -u PGHOSTADDR -u PGPORT \ -u PGDATABASE -u PGUSER -u PGPASSWORD -u PGPASSFILE \ -u PGSERVICE -u PGSERVICEFILE -u PGOPTIONS -u PGAPPNAME \ -u PGSSLMODE -u PGREQUIRESSL -u PGSSLCOMPRESSION \ -u PGSSLCERT -u PGSSLKEY -u PGSSLROOTCERT -u PGSSLCRL \ -u PGREQUIREPEER -u PGCHANNELBINDING -u PGTARGETSESSIONATTRS \ -u PGCONNECT_TIMEOUT -u PGCLIENTENCODING -u PGKRBSRVNAME \ -u PGGSSLIB -u PGSYSCONFDIR -u PGLOCALEDIR \ BRIDGESLLM_DATABASE_GUARD_BACKEND_PID="$( printf '%s' "${UPDATE_DATABASE_OPERATION_GUARD_BACKEND_PID:-4242}" )" \ python3 -c "${runner_source}" "${supervisor_expected_parent}" \ "${context}" "${ready_fifo}" "${start_fifo}" "${completion_path}" "$@" \ 3< <(printf '%s' "${db_url}") & supervisor_pid=$! supervisor_start="$( python3 - "${supervisor_pid}" <<'PY2' import sys text = open(f"/proc/{sys.argv[1]}/stat", "r", encoding="ascii").read() fields = text.rsplit(")", 1)[1].strip().split() if len(fields) < 20 or fields[0] in {"Z", "X", "x"}: raise SystemExit(1) print(fields[19]) PY2 )" || supervisor_start="" [[ "${supervisor_start}" =~ ^[1-9][0-9]*$ ]] || { kill -TERM "${supervisor_pid}" 2>/dev/null || true wait "${supervisor_pid}" 2>/dev/null || true eval "exec ${ready_fd}>&-" eval "exec ${start_fd}>&-" rm -f -- "${ready_fifo}" "${start_fifo}" return 1 } UPDATE_DATABASE_OPERATION_SUPERVISOR_PID="${supervisor_pid}" UPDATE_DATABASE_OPERATION_LATCH="${supervisor_pid}:${supervisor_start}:0:0:0" IFS= read -r -t 10 -u "${ready_fd}" metadata || metadata="" parsed="$( python3 - "${metadata}" "${supervisor_pid}" "${supervisor_start}" <<'PY2' import json import os import sys try: value = json.loads(sys.argv[1]) except ValueError: raise SystemExit(1) required = { "schema", "supervisorPid", "supervisorStartTime", "childPid", "childStartTime", "childPgid", } if ( not isinstance(value, dict) or set(value) != required or value["schema"] != "bridgesllm-update-operation-ready-v1" or value["supervisorPid"] != int(sys.argv[2]) or value["supervisorStartTime"] != int(sys.argv[3]) or value["childPid"] <= 1 or value["childPgid"] != value["childPid"] or value["childStartTime"] <= 0 ): raise SystemExit(1) pid = value["childPid"] text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() fields = text.rsplit(")", 1)[1].strip().split() if ( len(fields) < 20 or fields[0] in {"Z", "X", "x"} or int(fields[19]) != value["childStartTime"] or os.getpgid(pid) != value["childPgid"] ): raise SystemExit(1) print(value["childPid"], value["childStartTime"], value["childPgid"]) PY2 )" || parsed="" read -r child_pid child_start child_pgid <<<"${parsed}" if [[ ! "${child_pid}" =~ ^[1-9][0-9]*$ \ || ! "${child_start}" =~ ^[1-9][0-9]*$ \ || "${child_pgid}" != "${child_pid}" ]]; then kill -TERM "${supervisor_pid}" 2>/dev/null || true wait "${supervisor_pid}" 2>/dev/null || true UPDATE_DATABASE_OPERATION_LATCH="" UPDATE_DATABASE_OPERATION_SUPERVISOR_PID="" eval "exec ${ready_fd}>&-" eval "exec ${start_fd}>&-" rm -f -- "${ready_fifo}" "${start_fifo}" return 1 fi # Publish the complete identity before releasing the child to exec. UPDATE_DATABASE_OPERATION_LATCH="${supervisor_pid}:${supervisor_start}:${child_pid}:${child_start}:${child_pgid}" local test_pre_go_guard_failure=false if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_PRE_GO_GUARD_FAIL:-0}" \ == "1" ]]; then test_pre_go_guard_failure=true fi if ${test_pre_go_guard_failure} \ || { [[ -n "${UPDATE_DATABASE_OPERATION_GUARD_IDENTITY:-}" ]] \ && { ! update_database_guard_process_is_live \ "${UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID}" \ || ! update_database_guard_process_is_live \ "${UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID}" \ || ! verify_update_database_guard_identity \ "${UPDATE_DATABASE_OPERATION_GUARD_IDENTITY}"; }; }; then eval "exec ${ready_fd}>&-" || true eval "exec ${start_fd}>&-" || true rm -f -- "${ready_fifo}" "${start_fifo}" settle_active_update_database_operation >/dev/null 2>&1 || return 93 return 1 fi if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_START_WRITE_FAIL:-0}" \ == "1" ]]; then eval "exec ${start_fd}>&-" || true fi printf '%s\n' START >&"${start_fd}" || { eval "exec ${ready_fd}>&-" || true eval "exec ${start_fd}>&-" || true rm -f -- "${ready_fifo}" "${start_fifo}" settle_active_update_database_operation >/dev/null 2>&1 || return 93 return 1 } eval "exec ${ready_fd}>&-" eval "exec ${start_fd}>&-" rm -f -- "${ready_fifo}" "${start_fifo}" } settle_active_update_database_operation() { if [[ "${UPDATE_DATABASE_OPERATION_SETTLEMENT_IN_PROGRESS:-false}" \ == "true" ]]; then return 1 fi UPDATE_DATABASE_OPERATION_SETTLEMENT_IN_PROGRESS=true if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_SETTLEMENT_FAIL:-0}" \ == "1" ]]; then UPDATE_DATABASE_OPERATION_SETTLEMENT_IN_PROGRESS=false return 1 fi local latch="${UPDATE_DATABASE_OPERATION_LATCH:-}" local supervisor_pid="" supervisor_start="" child_pid="" child_start="" child_pgid="" local status=0 attempt descendant_count="" completion_status="" IFS=: read -r supervisor_pid supervisor_start child_pid child_start child_pgid \ <<<"${latch}" if [[ -z "${latch}" ]]; then cleanup_update_database_operation_guard UPDATE_DATABASE_OPERATION_SETTLEMENT_IN_PROGRESS=false return 0 fi if [[ ! "${supervisor_pid}" =~ ^[1-9][0-9]*$ \ || ! "${supervisor_start}" =~ ^[1-9][0-9]*$ ]]; then status=1 elif update_database_process_matches \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null; then # Signal only descendants proven to belong to the exact live supervisor. # TERM gets three seconds, then KILL gets three seconds. The supervisor is # never SIGKILLed: it alone may publish the completion proof, and recovery # must fail closed if that proof cannot be produced. for attempt in {1..60}; do descendant_count="$( signal_update_database_operation_descendants \ "${supervisor_pid}" "${supervisor_start}" 15 2>/dev/null )" || descendant_count="" if [[ "${descendant_count}" == "0" ]]; then signal_update_database_operation_process_exact \ "${supervisor_pid}" "${supervisor_start}" 15 2>/dev/null || true break fi update_database_process_matches \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null || break if (( attempt >= 30 )); then descendant_count="$( signal_update_database_operation_descendants \ "${supervisor_pid}" "${supervisor_start}" 9 2>/dev/null )" || descendant_count="" fi sleep 0.1 done for attempt in {1..60}; do update_database_process_matches \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null || break sleep 0.1 done if update_database_process_matches \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null; then status=1 fi fi if [[ "${supervisor_pid}" =~ ^[1-9][0-9]*$ ]]; then wait "${supervisor_pid}" 2>/dev/null || true fi completion_status="$( read_update_database_operation_completion \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null )" || completion_status="" [[ "${completion_status}" =~ ^[0-9]+$ ]] || status=1 if [[ "${status}" -eq 0 ]]; then cleanup_update_database_operation_guard UPDATE_DATABASE_OPERATION_LATCH="" UPDATE_DATABASE_OPERATION_SUPERVISOR_PID="" fi UPDATE_DATABASE_OPERATION_SETTLEMENT_IN_PROGRESS=false return "${status}" } update_database_guard_process_is_live() { local pid="$1" python3 - "${pid}" <<'PY2' import sys pid = sys.argv[1] if not pid.isdigit() or int(pid) <= 1: raise SystemExit(1) text = open(f"/proc/{pid}/stat", "r", encoding="ascii").read() fields = text.rsplit(")", 1)[1].strip().split() if len(fields) < 20 or fields[0] in {"Z", "X", "x"}: raise SystemExit(1) PY2 } run_contained_database_operation() { local db_url="$1" context="$2" shift 2 [[ "$#" -gt 0 ]] || return 1 UPDATE_DATABASE_OPERATION_GUARD_DIR="$( mktemp -d "${TMPDIR:-/tmp}/blp-update-dbguard.XXXXXX" )" || return 92 chmod 0700 "${UPDATE_DATABASE_OPERATION_GUARD_DIR}" || { cleanup_update_database_operation_guard return 92 } local launch_status=0 operation_status=0 completion_status="" launch_update_database_operation "${db_url}" "${context}" "$@" \ || launch_status=$? if [[ "${launch_status}" -ne 0 ]]; then [[ "${launch_status}" -ne 93 ]] || return 93 if [[ -n "${UPDATE_DATABASE_OPERATION_LATCH:-}" ]]; then settle_active_update_database_operation || return 93 else cleanup_update_database_operation_guard fi return 92 fi local supervisor_pid="${UPDATE_DATABASE_OPERATION_SUPERVISOR_PID}" local supervisor_start="${UPDATE_DATABASE_OPERATION_LATCH#*:}" supervisor_start="${supervisor_start%%:*}" while update_database_process_matches \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null; do sleep 0.1 done wait "${supervisor_pid}" || operation_status=$? completion_status="$( read_update_database_operation_completion \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null )" || completion_status="" if [[ ! "${completion_status}" =~ ^[0-9]+$ \ || "${completion_status}" -ne "${operation_status}" ]]; then return 93 fi cleanup_update_database_operation_guard UPDATE_DATABASE_OPERATION_LATCH="" UPDATE_DATABASE_OPERATION_SUPERVISOR_PID="" return "${operation_status}" } run_attested_database_operation() { local db_url="$1" context="$2" shift 2 [[ "$#" -gt 0 ]] || return 1 local source_only=false operation_status=0 guard_status=0 guard_started=false if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && -n "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" ]]; then source_only=true fi if ${source_only}; then assert_update_database_topology_unchanged \ "${db_url}" "${context}-guard-start" || return 92 if [[ "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_GUARD_FAIL_AFTER_READY:-0}" \ == "1" ]]; then return 92 fi UPDATE_DATABASE_OPERATION_GUARD_DIR="$( mktemp -d "${TMPDIR:-/tmp}/blp-update-dbguard.XXXXXX" )" || return 92 chmod 0700 "${UPDATE_DATABASE_OPERATION_GUARD_DIR}" else start_update_database_operation_guard "${db_url}" "${context}" || return 92 guard_started=true fi local launch_status=0 launch_update_database_operation "${db_url}" "${context}" "$@" \ || launch_status=$? if [[ "${launch_status}" -ne 0 ]]; then [[ "${launch_status}" -ne 93 ]] || return 93 if [[ -n "${UPDATE_DATABASE_OPERATION_LATCH:-}" ]]; then settle_active_update_database_operation || return 93 else cleanup_update_database_operation_guard fi return 92 fi local supervisor_pid="${UPDATE_DATABASE_OPERATION_SUPERVISOR_PID}" local supervisor_start="${UPDATE_DATABASE_OPERATION_LATCH#*:}" supervisor_start="${supervisor_start%%:*}" while update_database_process_matches \ "${supervisor_pid}" \ "${supervisor_start}" 2>/dev/null; do if ${guard_started} \ && { ! update_database_guard_process_is_live \ "${UPDATE_DATABASE_OPERATION_GUARD_CLIENT_PID}" \ || ! update_database_guard_process_is_live \ "${UPDATE_DATABASE_OPERATION_GUARD_WATCHDOG_PID}" \ || ! verify_update_database_guard_identity \ "${UPDATE_DATABASE_OPERATION_GUARD_IDENTITY}"; }; then guard_status=92 settle_active_update_database_operation || return 93 break fi sleep 0.1 done if [[ "${guard_status}" -eq 0 ]]; then wait "${supervisor_pid}" || operation_status=$? local completion_status="" completion_status="$( read_update_database_operation_completion \ "${supervisor_pid}" "${supervisor_start}" 2>/dev/null )" || completion_status="" if [[ ! "${completion_status}" =~ ^[0-9]+$ \ || "${completion_status}" -ne "${operation_status}" ]]; then # The exact supervisor is the only process authorized to prove its # recursive descendant tree empty. Never recover or clear the boot fence # after an unexpected supervisor death or a forged/mismatched receipt. return 93 fi if [[ "${guard_status}" -eq 0 ]]; then if ${guard_started}; then finish_update_database_operation_guard "${db_url}" "${context}" \ || guard_status=92 else assert_update_database_topology_unchanged \ "${db_url}" "${context}-guard-complete" || guard_status=92 fi cleanup_update_database_operation_guard UPDATE_DATABASE_OPERATION_LATCH="" UPDATE_DATABASE_OPERATION_SUPERVISOR_PID="" fi fi [[ "${guard_status}" -eq 0 ]] || return "${guard_status}" return "${operation_status}" } attest_update_database_ownership() { # Rollback recreates the public schema and its objects as the connecting # role with no explicit grants. That is only lossless when the database # already follows the installer contract: the DATABASE_URL role owns the # database, the public schema, and every dumpable public-schema object in it, # and no schema/object/column/default ACL or large-object state would be # flattened by --no-owner --no-privileges. This includes relations, routines, # types, collations, conversions, operators, operator classes/families, and # text-search dictionaries/configurations, extended statistics, and # extensions assigned to public. Extra user schemas are outside # the product contract and are rejected instead of silently surviving with # split database authority. local db_url="$1" context="${2:-admission}" local expected_variant="${3:-}" local connection_url result violations variant extra local status=0 connection_url="$(libpq_database_url "${db_url}")" || return 1 result="$(update_database_ownership_violations_sql \ | run_with_update_pgpass "${db_url}" \ psql --dbname="${connection_url}" --no-psqlrc \ -v ON_ERROR_STOP=1 -qAt \ 2>> "${LOG_FILE}")" || status=$? [[ "${status}" -eq 0 ]] || return 1 result="$(tr -d '[:space:]' <<<"${result}")" IFS='|' read -r violations variant extra <<<"${result}" [[ "${violations}" =~ ^[0-9]+$ \ && "${variant}" =~ ^(owner-null|pg-database-owner-default)$ \ && -z "${extra}" ]] || return 1 if [[ "${violations}" != "0" ]]; then echo "database ownership contract violations (${context}): ${violations}" \ >> "${LOG_FILE}" return 1 fi if [[ -n "${expected_variant}" && "${variant}" != "${expected_variant}" ]]; then echo "database ownership contract variant changed (${context}): expected ${expected_variant}, got ${variant}" \ >> "${LOG_FILE}" return 1 fi UPDATE_RECOVERY_DATABASE_CONTRACT_VARIANT="${variant}" } backup_database_for_update() { local db_url="$1" backup_dir="$2" local connection_url dump [[ -n "${db_url}" ]] || return 1 assert_update_database_topology_unchanged "${db_url}" backup \ || return 1 connection_url="$(libpq_database_url "${db_url}")" || return 1 dump="${backup_dir}/database-before-update.dump" if ! run_attested_database_operation \ "${db_url}" backup \ pg_dump --format=custom --no-owner --no-privileges \ --dbname="${connection_url}" --file="${dump}" \ >> "${LOG_FILE}" 2>&1; then rm -f -- "${dump}" return 1 fi if ! assert_update_database_topology_unchanged \ "${db_url}" backup-complete; then rm -f -- "${dump}" return 1 fi [[ -s "${dump}" ]] || return 1 chmod 0600 "${dump}" pg_restore --list "${dump}" >> "${LOG_FILE}" 2>&1 || return 1 fsync_update_payload "${dump}" || return 1 UPDATE_RECOVERY_DATABASE_DUMP="${dump}" UPDATE_RECOVERY_DATABASE_URL="${db_url}" ok "Database rollback snapshot created" } snapshot_update_database_contract() { local backup_dir="$1" local destination="${backup_dir}/database-contract.variant" local variant="${UPDATE_RECOVERY_DATABASE_CONTRACT_VARIANT:-}" [[ "${variant}" =~ ^(owner-null|pg-database-owner-default)$ \ && -d "${backup_dir}" && ! -L "${backup_dir}" \ && ! -e "${destination}" && ! -L "${destination}" ]] || return 1 (umask 077; printf '%s\n' "${variant}" > "${destination}") || return 1 chmod 0600 "${destination}" || return 1 fsync_update_payload "${destination}" } read_update_database_contract_snapshot() { local target="$1" backup_dir="$2" local manifest path="${backup_dir}/database-contract.variant" manifest="$(update_transaction_manifest_path "${target}" database-contract)" \ || return 1 update_artifact_manifest verify "${path}" "${manifest}" false || return 1 python3 - "${path}" <<'PY2' import os import stat import sys path = sys.argv[1] details = os.lstat(path) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or not 1 <= details.st_size <= 64 ): raise SystemExit(1) value = open(path, "r", encoding="ascii").read() if value not in {"owner-null\n", "pg-database-owner-default\n"}: raise SystemExit(1) print(value.rstrip("\n")) PY2 } restore_database_after_failed_update() { local db_url="${UPDATE_RECOVERY_DATABASE_URL:-}" dump="${UPDATE_RECOVERY_DATABASE_DUMP:-}" local status=0 [[ -n "${db_url}" && -s "${dump}" ]] || return 1 [[ "${UPDATE_RECOVERY_DATABASE_CONTRACT_VARIANT:-}" \ =~ ^(owner-null|pg-database-owner-default)$ ]] || status=1 if [[ "${status}" -eq 0 ]]; then restore_database_with_pgpass "${dump}" \ "${UPDATE_RECOVERY_DATABASE_CONTRACT_VARIANT}" "${db_url}" \ || status=1 fi if [[ "${status}" -eq 0 ]]; then # The restored database must land back on the attested ownership # contract; a mismatch means the rollback is NOT byte-faithful. attest_update_database_ownership \ "${db_url}" restore "${UPDATE_RECOVERY_DATABASE_CONTRACT_VARIANT}" \ || status=1 fi return "${status}" } update_database_competing_session_settlement_sql() { cat <<'SQL' SET search_path TO pg_catalog; DO $bridgesllm$ DECLARE deadline timestamptz := pg_catalog.clock_timestamp() + interval '10 seconds'; BEGIN PERFORM pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() AND backend_type = 'client backend' AND pid <> pg_catalog.pg_backend_pid() AND pid <> __BRIDGESLLM_GUARD_BACKEND_PID__; LOOP PERFORM pg_catalog.pg_stat_clear_snapshot(); EXIT WHEN NOT EXISTS ( SELECT 1 FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() AND backend_type = 'client backend' AND pid <> pg_catalog.pg_backend_pid() AND pid <> __BRIDGESLLM_GUARD_BACKEND_PID__ ); IF pg_catalog.clock_timestamp() >= deadline THEN RAISE EXCEPTION 'database backends did not terminate before schema reset'; END IF; PERFORM pg_catalog.pg_sleep(0.05); END LOOP; END $bridgesllm$; SQL } restore_database_with_pgpass() { local dump="$1" contract_variant="$2" db_url="$3" local connection_url schema_restore_sql session_settlement_sql connection_url="$(libpq_database_url "${db_url}")" || return 1 # A forward-migrated schema can hold objects the snapshot does not know # about, so pg_restore --clean cannot order its drops past them. Reset the # schema wholesale, then restore the snapshot into the empty schema (the # custom-format dump carries no CREATE SCHEMA public entry of its own). case "${contract_variant}" in owner-null) schema_restore_sql='DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;' ;; pg-database-owner-default) schema_restore_sql='DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public; ALTER SCHEMA public OWNER TO pg_database_owner; REVOKE ALL ON SCHEMA public FROM PUBLIC; GRANT USAGE ON SCHEMA public TO PUBLIC; GRANT CREATE, USAGE ON SCHEMA public TO pg_database_owner;' ;; *) return 1 ;; esac session_settlement_sql="$( update_database_competing_session_settlement_sql )" || return 1 # One live, PID-bound guard spans session termination, schema reset, and # pg_restore; there is no # unguarded empty-schema gap. Only that exact guard backend is exempted from # termination. assert_update_database_topology_unchanged "${db_url}" recovery-schema-reset \ || return 1 run_attested_database_operation \ "${db_url}" recovery-database-restore \ bash -c ' set -Eeuo pipefail connection_url="$1" dump="$2" schema_restore_sql="$3" termination_sql="$4" guard_backend_pid="${BRIDGESLLM_DATABASE_GUARD_BACKEND_PID:-}" [[ "${guard_backend_pid}" =~ ^[1-9][0-9]*$ ]] || exit 91 termination_sql="${termination_sql//__BRIDGESLLM_GUARD_BACKEND_PID__/${guard_backend_pid}}" # One exact psql session delivers termination, waits until every # non-guard backend has disappeared, and only then resets the schema. # Signal delivery alone is not settlement: a PostgreSQL backend can # continue abort/commit work after pg_terminate_backend() returns true. psql --dbname="${connection_url}" --no-psqlrc -v ON_ERROR_STOP=1 \ -c "${termination_sql}" \ -c "${schema_restore_sql}" pg_restore --no-owner --no-privileges --exit-on-error \ --dbname="${connection_url}" "${dump}" ' bridgesllm-database-restore \ "${connection_url}" "${dump}" "${schema_restore_sql}" \ "${session_settlement_sql}" \ >> "${LOG_FILE}" 2>&1 } run_migrations_safe() { local db_url="$1" local postflight_db_url="" local guarded_update=false local backend_dir="${PORTAL_DIR}/backend" local migration_dir="${backend_dir}/prisma/migrations" # Validate the connection string with a real URI parser before handing it to # Prisma/libpq. Reinstalls may retain encoded credentials, non-default # ports, and connection options; shell/sed reconstruction corrupts all three. pg_url_component "${db_url}" host >/dev/null \ && pg_url_component "${db_url}" port >/dev/null \ && pg_url_component "${db_url}" database >/dev/null \ && pg_url_component "${db_url}" user >/dev/null \ || fail "Configured DATABASE_URL is invalid or incomplete." postflight_db_url="$(libpq_database_url "${db_url}")" \ || fail "Configured DATABASE_URL cannot be used for database postflight verification." if $UPDATE_MODE; then current_update_receipt_target >/dev/null 2>&1 \ || fail "Update migration requires exactly one valid durable update receipt; refusing an unguarded database migration." guarded_update=true fi # ── Pre-flight: ensure migration files exist ── local migration_count=0 if [[ -d "${migration_dir}" ]]; then migration_count=$(find "${migration_dir}" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l) fi if (( migration_count == 0 )); then echo "" echo -e " ${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e " ${RED}${BOLD} DATABASE SETUP ERROR${NC}" echo "" echo -e " ${WHITE} No migration files found in:${NC}" echo -e " ${DIM} ${migration_dir}${NC}" echo "" echo -e " ${WHITE} This usually means the release tarball is incomplete.${NC}" echo -e " ${WHITE} The portal cannot start without database tables.${NC}" echo "" echo -e " ${CYAN} How to fix:${NC}" echo -e " ${WHITE} 1. Re-download the installer and run again:${NC}" echo -e " ${DIM} curl -fsSL https://bridgesllm.ai/install.sh | sudo bash${NC}" echo "" echo -e " ${WHITE} 2. Or manually re-download the portal tarball:${NC}" echo -e " ${DIM} curl -fsSL https://bridgesllm.ai/releases/${VERSION}/portal.tar.gz -o /tmp/portal.tar.gz${NC}" echo -e " ${DIM} tar xzf /tmp/portal.tar.gz -C /tmp${NC}" echo -e " ${DIM} cp -r /tmp/portal/backend/prisma/migrations ${migration_dir}${NC}" echo -e " ${DIM} cd ${backend_dir} && npx prisma migrate deploy${NC}" echo -e " ${DIM} systemctl restart bridgesllm-product${NC}" echo "" echo -e " ${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" fail "No database migrations found — release package may be corrupt" fi info "Found ${migration_count} migration(s)" # Any partially applied forward migration must restore with the previous # runtime. The updater keeps this armed until exact-version readiness passes. if $UPDATE_MODE && [[ -s "${UPDATE_RECOVERY_DATABASE_DUMP:-}" ]]; then UPDATE_RECOVERY_DATABASE_RESTORE_REQUIRED=true fi if $DRY_RUN; then info "[dry-run] Would migrate, verify, and generate the configured database" return 0 fi # ── Run migrations ── # Keep credentials out of argv/log output. The subshell receives the URL in # its environment and invokes Prisma directly, so URI query parameters and # percent-encoded credentials are preserved byte-for-byte. info "Running database migrations..." if ${guarded_update}; then assert_update_database_topology_unchanged "${db_url}" prisma-migrate \ || fail "The local PostgreSQL system or storage topology changed before migration; rollback remains armed." if ! run_attested_database_operation \ "${db_url}" prisma-migrate \ bash -c 'cd "$1"; exec npx prisma migrate deploy' \ bridgesllm-prisma-migrate "${backend_dir}" \ >> "${LOG_FILE}" 2>&1; then fail "Database migrations failed against the configured database — check ${LOG_FILE}." fi assert_update_database_topology_unchanged \ "${db_url}" prisma-migrate-complete \ || fail "The local PostgreSQL system or storage topology changed while migrations were running; rollback remains armed." elif ! run_contained_database_operation \ "${db_url}" prisma-migrate \ bash -c 'cd "$1"; exec npx prisma migrate deploy' \ bridgesllm-prisma-migrate "${backend_dir}" \ >> "${LOG_FILE}" 2>&1; then fail "Database migrations failed against the configured database — check ${LOG_FILE}." fi ok "Database migrations applied" # ── Post-flight: verify tables were actually created ── # libpq only expands connection URIs passed as the dbname parameter, never # via PGDATABASE, so the URI goes to --dbname explicitly. The password is # stripped from that URI (no secret in argv) and supplied through an # anonymous memfd; TLS/connectivity query options survive in the URI itself. local table_count=0 table_count=$(run_with_update_pgpass "${db_url}" \ psql --dbname="${postflight_db_url}" --no-psqlrc -qAtc \ "SET search_path TO pg_catalog; SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE'" \ 2>> "${LOG_FILE}" | tr -d '[:space:]' || echo "0") table_count="${table_count:-0}" # _prisma_migrations table always exists; we need at least a few more if (( table_count < 3 )); then echo "" echo -e " ${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e " ${RED}${BOLD} DATABASE MIGRATION FAILED${NC}" echo "" echo -e " ${WHITE} Migrations ran but only ${table_count} table(s) were created.${NC}" echo -e " ${WHITE} Expected 10+. The database is incomplete.${NC}" echo "" echo -e " ${CYAN} How to fix:${NC}" echo -e " ${WHITE} 1. Check the install log for errors:${NC}" echo -e " ${DIM} tail -50 ${LOG_FILE}${NC}" echo "" echo -e " ${WHITE} 2. Retry migrations using DATABASE_URL from:${NC}" echo -e " ${DIM} cd ${backend_dir}${NC}" echo -e " ${DIM} ${backend_dir}/.env.production${NC}" echo "" echo -e " ${WHITE} 3. If that fails, verify the configured database is reachable.${NC}" echo -e " ${DIM} Then restart: systemctl restart bridgesllm-product${NC}" echo "" echo -e " ${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" fail "Database migration incomplete — only ${table_count} tables created (expected 10+)" fi if ! ( cd "${backend_dir}" run_without_database_authority npx prisma generate ) >> "${LOG_FILE}" 2>&1; then fail "Database client generation failed — check ${LOG_FILE}." fi if ! verify_prisma_client_runtime "${backend_dir}" migration-prisma-check \ >> "${LOG_FILE}" 2>&1; then fail "Rust-free database runtime verification failed after migrations — check ${LOG_FILE}." fi ok "Database ready ${DIM}(${table_count} tables, ${migration_count} migrations)${NC}" } verify_portal_service_health() { local service_name="${1:-bridgesllm-product}" local health_url="${2:-http://127.0.0.1:4001/health}" local timeout_secs="${3:-60}" local waited=0 tick=0 while (( waited < timeout_secs )); do if systemctl is-active --quiet "${service_name}" && curl -fsS --max-time 2 "${health_url}" >> "$LOG_FILE" 2>&1; then if [[ -t 1 ]]; then printf "\r%-120s\r" ""; fi ok "Portal is healthy" return 0 fi if [[ -t 1 ]]; then draw_pulse_bar "$tick" "Waiting for portal" "$(format_elapsed $waited)" fi tick=$(( tick + 3 )) sleep 2 waited=$((waited + 2)) done if [[ -t 1 ]]; then printf "\r%-120s\r" ""; fi journalctl -u "${service_name}" -n 50 --no-pager >> "$LOG_FILE" 2>&1 || true return 1 } portal_service_restart_count() { local service_name="${1:-bridgesllm-product}" systemctl show "${service_name}" -p NRestarts --value 2>/dev/null | tr -cd '0-9' } verify_portal_update_readiness() { # Startup migrations (for example legacy-project-retirement) can hold the # service in a bootstrap "starting" state for many minutes on a populated # install. The budget must cover that, while a crashed or # crash-looping service must still fail fast instead of burning the budget. local expected_version="$1" probe_token="$2" timeout_secs="${3:-900}" local service_name="${4:-bridgesllm-product}" local readiness_url="${5:-http://127.0.0.1:4001/health/update-ready}" local expected_validation_mode="${6:-canonical}" local response_file baseline_restarts current_restarts # Budget in real elapsed seconds: probe latency inside each iteration must # consume the budget, or a 900s budget silently stretches toward an hour. local start_seconds="${SECONDS}" [[ -n "${expected_version}" && -n "${probe_token}" ]] || return 1 baseline_restarts="$(portal_service_restart_count "${service_name}")" response_file="$(mktemp /tmp/bridgesllm-readiness.XXXXXX)" chmod 0600 "${response_file}" while (( SECONDS - start_seconds < timeout_secs )); do if ! systemctl is-active --quiet "${service_name}"; then rm -f "${response_file}" warn "Portal service is no longer active while waiting for readiness" return 1 fi current_restarts="$(portal_service_restart_count "${service_name}")" if [[ -n "${baseline_restarts}" && -n "${current_restarts}" ]] \ && (( current_restarts > baseline_restarts )); then rm -f "${response_file}" warn "Portal service restarted while waiting for readiness (crash loop)" return 1 fi if systemctl is-active --quiet "${service_name}" \ && curl -fsS --max-time 3 -H "x-portal-update-probe: ${probe_token}" \ -o "${response_file}" "${readiness_url}" 2>> "${LOG_FILE}" \ && python3 - "${response_file}" "${expected_version}" "${expected_validation_mode}" <<'PY2' import json import sys try: payload = json.load(open(sys.argv[1], encoding="utf-8")) except Exception: raise SystemExit(1) # Require the readiness contract fields; tolerate additive diagnostic fields # (4.0 attaches mailboxReconciliation) so honest extra detail cannot fail a # ready deploy. if not isinstance(payload, dict): raise SystemExit(1) expected_version = sys.argv[2] mode = sys.argv[3] validation_value = payload.get("updateValidation") if mode == "candidate" and validation_value is not True: raise SystemExit(1) if mode == "canonical" and validation_value is not False: raise SystemExit(1) if mode == "legacy-canonical" and ( "updateValidation" in payload or "updateValidationContract" in payload ): raise SystemExit(1) if mode in {"candidate", "canonical"} and ( payload.get("updateValidationContract") != "BRIDGESLLM_UPDATE_VALIDATION_CONTRACT_V1" ): raise SystemExit(1) if mode not in {"candidate", "canonical", "legacy-canonical"}: raise SystemExit(1) if ( payload.get("status") != "ready" or payload.get("version") != expected_version or payload.get("database") != "ready" ): raise SystemExit(1) PY2 then rm -f "${response_file}" ok "Portal ${expected_version} authenticated readiness verified" return 0 fi sleep 2 done rm -f "${response_file}" return 1 } UPDATE_RECOVERY_BACKUP_DIR="" UPDATE_RECOVERY_ROLLBACK_RUNTIME=false openclaw_standard_state_static_is_confirmed() { local allow_dynamic_unit="${1:-false}" [[ -z "${OPENCLAW_STATE_DIR:-}" \ && -z "${OPENCLAW_HOME:-}" \ && -z "${OPENCLAW_PROFILE:-}" \ && -z "${OPENCLAW_CONFIG_PATH:-}" \ && -z "${CLAWDBOT_STATE_DIR:-}" \ && -z "${CLAWDBOT_HOME:-}" \ && -z "${CLAWDBOT_PROFILE:-}" \ && -z "${CLAWDBOT_CONFIG_PATH:-}" ]] || return 1 [[ "${HOME:-/root}" == "/root" && -d "/root/.openclaw" && ! -L "/root/.openclaw" ]] || return 1 local unit_user unit_env unit_text exec_start exec_path unit_user="$(systemctl show openclaw-gateway -p User --value 2>/dev/null || true)" [[ -z "${unit_user}" || "${unit_user}" == "root" ]] || return 1 unit_env="$(systemctl show openclaw-gateway -p Environment --value 2>/dev/null || true)" if printf '%s\n' "${unit_env}" | grep -Eq '(^|[[:space:]])(OPENCLAW|CLAWDBOT)_(STATE_DIR|HOME|PROFILE|CONFIG_PATH)='; then return 1 fi if printf '%s\n' "${unit_env}" | grep -Eq '(^|[[:space:]])HOME=' \ && ! printf '%s\n' "${unit_env}" | grep -Eq '(^|[[:space:]])HOME=/root($|[[:space:]])'; then return 1 fi unit_text="$(systemctl cat openclaw-gateway --no-pager 2>/dev/null || true)" # EnvironmentFile contents and named profiles can redirect state invisibly. # Custom units are supported, but their OpenClaw package must be upgraded by # an operator who knows that layout rather than by this default-path repair. if ! $allow_dynamic_unit \ && printf '%s\n' "${unit_text}" | grep -Eq '^[[:space:]]*EnvironmentFile='; then return 1 fi if printf '%s\n' "${unit_text}" | grep -Eq -- '^[[:space:]]*ExecStart=.*(^|[[:space:]])--(profile|dev|config|state-dir|home)($|=|[[:space:]])'; then return 1 fi exec_start="$(systemctl show openclaw-gateway -p ExecStart --value 2>/dev/null || true)" if printf '%s\n' "${exec_start}" | grep -Eq -- '(^|[[:space:]])--(profile|dev|config|state-dir|home)($|=|[[:space:]])'; then return 1 fi exec_path="$(printf '%s\n' "${exec_start}" | sed -n 's/.*path=\([^ ;]*\).*/\1/p' | head -1)" if [[ -n "${exec_path}" && -f "${exec_path}" ]]; then if grep -Eq -- '(OPENCLAW|CLAWDBOT)_(STATE_DIR|HOME|PROFILE|CONFIG_PATH)=|(^|[[:space:]])--(profile|dev|config|state-dir|home)($|=|[[:space:]])' "${exec_path}"; then return 1 fi local wrapper_home_assignments wrapper_home_assignments="$(grep -E '^[[:space:]]*(export[[:space:]]+)?HOME=' "${exec_path}" || true)" if [[ -n "${wrapper_home_assignments}" ]] \ && printf '%s\n' "${wrapper_home_assignments}" \ | grep -Evq "^[[:space:]]*(export[[:space:]]+)?HOME=(/root|\"/root\"|'/root')[[:space:]]*(#.*)?$"; then return 1 fi # A healthy process can prove the result of sourced files via /proc. An # inactive gateway cannot, so rescue mode refuses dynamic wrapper state. if ! $allow_dynamic_unit \ && grep -Eq '^[[:space:]]*(source|\.)[[:space:]]+' "${exec_path}"; then return 1 fi fi return 0 } openclaw_standard_state_is_confirmed() { openclaw_standard_state_static_is_confirmed true || return 1 systemctl is-active --quiet openclaw-gateway || return 1 openclaw_gateway_http_ready || return 1 local main_pid gateway_home main_pid="$(systemctl show openclaw-gateway -p MainPID --value 2>/dev/null || true)" [[ "${main_pid}" =~ ^[1-9][0-9]*$ && -r "/proc/${main_pid}/environ" && -r "/proc/${main_pid}/cmdline" ]] || return 1 # /proc is the authority here. It includes EnvironmentFile and wrapper-script # exports that cannot be inferred safely from the unit text while inactive. gateway_home="$(tr '\0' '\n' < "/proc/${main_pid}/environ" | sed -n 's/^HOME=//p' | head -1)" [[ "${gateway_home}" == "/root" ]] || return 1 if tr '\0' '\n' < "/proc/${main_pid}/environ" | grep -Eq '^(OPENCLAW|CLAWDBOT)_(STATE_DIR|HOME|PROFILE|CONFIG_PATH)='; then return 1 fi if tr '\0' ' ' < "/proc/${main_pid}/cmdline" | grep -Eq -- '(^|[[:space:]])--(profile|dev|config|state-dir|home)($|=|[[:space:]])'; then return 1 fi return 0 } prepare_openclaw_upgrade_state() { if $SKIP_OPENCLAW || ! command -v node >/dev/null 2>&1; then return 0 fi local needs_preparation=false if $OPENCLAW_PACKAGE_UPDATED && $OPENCLAW_STATE_EXISTED_BEFORE_UPDATE; then needs_preparation=true elif ! openclaw_gateway_http_ready \ && { [[ -e "/root/.clawdbot" ]] || [[ -f "/root/.openclaw/plugins/installs.json" ]]; }; then needs_preparation=true fi if ! $needs_preparation; then ok "OpenClaw legacy-state preparation not needed" return 0 fi local layout_confirmed=false if $OPENCLAW_PACKAGE_UPDATED && $OPENCLAW_STATE_EXISTED_BEFORE_UPDATE; then if $OPENCLAW_GATEWAY_WAS_ACTIVE; then openclaw_standard_state_is_confirmed && layout_confirmed=true else # A fresh Portal install may inherit a CLI/state directory but no gateway # unit. By this point configure_services has created the standard unit; # prove that static layout before the first new-runtime boot. openclaw_standard_state_static_is_confirmed && layout_confirmed=true fi else # Rescue an already-failed 2026.7.1 first start using static unit proof. A # healthy package upgrade requires the stronger live /proc baseline above. openclaw_standard_state_static_is_confirmed && layout_confirmed=true fi if ! $layout_confirmed; then warn "OpenClaw uses a custom or unverified state layout; refusing to alter default-path migration artifacts automatically." if ! openclaw_gateway_http_ready; then systemctl stop openclaw-gateway >> "$LOG_FILE" 2>&1 || true warn "Stopped the unready gateway to prevent a continuing restart loop; custom-state recovery requires an operator." fi return 1 fi if ! $OPENCLAW_PACKAGE_UPDATED && ! openclaw_gateway_http_ready; then # A failed first 2026.7 start may still be cycling under Restart=always. # Stop it before touching warning sources so no new migration owner can # race the repair or extend the upstream startup-migration lease. systemctl stop openclaw-gateway >> "$LOG_FILE" 2>&1 || true local stop_waited=0 while systemctl is-active --quiet openclaw-gateway && (( stop_waited < 30 )); do sleep 2 stop_waited=$((stop_waited + 2)) done if systemctl is-active --quiet openclaw-gateway; then warn "Could not stop the unready OpenClaw gateway before migration recovery." return 1 fi OPENCLAW_RESCUE_MODE=true fi local helper="${PORTAL_DIR}/backend/dist/services/openclawConfigManager.js" if [[ ! -f "${helper}" ]]; then warn "OpenClaw upgrade-state helper is missing." return 1 fi local manifest_dir="${UPDATE_RECOVERY_BACKUP_DIR:-/tmp}" mkdir -p "${manifest_dir}" OPENCLAW_UPGRADE_STATE_MANIFEST="${manifest_dir}/openclaw-upgrade-state-${TIMESTAMP}.json" if ! PORTAL_OPENCLAW_STANDARD_STATE_CONFIRMED=1 \ NODE_PATH="${PORTAL_DIR}/backend/node_modules" \ node -e ' const fs = require("fs"); const helper = require(process.argv[1]); const output = process.argv[2]; const result = helper.prepareOpenClawUpgradeState(); try { fs.writeFileSync(output, JSON.stringify(result, null, 2) + "\n", { mode: 0o600 }); fs.chmodSync(output, 0o600); } catch (error) { const restored = helper.restoreOpenClawUpgradeState(result); console.error(JSON.stringify({ manifestWriteFailed: String(error), restored })); process.exit(44); } console.log(JSON.stringify(result)); if (!result.readyForGatewayStart) process.exit(42); ' "${helper}" "${OPENCLAW_UPGRADE_STATE_MANIFEST}" >> "$LOG_FILE" 2>&1; then warn "OpenClaw legacy state could not be reconciled safely; the old runtime will be restored." return 1 fi ok "OpenClaw legacy state checked and recoverably preserved" if $OPENCLAW_RESCUE_MODE && ! wait_for_openclaw_startup_migration_lease; then warn "OpenClaw's orphaned startup-migration lease did not expire safely within the recovery window." return 1 fi } restore_prepared_openclaw_state() { [[ -n "${OPENCLAW_UPGRADE_STATE_MANIFEST:-}" && -f "${OPENCLAW_UPGRADE_STATE_MANIFEST}" ]] || return 0 local helper="${PORTAL_DIR}/backend/dist/services/openclawConfigManager.js" [[ -f "${helper}" ]] || return 1 NODE_PATH="${PORTAL_DIR}/backend/node_modules" node -e ' const fs = require("fs"); const helper = require(process.argv[1]); const preparation = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); const result = helper.restoreOpenClawUpgradeState(preparation); console.log(JSON.stringify(result)); if (!result.restored) process.exit(43); ' "${helper}" "${OPENCLAW_UPGRADE_STATE_MANIFEST}" >> "$LOG_FILE" 2>&1 } openclaw_gateway_http_ready() { curl -fsS --max-time 5 http://127.0.0.1:18789/readyz >/dev/null 2>&1 } wait_for_openclaw_gateway_http_ready() { local timeout_secs="${1:-60}" local waited=0 while (( waited < timeout_secs )); do openclaw_gateway_http_ready && return 0 sleep 3 waited=$((waited + 3)) done return 1 } # The Portal restarts the OpenClaw gateway from its own boot path (visible # browser agent reconciliation, Remote Desktop recovery), so in the seconds # after the Portal runtime is replaced the gateway can legitimately be # stopping, starting, or up but not yet answering /readyz. Sampling that # instant as if it were steady state reported a perfectly healthy host as # "not stably ready" and aborted the update. Let the unit settle first; the # real baseline gates still judge whatever we settle on. settle_openclaw_gateway_before_converge() { local timeout_secs="${1:-180}" local waited=0 state="" systemctl list-unit-files openclaw-gateway.service >/dev/null 2>&1 || return 0 systemctl is-enabled openclaw-gateway >/dev/null 2>&1 \ || systemctl is-active --quiet openclaw-gateway \ || return 0 while (( waited < timeout_secs )); do state="$(systemctl show openclaw-gateway -p ActiveState --value 2>/dev/null || true)" if [[ "${state}" == "active" ]] && openclaw_gateway_http_ready; then return 0 fi # A unit that has given up is a real fault, not a transient restart, and # waiting the full window would only delay an accurate diagnosis. [[ "${state}" == "failed" ]] && return 1 sleep 3 waited=$((waited + 3)) done return 1 } openclaw_startup_migration_lease_remaining_seconds() { NODE_NO_WARNINGS=1 node - <<'NODE' 2>> "$LOG_FILE" const { DatabaseSync } = require('node:sqlite'); let db; try { db = new DatabaseSync('/root/.openclaw/state/openclaw.sqlite', { readOnly: true }); const table = db.prepare("select 1 from sqlite_master where type='table' and name='state_leases'").get(); if (!table) { console.log('0'); process.exit(0); } const row = db.prepare("select expires_at from state_leases where scope='startup-migrations' and lease_key='global'").get(); const expiresAt = Number(row?.expires_at || 0); console.log(String(Math.max(0, Math.ceil((expiresAt - Date.now()) / 1000)))); } catch (error) { console.error(`Could not inspect the OpenClaw startup-migration lease: ${error instanceof Error ? error.message : String(error)}`); process.exit(1); } finally { try { db?.close(); } catch {} } NODE } wait_for_openclaw_startup_migration_lease() { local remaining waited=0 max_wait remaining="$(openclaw_startup_migration_lease_remaining_seconds)" || return 1 [[ "${remaining}" =~ ^[0-9]+$ ]] || return 1 (( remaining == 0 )) && return 0 if (( remaining > 315 )); then warn "OpenClaw reported an unexpectedly long startup-migration lease (${remaining}s); refusing to delete or bypass it." return 1 fi max_wait=$((remaining + 15)) info "Waiting up to ${max_wait}s for OpenClaw's orphaned startup-migration lease to expire (the lease is not deleted or bypassed)" while (( waited < max_wait )); do remaining="$(openclaw_startup_migration_lease_remaining_seconds)" || return 1 [[ "${remaining}" =~ ^[0-9]+$ ]] || return 1 (( remaining == 0 )) && return 0 sleep 3 waited=$((waited + 3)) if (( waited % 30 == 0 )); then progress "OpenClaw migration lease: ${remaining}s remaining" fi done return 1 } openclaw_gateway_pid() { systemctl show openclaw-gateway -p MainPID --value 2>/dev/null || true } openclaw_gateway_restart_count() { systemctl show openclaw-gateway -p NRestarts --value 2>/dev/null || true } capture_openclaw_gateway_baseline() { local expected_version="$1" systemctl is-active --quiet openclaw-gateway || return 1 openclaw_gateway_http_ready || return 1 OPENCLAW_ALLOW_ROOT=1 openclaw gateway status --require-rpc --timeout 10000 >> "$LOG_FILE" 2>&1 || return 1 [[ "$(openclaw_gateway_version)" == "${expected_version}" ]] || return 1 OPENCLAW_BASELINE_PID="$(openclaw_gateway_pid)" OPENCLAW_BASELINE_RESTARTS="$(openclaw_gateway_restart_count)" [[ "${OPENCLAW_BASELINE_PID}" =~ ^[1-9][0-9]*$ && "${OPENCLAW_BASELINE_RESTARTS}" =~ ^[0-9]+$ ]] || return 1 return 0 } openclaw_startup_checkpoint_matches() { local expected_version="$1" OPENCLAW_EXPECTED_CHECKPOINT_VERSION="${expected_version}" node - <<'NODE' >> "$LOG_FILE" 2>&1 const { DatabaseSync } = require('node:sqlite'); const expected = process.env.OPENCLAW_EXPECTED_CHECKPOINT_VERSION; const normalizeStableRuntime = (value) => String(value || '').replace(/-\d+$/, ''); let db; try { db = new DatabaseSync('/root/.openclaw/state/openclaw.sqlite', { readOnly: true }); const row = db.prepare("select app_version from schema_meta where meta_key='startup-migrations'").get(); if (normalizeStableRuntime(row?.app_version) !== normalizeStableRuntime(expected)) process.exit(1); } catch (error) { console.error(`OpenClaw startup checkpoint check failed: ${error instanceof Error ? error.message : String(error)}`); process.exit(1); } finally { try { db?.close(); } catch {} } NODE } openclaw_gateway_log_is_clean_since_start() { local started_at started_at="$(systemctl show openclaw-gateway -p ExecMainStartTimestamp --value 2>/dev/null || true)" [[ -n "${started_at}" && "${started_at}" != "n/a" ]] || return 1 ! journalctl -u openclaw-gateway --since "${started_at}" --no-pager 2>/dev/null \ | grep -Eqi 'refusing to report the gateway ready|startup migrations did not complete cleanly|state lease.*(contention|held|failed)|Main process exited|Failed with result' } verify_openclaw_gateway_stable() { local expected_version="$1" local stability_seconds="${2:-18}" local cli_version gateway_version initial_pid initial_restarts waited=0 wait_for_openclaw_gateway_http_ready 60 || return 1 cli_version="$(openclaw_cli_version)" gateway_version="$(openclaw_gateway_version)" [[ -n "${expected_version}" && "${cli_version}" == "${expected_version}" && "${gateway_version}" == "${expected_version}" ]] || return 1 OPENCLAW_ALLOW_ROOT=1 openclaw gateway status --require-rpc --timeout 10000 >> "$LOG_FILE" 2>&1 || return 1 initial_pid="$(openclaw_gateway_pid)" initial_restarts="$(openclaw_gateway_restart_count)" [[ "${initial_pid}" =~ ^[1-9][0-9]*$ && "${initial_restarts}" =~ ^[0-9]+$ ]] || return 1 while (( waited < stability_seconds )); do sleep 3 waited=$((waited + 3)) systemctl is-active --quiet openclaw-gateway || return 1 openclaw_gateway_http_ready || return 1 [[ "$(openclaw_gateway_pid)" == "${initial_pid}" ]] || return 1 [[ "$(openclaw_gateway_restart_count)" == "${initial_restarts}" ]] || return 1 done OPENCLAW_ALLOW_ROOT=1 openclaw gateway status --require-rpc --timeout 10000 >> "$LOG_FILE" 2>&1 || return 1 openclaw_gateway_log_is_clean_since_start || return 1 if [[ "${expected_version}" == "${PIN_OPENCLAW_RUNTIME_VERSION}" ]]; then openclaw_startup_checkpoint_matches "${expected_version}" || return 1 fi return 0 } node_package_name_from_dir() { local package_dir="$1" [[ -f "${package_dir}/package.json" ]] || return 1 node -e ' const fs = require("fs"); try { const data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(String(data.name || "")); } catch (_) { process.exit(1); } ' "${package_dir}/package.json" 2>/dev/null } node_package_version_from_dir() { local package_dir="$1" [[ -f "${package_dir}/package.json" ]] || return 1 node -e ' const fs = require("fs"); try { const data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(String(data.version || "")); } catch (_) { process.exit(1); } ' "${package_dir}/package.json" 2>/dev/null } openclaw_core_package_dir() { local global_root candidate binary resolved package_name depth global_root="$(npm root -g 2>/dev/null || true)" candidate="${global_root}/openclaw" if [[ -n "${global_root}" && -d "${candidate}" ]] \ && [[ "$(node_package_name_from_dir "${candidate}" || true)" == "openclaw" ]]; then printf '%s\n' "${candidate}" return 0 fi binary="$(command -v openclaw 2>/dev/null || true)" [[ -n "${binary}" ]] || return 1 resolved="$(readlink -f "${binary}" 2>/dev/null || true)" [[ -n "${resolved}" ]] || return 1 candidate="$(dirname "${resolved}")" for depth in 1 2 3 4 5 6; do package_name="$(node_package_name_from_dir "${candidate}" || true)" if [[ "${package_name}" == "openclaw" ]]; then printf '%s\n' "${candidate}" return 0 fi [[ "${candidate}" != "/" ]] || break candidate="$(dirname "${candidate}")" done return 1 } openclaw_core_package_version() { local package_dir package_dir="$(openclaw_core_package_dir || true)" [[ -n "${package_dir}" ]] || return 1 node_package_version_from_dir "${package_dir}" } verify_openclaw_core_package_pin() { [[ "$(openclaw_core_package_version || true)" == "${PIN_OPENCLAW_CORE_PACKAGE_VERSION}" ]] \ && [[ "$(openclaw_cli_version || true)" == "${PIN_OPENCLAW_RUNTIME_VERSION}" ]] } stage_openclaw_rollback_package() { local expected_version="$1" local package_dir backup_dir packed_name packed_path packed_version package_dir="$(openclaw_core_package_dir || true)" backup_dir="${UPDATE_RECOVERY_BACKUP_DIR:-/tmp}/openclaw-package" [[ -d "${package_dir}" && -f "${package_dir}/package.json" ]] || return 1 mkdir -p "${backup_dir}" packed_name="$(cd "${backup_dir}" && npm pack --ignore-scripts --silent "${package_dir}" 2>> "$LOG_FILE" | tail -1)" packed_path="${backup_dir}/${packed_name}" [[ -n "${packed_name}" && -f "${packed_path}" ]] || return 1 packed_version="$(tar -xOf "${packed_path}" package/package.json 2>/dev/null \ | node -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { console.log(JSON.parse(s).version || ""); } catch {} });' \ | head -1)" if [[ "${packed_version}" != "${expected_version}" ]]; then rm -f "${packed_path}" return 1 fi chmod 600 "${packed_path}" 2>/dev/null || true OPENCLAW_ROLLBACK_PACKAGE_TARBALL="${packed_path}" return 0 } openclaw_installation_state_present() { # The Portal's media pipeline creates /root/.openclaw/media on boot even # when OpenClaw itself was never installed, so a media cache alone is not # installation state that needs repair. Anything else under the state dir # is treated as a real OpenClaw installation footprint. local state_dir="${1:-/root/.openclaw}" entry [[ -d "${state_dir}" ]] || return 1 for entry in "${state_dir}"/* "${state_dir}"/.[!.]* "${state_dir}"/..?*; do [[ -e "${entry}" || -L "${entry}" ]] || continue [[ "$(basename -- "${entry}")" == "media" ]] && continue return 0 done return 1 } converge_openclaw_core_package() { $SKIP_OPENCLAW && return 0 command -v npm >/dev/null 2>&1 || fail "npm is required to install the tested OpenClaw runtime." local current_package_version="" current_runtime_version="" OPENCLAW_STATE_EXISTED_BEFORE_UPDATE=false openclaw_installation_state_present /root/.openclaw \ && OPENCLAW_STATE_EXISTED_BEFORE_UPDATE=true OPENCLAW_GATEWAY_WAS_ENABLED=false OPENCLAW_GATEWAY_WAS_ACTIVE=false settle_openclaw_gateway_before_converge 180 || true systemctl is-enabled openclaw-gateway >/dev/null 2>&1 && OPENCLAW_GATEWAY_WAS_ENABLED=true systemctl is-active --quiet openclaw-gateway && OPENCLAW_GATEWAY_WAS_ACTIVE=true if command -v openclaw >/dev/null 2>&1; then OPENCLAW_PACKAGE_PREEXISTED=true current_package_version="$(openclaw_core_package_version || true)" current_runtime_version="$(openclaw_cli_version || true)" if [[ "${current_package_version}" == "${PIN_OPENCLAW_CORE_PACKAGE_VERSION}" \ && "${current_runtime_version}" == "${PIN_OPENCLAW_RUNTIME_VERSION}" ]]; then ok "OpenClaw ${current_package_version} (tested core revision)" return 0 fi if [[ -z "${current_package_version}" || -z "${current_runtime_version}" ]]; then fail "The existing OpenClaw installation could not be identified precisely; refusing to overwrite it without a verifiable rollback package." fi if $OPENCLAW_GATEWAY_WAS_ACTIVE; then local baseline_attempt=0 baseline_confirmed=false # The Portal can restart the gateway again between settling and probing, # so a single miss is retried rather than treated as an unsafe host. while (( baseline_attempt < 3 )); do baseline_attempt=$((baseline_attempt + 1)) if capture_openclaw_gateway_baseline "${current_runtime_version}" \ && openclaw_standard_state_is_confirmed; then baseline_confirmed=true break fi (( baseline_attempt < 3 )) || break settle_openclaw_gateway_before_converge 60 || true done if ! $baseline_confirmed; then fail "The existing OpenClaw gateway is not stably ready in the standard state layout; refusing to replace ${current_package_version} without a safe live rollback baseline." fi elif $OPENCLAW_GATEWAY_WAS_ENABLED; then fail "The existing OpenClaw gateway is enabled but not running; refusing to replace ${current_package_version} without a safe live rollback baseline." fi if ! stage_openclaw_rollback_package "${current_package_version}"; then fail "Could not preserve an exact local rollback package for OpenClaw ${current_package_version}." fi if ! capture_openclaw_codex_plugin_baseline; then fail "Could not prove and preserve the existing OpenClaw Codex plugin baseline before replacing the core package." fi OPENCLAW_PREUPDATE_PACKAGE_VERSION="${current_package_version}" OPENCLAW_PREUPDATE_RUNTIME_VERSION="${current_runtime_version}" else OPENCLAW_PACKAGE_PREEXISTED=false if $OPENCLAW_STATE_EXISTED_BEFORE_UPDATE; then fail "OpenClaw state exists but its runtime is missing; repair that installation explicitly before Portal convergence." fi # This is a genuinely fresh host: capture the pre-install plugin state as # absent before installing a core package that bundles the Codex plugin. # Inspecting only after core installation would misclassify that newly # bundled copy as the rollback baseline. OPENCLAW_CODEX_PLUGIN_PREEXISTED=false OPENCLAW_CODEX_PLUGIN_PREUPDATE_VERSION="" OPENCLAW_CODEX_PLUGIN_BASELINE_CAPTURED=true fi OPENCLAW_PACKAGE_UPDATE_ATTEMPTED=true info "Converging OpenClaw core to ${PIN_OPENCLAW_CORE_PACKAGE_VERSION}..." if ! npm install -g "openclaw@${PIN_OPENCLAW_CORE_PACKAGE_VERSION}" >> "$LOG_FILE" 2>&1; then fail "OpenClaw core package installation failed or ended in a partial state; automatic rollback is starting." fi OPENCLAW_PACKAGE_UPDATED=true if ! verify_openclaw_core_package_pin; then fail "OpenClaw core did not converge to package ${PIN_OPENCLAW_CORE_PACKAGE_VERSION} / runtime ${PIN_OPENCLAW_RUNTIME_VERSION}; automatic rollback is starting." fi ok "OpenClaw ${PIN_OPENCLAW_CORE_PACKAGE_VERSION} package staged (gateway restart deferred)" } rollback_openclaw_package_update() { $OPENCLAW_ROLLBACK_IN_PROGRESS && return 1 OPENCLAW_ROLLBACK_IN_PROGRESS=true local defer_gateway_restart="${1:-false}" local rollback_ok=true local restart_gateway=false if $OPENCLAW_PACKAGE_UPDATE_ATTEMPTED || $OPENCLAW_PACKAGE_UPDATED || [[ -n "${OPENCLAW_UPGRADE_STATE_MANIFEST:-}" ]]; then $OPENCLAW_GATEWAY_WAS_ACTIVE && restart_gateway=true systemctl stop openclaw-gateway >> "$LOG_FILE" 2>&1 || true fi if $OPENCLAW_PACKAGE_UPDATE_ATTEMPTED || $OPENCLAW_PACKAGE_UPDATED; then if $OPENCLAW_PACKAGE_PREEXISTED; then warn "Rolling OpenClaw back to package ${OPENCLAW_PREUPDATE_PACKAGE_VERSION:-unknown}" local rollback_source="openclaw@${OPENCLAW_PREUPDATE_PACKAGE_VERSION}" if [[ -n "${OPENCLAW_ROLLBACK_PACKAGE_TARBALL:-}" && -f "${OPENCLAW_ROLLBACK_PACKAGE_TARBALL}" ]]; then rollback_source="${OPENCLAW_ROLLBACK_PACKAGE_TARBALL}" fi if [[ -z "${OPENCLAW_PREUPDATE_PACKAGE_VERSION:-}" ]] \ || ! npm install -g "${rollback_source}" >> "$LOG_FILE" 2>&1 \ || [[ "$(openclaw_core_package_version || true)" != "${OPENCLAW_PREUPDATE_PACKAGE_VERSION}" ]] \ || [[ "$(openclaw_cli_version || true)" != "${OPENCLAW_PREUPDATE_RUNTIME_VERSION}" ]]; then warn "Could not restore the exact OpenClaw package/runtime pair during automatic rollback." rollback_ok=false fi else warn "Removing the newly installed OpenClaw package" if ! npm uninstall -g openclaw >> "$LOG_FILE" 2>&1; then warn "Could not remove the newly installed OpenClaw package during automatic rollback." rollback_ok=false fi fi fi if ! restore_prepared_openclaw_state; then warn "Could not restore one or more preserved OpenClaw legacy artifacts automatically." rollback_ok=false fi if $restart_gateway && [[ "${defer_gateway_restart}" != "true" ]]; then systemctl start openclaw-gateway >> "$LOG_FILE" 2>&1 || true if [[ -z "${OPENCLAW_PREUPDATE_RUNTIME_VERSION:-}" ]] \ || ! verify_openclaw_gateway_stable "${OPENCLAW_PREUPDATE_RUNTIME_VERSION}" 18; then local restored_version restored_version="$(openclaw_cli_version || true)" warn "OpenClaw rollback health verification failed (expected ${OPENCLAW_PREUPDATE_RUNTIME_VERSION:-unknown}, found ${restored_version:-unknown})." systemctl stop openclaw-gateway >> "$LOG_FILE" 2>&1 || true rollback_ok=false fi fi if $rollback_ok; then OPENCLAW_PACKAGE_UPDATED=false OPENCLAW_PACKAGE_UPDATE_ATTEMPTED=false ok "Previous OpenClaw runtime restored" fi OPENCLAW_ROLLBACK_IN_PROGRESS=false $rollback_ok } recover_interrupted_update() { echo "" warn "Update interrupted before commit; resuming the durable recovery transaction." if recover_pending_update_transaction; then ok "Previous Portal runtime restored and verified" return 0 fi warn "Automatic recovery did not complete. Portal remains boot-fenced and stopped; the root-only transaction journal and rollback artifacts were preserved." return 1 } ollama_client_version() { command -v ollama >/dev/null 2>&1 || return 1 # `ollama --version` can block while querying a stale daemon. Point it at a # closed loopback port and discard inherited endpoints/proxies so it reports # the embedded client version without consulting operator-controlled routing. /usr/bin/env -i \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ HOME="${HOME:-/root}" \ LANG=C \ LC_ALL=C \ OLLAMA_HOST=http://127.0.0.1:1 \ timeout 10s ollama --version 2>&1 \ | sed -nE 's/.*(client version is|ollama version is) ([0-9]+\.[0-9]+\.[0-9]+).*/\2/p' \ | head -1 || true } ollama_server_version() { curl --noproxy '*' -fsS --max-time 3 http://127.0.0.1:11434/api/version 2>/dev/null \ | python3 -c 'import json,sys; print(json.load(sys.stdin).get("version", ""))' 2>/dev/null || true } verify_ollama_client_server_parity() { local timeout_secs="${1:-60}" waited=0 client_version server_version while (( waited < timeout_secs )); do client_version="$(ollama_client_version || true)" server_version="$(ollama_server_version || true)" if systemctl is-active --quiet ollama \ && [[ -n "${client_version}" && "${client_version}" == "${server_version}" ]]; then ok "Ollama ${client_version} client/server parity verified" return 0 fi sleep 2 waited=$((waited + 2)) done warn "Ollama client/server mismatch after restart (client: ${client_version:-unknown}, server: ${server_version:-unavailable})." return 1 } install_or_update_ollama() { $SKIP_OLLAMA && return 0 if ! command -v zstd >/dev/null 2>&1; then spin "Installing zstd (required by Ollama installer)" "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zstd" fi local previous_version previous_version="$(ollama_client_version || true)" if ! spin "Installing/updating Ollama (${previous_version:-not installed})" \ "curl -fsSL '${OLLAMA_INSTALLER_URL}' | sh >/dev/null 2>&1"; then fail "The official Ollama installer failed." fi command -v ollama >/dev/null 2>&1 || fail "Ollama installer exited successfully but no ollama executable was found." systemctl daemon-reload >> "$LOG_FILE" 2>&1 || true systemctl enable ollama >> "$LOG_FILE" 2>&1 || true if ! systemctl restart ollama >> "$LOG_FILE" 2>&1; then fail "Ollama was installed but its service could not be restarted." fi if ! verify_ollama_client_server_parity 60; then fail "Ollama did not reach client/server version parity after its upgrade restart." fi } converge_grok_build() { local helper="${PORTAL_DIR}/installer/grok-build-runtime.sh" [[ -f "${helper}" ]] || { warn "Grok Build runtime helper is missing from the installed Portal artifact." return 1 } GROK_BIN_DIR=/usr/local/bin bash "${helper}" converge >> "${LOG_FILE}" 2>&1 || return 1 GROK_BIN_DIR=/usr/local/bin bash "${helper}" verify >> "${LOG_FILE}" 2>&1 } converge_antigravity() { local helper="${PORTAL_DIR}/installer/antigravity-runtime.sh" [[ -f "${helper}" ]] || { warn "Antigravity runtime helper is missing from the installed Portal artifact." return 1 } ANTIGRAVITY_BIN_DIR=/usr/local/bin bash "${helper}" converge >> "${LOG_FILE}" 2>&1 || return 1 ANTIGRAVITY_BIN_DIR=/usr/local/bin bash "${helper}" verify >> "${LOG_FILE}" 2>&1 } npm_global_package_version() { local package_name="$1" global_root package_json global_root="$(npm root -g 2>/dev/null)" || return 1 package_json="${global_root}/${package_name}/package.json" [[ -f "${package_json}" ]] || return 1 node - "${package_json}" <<'NODE' const fs = require('fs'); try { const parsed = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); if (typeof parsed.version !== 'string' || !parsed.version) process.exit(1); process.stdout.write(parsed.version); } catch { process.exit(1); } NODE } verify_pinned_npm_cli() { local package_name="$1" expected_version="$2" command_name="$3" version_arg="$4" local package_version cli_output package_version="$(npm_global_package_version "${package_name}" 2>/dev/null || true)" [[ "${package_version}" == "${expected_version}" ]] || return 1 command -v "${command_name}" >/dev/null 2>&1 || return 1 cli_output="$(timeout 15 "${command_name}" "${version_arg}" 2>/dev/null || true)" grep -Fq "${expected_version}" <<<"${cli_output}" } converge_pinned_npm_cli() { local label="$1" package_name="$2" expected_version="$3" command_name="$4" version_arg="$5" local baseline_version="" had_package=false if verify_pinned_npm_cli "${package_name}" "${expected_version}" "${command_name}" "${version_arg}"; then ok "${label} ${expected_version} (verified)" return 0 fi baseline_version="$(npm_global_package_version "${package_name}" 2>/dev/null || true)" [[ -z "${baseline_version}" ]] || had_package=true if ! ${had_package} && command -v "${command_name}" >/dev/null 2>&1; then warn "${label} is present but is not owned by the expected global npm package; refusing to overwrite it." return 1 fi info "Converging ${label} to Portal-tested version ${expected_version}..." if npm install -g --no-audit --no-fund "${package_name}@${expected_version}" >> "${LOG_FILE}" 2>&1 \ && verify_pinned_npm_cli "${package_name}" "${expected_version}" "${command_name}" "${version_arg}"; then ok "${label} ${expected_version} (verified)" return 0 fi warn "${label} ${expected_version} failed verification; restoring its previous global package state." if ${had_package}; then npm install -g --no-audit --no-fund "${package_name}@${baseline_version}" >> "${LOG_FILE}" 2>&1 || true else npm uninstall -g "${package_name}" >> "${LOG_FILE}" 2>&1 || true fi return 1 } update_dependencies() { info "Checking Portal runtime compatibility..." # This must be the first package-runtime action in the updater. OpenClaw's # npm package cannot even be inspected safely on unsupported Node lanes. ensure_supported_node_runtime if $MAINTAIN_TOOLS; then spin "Updating Node.js within the supported lane" "apt-get update -qq && apt-get install -y -qq --only-upgrade nodejs 2>/dev/null" || true ensure_supported_node_runtime fi # The Portal updater owns only the Portal and its tested OpenClaw runtime # pair. Ollama and coding CLIs are independent operator tools: changing them # during an ordinary Portal update makes rollback non-atomic and previously # caused unrelated provider drift. They move only on fresh install, an # explicit --maintain-tools run, or a deliberate Admin maintenance action. # OpenClaw core uses the same exact tested package revision on fresh installs # and updates. Its gateway restart remains deferred until state preparation. converge_openclaw_core_package if $MAINTAIN_TOOLS; then # ClawHub powers Skills marketplace search/install. if ! $SKIP_OPENCLAW; then converge_pinned_npm_cli "ClawHub" "clawhub" "${PIN_CLAWHUB_VERSION}" "clawhub" "--cli-version" \ || warn "ClawHub could not be converged; Skills marketplace actions will remain unavailable." fi # Ollama's installer updates the client before the daemon. Restart and prove # the API-reported server version matches before continuing. if $SKIP_OLLAMA; then info "Skipping Ollama (--skip-ollama)" else install_or_update_ollama fi converge_pinned_npm_cli "Codex CLI" "@openai/codex" "${PIN_CODEX_CLI_VERSION}" "codex" "--version" \ || warn "Codex CLI could not be converged to ${PIN_CODEX_CLI_VERSION}; its provider will remain unavailable." converge_pinned_npm_cli "Claude Code" "@anthropic-ai/claude-code" "${PIN_CLAUDE_CODE_VERSION}" "claude" "--version" \ || warn "Claude Code could not be converged to ${PIN_CLAUDE_CODE_VERSION}; its provider will remain unavailable." if ! converge_antigravity; then warn "Antigravity could not be converged to ${PIN_ANTIGRAVITY_VERSION}; its provider will remain unavailable." else ok "Antigravity ${PIN_ANTIGRAVITY_VERSION} (verified)" fi # Grok Build remains exact-pinned when the operator explicitly chooses tool # maintenance. Normal Portal updates only report drift and fail the provider # closed; they do not replace a working CLI behind the operator's back. if ! converge_grok_build; then warn "Grok Build could not be converged to ${PIN_GROK_BUILD_VERSION}; its Portal provider will remain unavailable." else ok "Grok Build ${PIN_GROK_BUILD_VERSION} (verified)" fi else info "Optional tool maintenance skipped (use --maintain-tools to update Ollama and coding CLIs)" fi # Caddy terminates HTTPS and may carry local/beta site config. Do not mutate it # from a Portal app update; the admin maintenance UI treats Caddy as protected # infrastructure and installer updates must follow that same boundary. if command -v caddy &>/dev/null; then local current_caddy current_caddy="$(caddy version 2>/dev/null | awk '{print $1}' || echo 'installed')" ok "Caddy ${current_caddy:-installed} (manual review)" fi # ── Tier 3: Notify only ── # PostgreSQL — never auto-update major. Minor via apt is safe but requires restart. # Docker — user-managed. Don't touch. ok "Portal runtime compatibility checked" } # ── Progress display helpers ────────────────────────────────── format_elapsed() { local secs="$1" if (( secs >= 60 )); then printf '%dm %ds' $((secs / 60)) $((secs % 60)) else printf '%ds' "$secs" fi } format_bytes() { # Pure bash integer arithmetic: minimal hosts do not ship `bc`, and a failed # substitution here must never feed printf a non-number under `set -e`. local bytes="${1:-0}" tenths [[ "$bytes" =~ ^[0-9]+$ ]] || bytes=0 if (( bytes >= 1073741824 )); then tenths=$(( (bytes * 10) / 1073741824 )) printf '%d.%d GB' $(( tenths / 10 )) $(( tenths % 10 )) elif (( bytes >= 1048576 )); then tenths=$(( (bytes * 10) / 1048576 )) printf '%d.%d MB' $(( tenths / 10 )) $(( tenths % 10 )) elif (( bytes >= 1024 )); then printf '%d KB' $(( bytes / 1024 )) else printf '%d B' "$bytes" fi } # Render a determinate progress bar (0–100%) draw_pct_bar() { local pct="$1" msg="$2" detail="${3:-}" local bar_width=24 local filled=$(( (pct * bar_width) / 100 )) (( filled > bar_width )) && filled=$bar_width local empty=$(( bar_width - filled )) local bar="" i for ((i = 0; i < filled; i++)); do bar+="█"; done for ((i = 0; i < empty; i++)); do bar+="░"; done if [[ -n "$detail" ]]; then printf "\r ${CYAN}[${bar}]${NC} ${DIM}%3d%%${NC} ${msg} ${DIM}${detail}${NC} " "$pct" else printf "\r ${CYAN}[${bar}]${NC} ${DIM}%3d%%${NC} ${msg} " "$pct" fi } # Render an indeterminate progress bar (pulsing glow) draw_pulse_bar() { local tick="$1" msg="$2" detail="${3:-}" local bar_width=24 local frames=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏') local frame="${frames[$(( tick % ${#frames[@]} ))]}" # Pulse: a bright segment that sweeps back and forth local cycle=$(( bar_width * 2 )) local pos=$(( tick % cycle )) if (( pos >= bar_width )); then pos=$(( cycle - pos )) fi local bar="" i for ((i = 0; i < bar_width; i++)); do local dist=$(( i - pos )) (( dist < 0 )) && dist=$(( -dist )) if (( dist == 0 )); then bar+="█" elif (( dist == 1 )); then bar+="▓" elif (( dist == 2 )); then bar+="▒" else bar+="░" fi done if [[ -n "$detail" ]]; then printf "\r ${CYAN}${frame}${NC} ${CYAN}[${bar}]${NC} ${msg} ${DIM}${detail}${NC} " else printf "\r ${CYAN}${frame}${NC} ${CYAN}[${bar}]${NC} ${msg} " fi } # ── Core spin: indeterminate progress with elapsed time ────── spin() { local msg="$1"; shift if $DRY_RUN; then echo " [dry-run] $*" >> "$LOG_FILE" 2>&1 return fi if ! $PACKAGE_MANAGER_REPAIR_ACTIVE && command_needs_package_manager "$*"; then wait_for_package_manager_ready "$msg" fi bash -c "$*" >> "$LOG_FILE" 2>&1 & local pid=$! local start_ts tick=0 start_ts=$(date +%s) if [[ -t 1 ]]; then while kill -0 "$pid" 2>/dev/null; do local now elapsed now=$(date +%s) elapsed=$(( now - start_ts )) draw_pulse_bar "$tick" "$msg" "$(format_elapsed $elapsed)" tick=$(( tick + 1 )) sleep 0.15 done else echo -e " ${CYAN}⠿${NC} ${msg}..." fi wait "$pid" local rc=$? if [[ -t 1 ]]; then printf "\r%-120s\r" "" fi if [[ $rc -eq 0 ]]; then local end_ts total end_ts=$(date +%s) total=$(( end_ts - start_ts )) if (( total > 2 )); then ok "${msg} ${DIM}($(format_elapsed $total))${NC}" else ok "${msg}" fi fi return $rc } # ── Download with real progress (curl + actual bytes) ──────── spin_download() { local msg="$1" url="$2" dest="$3" if $DRY_RUN; then echo " [dry-run] curl $url → $dest" >> "$LOG_FILE" 2>&1 return fi # Get total size via HEAD request for real percentage local total_bytes=0 total_bytes=$(curl -fsSLI "$url" 2>/dev/null | grep -i '^content-length:' | awk '{print $2}' | tr -d '\r' || echo 0) # Download in background, track file size for real progress curl -fSL "$url" -o "$dest" >> "$LOG_FILE" 2>&1 & local pid=$! local start_ts tick=0 start_ts=$(date +%s) if [[ -t 1 ]]; then while kill -0 "$pid" 2>/dev/null; do local now elapsed current_bytes=0 pct=0 speed_str="" now=$(date +%s) elapsed=$(( now - start_ts )) if [[ -f "$dest" ]]; then current_bytes=$(stat -c%s "$dest" 2>/dev/null || echo 0) fi if (( total_bytes > 0 && current_bytes > 0 )); then pct=$(( (current_bytes * 100) / total_bytes )) (( pct > 100 )) && pct=100 if (( elapsed > 0 )); then local speed=$(( current_bytes / elapsed )) speed_str="$(format_bytes $current_bytes)/$(format_bytes $total_bytes) $(format_bytes $speed)/s" else speed_str="$(format_bytes $current_bytes)/$(format_bytes $total_bytes)" fi draw_pct_bar "$pct" "$msg" "$speed_str" else # Indeterminate (no content-length) if (( current_bytes > 0 )); then draw_pulse_bar "$tick" "$msg" "$(format_bytes $current_bytes) $(format_elapsed $elapsed)" else draw_pulse_bar "$tick" "$msg" "$(format_elapsed $elapsed)" fi fi tick=$(( tick + 1 )) sleep 0.3 done else echo -e " ${CYAN}⠿${NC} ${msg}..." fi wait "$pid" local rc=$? if [[ -t 1 ]]; then printf "\r%-120s\r" "" fi if [[ $rc -eq 0 ]]; then local final_size=0 [[ -f "$dest" ]] && final_size=$(stat -c%s "$dest" 2>/dev/null || echo 0) local end_ts total end_ts=$(date +%s) total=$(( end_ts - start_ts )) if (( total > 2 )); then ok "${msg} ${DIM}($(format_bytes $final_size), $(format_elapsed $total))${NC}" else ok "${msg} ${DIM}($(format_bytes $final_size))${NC}" fi fi return $rc } # ── Apt install with package counting ──────────────────────── spin_apt() { local msg="$1"; shift # $@ = list of expected package names local expected_pkgs=("$@") local total=${#expected_pkgs[@]} if $DRY_RUN; then echo " [dry-run] apt install ${expected_pkgs[*]}" >> "$LOG_FILE" 2>&1 return fi if ! $PACKAGE_MANAGER_REPAIR_ACTIVE; then wait_for_package_manager_ready "$msg" fi DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${expected_pkgs[@]}" >> "$LOG_FILE" 2>&1 & local pid=$! local start_ts tick=0 start_ts=$(date +%s) if [[ -t 1 ]]; then while kill -0 "$pid" 2>/dev/null; do local now elapsed installed=0 current_pkg="" now=$(date +%s) elapsed=$(( now - start_ts )) # Count how many of our target packages are now installed for pkg in "${expected_pkgs[@]}"; do if dpkg -s "$pkg" &>/dev/null 2>&1; then installed=$(( installed + 1 )) fi done # Try to identify what's being worked on from the log tail current_pkg="$(tail -3 "$LOG_FILE" 2>/dev/null | grep -oP '(?:Setting up|Unpacking|Preparing to unpack) \K[^ ]+' | tail -1 || true)" current_pkg="${current_pkg%%:*}" # strip arch suffix if (( total > 0 && installed > 0 )); then local pct=$(( (installed * 100) / total )) local detail="${installed}/${total}" [[ -n "$current_pkg" ]] && detail="${detail} · ${current_pkg}" draw_pct_bar "$pct" "$msg" "$detail" else local detail="$(format_elapsed $elapsed)" [[ -n "$current_pkg" ]] && detail="${detail} · ${current_pkg}" draw_pulse_bar "$tick" "$msg" "$detail" fi tick=$(( tick + 1 )) sleep 0.5 done else echo -e " ${CYAN}⠿${NC} ${msg}..." fi wait "$pid" local rc=$? if [[ -t 1 ]]; then printf "\r%-120s\r" "" fi if [[ $rc -eq 0 ]]; then local end_ts total_t end_ts=$(date +%s) total_t=$(( end_ts - start_ts )) if (( total_t > 2 )); then ok "${msg} ${DIM}(${total} packages, $(format_elapsed $total_t))${NC}" else ok "${msg}" fi fi return $rc } rand_hex() { openssl rand -hex "${1:-32}"; } rand_pass() { openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c "${1:-24}"; } settle_openclaw_compatibility_hotfix_process() { local pid="${OPENCLAW_COMPAT_HOTFIX_PID:-}" state="" attempt [[ -n "${pid}" ]] || return 0 if [[ ! "${pid}" =~ ^[1-9][0-9]*$ ]]; then OPENCLAW_COMPAT_HOTFIX_PID="" return 1 fi if kill -0 "${pid}" 2>/dev/null; then kill -TERM -- "-${pid}" 2>/dev/null \ || kill -TERM "${pid}" 2>/dev/null \ || true for attempt in {1..50}; do if ! kill -0 "${pid}" 2>/dev/null; then break fi state="$(awk '{print $3}' "/proc/${pid}/stat" 2>/dev/null || true)" [[ "${state}" == "Z" ]] && break sleep 0.1 done if kill -0 "${pid}" 2>/dev/null \ && [[ "$(awk '{print $3}' "/proc/${pid}/stat" 2>/dev/null || true)" != "Z" ]]; then kill -KILL -- "-${pid}" 2>/dev/null \ || kill -KILL "${pid}" 2>/dev/null \ || true fi fi wait "${pid}" 2>/dev/null || true OPENCLAW_COMPAT_HOTFIX_PID="" ! kill -0 "${pid}" 2>/dev/null } run_openclaw_compatibility_hotfix() { local hotfix_script="$1" openclaw_dist="$2" local hotfix_log="${3:-${LOG_FILE}}" status=0 observed_pgid="" local pgid_attempt command -v setsid >/dev/null 2>&1 || return 1 setsid env \ PORTAL_OPENCLAW_HOTFIX_STRICT=1 \ PORTAL_REQUIRED_OPENCLAW_PACKAGE_VERSION="${PIN_OPENCLAW_CORE_PACKAGE_VERSION}" \ bash "${hotfix_script}" "${openclaw_dist}" \ >> "${hotfix_log}" 2>&1 & OPENCLAW_COMPAT_HOTFIX_PID=$! for pgid_attempt in {1..20}; do observed_pgid="$(ps -o pgid= -p "${OPENCLAW_COMPAT_HOTFIX_PID}" 2>/dev/null \ | tr -d '[:space:]')" [[ -z "${observed_pgid}" \ || "${observed_pgid}" == "${OPENCLAW_COMPAT_HOTFIX_PID}" ]] && break sleep 0.05 done if [[ -z "${observed_pgid}" ]]; then if wait "${OPENCLAW_COMPAT_HOTFIX_PID}"; then status=0 else status=$? fi OPENCLAW_COMPAT_HOTFIX_PID="" return "${status}" fi if [[ "${observed_pgid}" != "${OPENCLAW_COMPAT_HOTFIX_PID}" ]]; then settle_openclaw_compatibility_hotfix_process || true return 1 fi if wait "${OPENCLAW_COMPAT_HOTFIX_PID}"; then status=0 else status=$? fi OPENCLAW_COMPAT_HOTFIX_PID="" return "${status}" } recover_uncommitted_openclaw_after_signal() { settle_openclaw_compatibility_hotfix_process || return 1 if [[ "${UPDATE_RECOVERY_ARMED:-false}" != "true" \ && "${OPENCLAW_UPGRADE_COMMITTED:-false}" != "true" ]] \ && declare -F rollback_openclaw_tested_pair >/dev/null 2>&1; then rollback_openclaw_tested_pair || warn "The OpenClaw core/plugin pair needs manual recovery; see ${LOG_FILE}." fi } update_transaction_test_root() { local root="${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" [[ -n "${root}" ]] || return 1 [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" ]] || return 1 python3 - "${root}" <<'PY2' import os import sys root = sys.argv[1] if ( not os.path.isabs(root) or root != os.path.normpath(root) or root == os.path.sep or any(ord(char) < 32 or ord(char) == 127 for char in root) ): raise SystemExit(1) print(root) PY2 } update_transaction_state_path() { local production_path="$1" local test_root="" test_root="$(update_transaction_test_root 2>/dev/null || true)" if [[ -n "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" && -z "${test_root}" ]]; then return 1 fi if [[ -n "${test_root}" ]]; then printf '%s%s\n' "${test_root}" "${production_path}" else printf '%s\n' "${production_path}" fi } recover_update_after_signal() { local active_journal cutover_journal active_journal="$(update_transaction_state_path "${UPDATE_ACTIVE_JOURNAL}")" || return 1 cutover_journal="$(update_transaction_state_path "${UPDATE_CUTOVER_JOURNAL}")" || return 1 if [[ -e "${active_journal}" || -L "${active_journal}" \ || -e "${cutover_journal}" || -L "${cutover_journal}" ]]; then declare -F recover_pending_update_transaction >/dev/null 2>&1 \ && recover_pending_update_transaction return fi recover_uncommitted_openclaw_after_signal } handle_installer_signal() { local exit_code="$1" message="$2" local interrupted_phase="${DASHBOARD_UPDATE_PROGRESS_PHASE:-failure}" trap '' SIGINT TERM HUP trap - ERR set +e if [[ "${DASHBOARD_UPDATE_PORTAL_COMMITTED:-false}" == "true" ]]; then dashboard_update_progress updated_with_errors \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${interrupted_phase}" \ "Portal updated; follow-up work was interrupted" "${message}" elif [[ "${UPDATE_RECOVERY_ARMED:-false}" == "true" ]]; then dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Installer interrupted — checking recovery" "${message}" else dashboard_update_progress running \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" failure \ "Installer interrupted" "${message}" fi echo "" echo -e "\n ${YELLOW}⚠ ${message}${NC}" echo "" if ! settle_openclaw_compatibility_hotfix_process; then warn "The OpenClaw compatibility patch process could not be proven stopped. Recovery was not started." [[ "${UPDATE_RECOVERY_ARMED:-false}" != "true" ]] \ || dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${interrupted_phase}" \ "Automatic recovery needs attention" \ "A compatibility process could not be proven stopped. Do not start another update." exit "${exit_code}" fi if declare -F settle_active_update_database_operation >/dev/null 2>&1 \ && ! settle_active_update_database_operation; then warn "The active database operation could not be proven stopped. Recovery was not started; the Portal remains boot-fenced." dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${interrupted_phase}" \ "Automatic recovery needs attention" \ "The database operation could not be proven stopped. Do not start another update." exit "${exit_code}" fi if ! recover_update_after_signal; then warn "Update recovery did not complete. Portal remains boot-fenced and recovery artifacts were preserved." dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${interrupted_phase}" \ "Automatic recovery needs attention" \ "Recovery after interruption did not complete. Do not start another update." elif [[ "${DASHBOARD_UPDATE_PORTAL_COMMITTED:-false}" == "true" ]]; then # A cutover receipt recovers forward, not backward. Recovery may have # committed the target Portal after the signal arrived, so preserve that # exact outcome instead of letting the outer nonzero exit look like an # ordinary pre-commit failure. dashboard_update_progress updated_with_errors \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${interrupted_phase}" \ "Portal updated; follow-up work was interrupted" "${message}" fi exit "${exit_code}" } handle_sigint() { handle_installer_signal 130 "Installation cancelled by user" } handle_sigterm() { handle_installer_signal 143 "Installation terminated" } handle_sighup() { handle_installer_signal 129 "Installer session disconnected" } handle_err() { local exit_code=$? local line_no=${1:-unknown} fail "Unexpected error (exit ${exit_code}) at line ${line_no}" } handle_update_transaction_err() { local exit_code="${1:-1}" local failed_phase="${DASHBOARD_UPDATE_PROGRESS_PHASE:-failure}" [[ "${exit_code}" =~ ^[1-9][0-9]*$ ]] || exit_code=1 trap - ERR trap '' SIGINT TERM HUP set +e dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Unexpected installer error — recovering" \ "The durable transaction is restoring or completing a verified Portal state." if ! settle_openclaw_compatibility_hotfix_process; then warn "The OpenClaw compatibility patch process could not be proven stopped. Recovery was not started." dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${failed_phase}" \ "Automatic recovery needs attention" \ "A compatibility process could not be proven stopped. Do not start another update." exit "${exit_code}" fi if declare -F settle_active_update_database_operation >/dev/null 2>&1 \ && ! settle_active_update_database_operation; then warn "The active database operation could not be proven stopped. Recovery was not started; the Portal remains boot-fenced." dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${failed_phase}" \ "Automatic recovery needs attention" \ "The database operation could not be proven stopped. Do not start another update." exit "${exit_code}" fi if ! recover_pending_update_transaction; then warn "Update recovery did not complete. Portal remains boot-fenced and recovery artifacts were preserved." dashboard_update_progress recovery_required \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${failed_phase}" \ "Automatic recovery needs attention" \ "The transaction stopped without a verified terminal state. Do not start another update." elif [[ "${DASHBOARD_UPDATE_PORTAL_COMMITTED:-false}" == "true" ]]; then dashboard_update_progress updated_with_errors \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" "${failed_phase}" \ "Portal updated; follow-up work failed" \ "The cutover recovered forward, but the installer exited before completing host integration." fi exit "${exit_code}" } trap 'handle_err $LINENO' ERR trap handle_sigint SIGINT trap handle_sigterm TERM trap handle_sighup HUP ensure_keyring_from_url() { local url="$1" local keyring="$2" local tmp tmp="$(mktemp)" if $DRY_RUN; then echo " [dry-run] refresh keyring ${keyring} from ${url}" >> "$LOG_FILE" 2>&1 rm -f "$tmp" return fi curl -fsSL "${url}" | gpg --dearmor > "${tmp}" install -D -m 0644 "${tmp}" "${keyring}" rm -f "${tmp}" } parse_strict_systemd_environment_file() { local file="$1" local key="${2:-}" [[ -f "${file}" && ! -L "${file}" ]] || return 1 [[ -z "${key}" || "${key}" =~ ^[A-Z][A-Z0-9_]*$ ]] || return 1 python3 - "${file}" "${key}" <<'PY2' import re import sys from pathlib import Path file_path, key = sys.argv[1], sys.argv[2] try: payload = Path(file_path).read_bytes() text = payload.decode("utf-8") except (OSError, UnicodeError): raise SystemExit(1) if b"\x00" in payload or "\r" in text: raise SystemExit(1) # Accept one deliberately small, systemd-equivalent EnvironmentFile subset. # Backslashes are forbidden rather than reimplemented: in systemd they can # escape characters or continue a physical line, which was the source of the # installer/service split-authority bug. Assignments must be KEY=value with # optional whole-value single/double quoting and no shell-only `export`. assignment = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)=(.*)$") values = {} for line_number, raw in enumerate(text.split("\n"), start=1): if not raw or raw.lstrip().startswith(("#", ";")): continue match = assignment.fullmatch(raw) if match is None: print(f"unsupported environment syntax on line {line_number}", file=sys.stderr) raise SystemExit(1) name, value = match.groups() if name in values: print(f"duplicate assignment of {name}", file=sys.stderr) raise SystemExit(1) if "\\" in value: print(f"backslash syntax is not accepted on line {line_number}", file=sys.stderr) raise SystemExit(1) if any(ord(char) < 32 or ord(char) == 127 for char in value): print(f"control character in value on line {line_number}", file=sys.stderr) raise SystemExit(1) if value[:1] in {"'", '"'}: quote = value[0] if len(value) < 2 or value[-1] != quote or quote in value[1:-1]: print(f"unsupported quoted value on line {line_number}", file=sys.stderr) raise SystemExit(1) value = value[1:-1] elif any(char in value for char in "\"'") or value != value.strip(" \t"): print(f"unsupported unquoted value on line {line_number}", file=sys.stderr) raise SystemExit(1) values[name] = value if key and key in values: print(values[key]) PY2 } read_env_value() { local file="$1" local key="$2" parse_strict_systemd_environment_file "${file}" "${key}" } assert_env_file_no_duplicate_keys() { # This is intentionally the same parser used for every installer read. # Besides duplicates, it rejects systemd quoting/escape/continuation forms # outside the generated strict subset before any snapshot or mutation. parse_strict_systemd_environment_file "$1" } env_file_has_assignment() { local file="$1" local key="$2" [[ -f "${file}" && ! -L "${file}" ]] || return 2 [[ "${key}" =~ ^[A-Z][A-Z0-9_]*$ ]] || return 2 parse_strict_systemd_environment_file "${file}" || return 2 python3 - "${file}" "${key}" <<'PY2' import re import sys from pathlib import Path path = Path(sys.argv[1]) key = sys.argv[2] assignment = re.compile(rf"^{re.escape(key)}=") try: found = any(assignment.match(line) for line in path.read_text().splitlines()) except (OSError, UnicodeError): raise SystemExit(2) raise SystemExit(0 if found else 1) PY2 } assert_database_process_environment_safe() { local key for key in \ PRISMA_CLIENT_ENGINE_TYPE \ PRISMA_QUERY_ENGINE_BINARY \ PRISMA_QUERY_ENGINE_LIBRARY \ PRISMA_CLIENT_GET_TIME \ NODE_PG_FORCE_NATIVE \ NODE_TLS_REJECT_UNAUTHORIZED \ PGUSER \ PGDATABASE \ PGPORT \ PGHOST \ PGPASSWORD \ PGBINARY \ PGOPTIONS \ PGSSLMODE \ PGSSLNEGOTIATION \ PGCLIENT_ENCODING \ PGREPLICATION \ PGAPPNAME \ PGCONNECT_TIMEOUT; do [[ ! -v "${key}" ]] || return 1 done } assert_prisma_runtime_environment_safe() { local env_file="$1" key status assert_database_process_environment_safe || return 1 assert_env_file_no_duplicate_keys "${env_file}" || return 1 for key in \ PRISMA_CLIENT_ENGINE_TYPE \ PRISMA_QUERY_ENGINE_BINARY \ PRISMA_QUERY_ENGINE_LIBRARY \ PRISMA_CLIENT_GET_TIME \ NODE_PG_FORCE_NATIVE \ NODE_TLS_REJECT_UNAUTHORIZED \ PGUSER \ PGDATABASE \ PGPORT \ PGHOST \ PGPASSWORD \ PGBINARY \ PGOPTIONS \ PGSSLMODE \ PGSSLNEGOTIATION \ PGCLIENT_ENCODING \ PGREPLICATION \ PGAPPNAME \ PGCONNECT_TIMEOUT; do status=0 env_file_has_assignment "${env_file}" "${key}" || status=$? case "${status}" in 1) ;; *) return 1 ;; esac done } remove_env_assignment_atomic() { local file="$1" local key="$2" [[ -f "${file}" && ! -L "${file}" ]] || return 1 [[ "${key}" =~ ^[A-Z][A-Z0-9_]*$ ]] || return 1 parse_strict_systemd_environment_file "${file}" || return 1 python3 - "${file}" "${key}" <<'PY2' import os import re import stat import sys import tempfile from pathlib import Path path = Path(sys.argv[1]) key = sys.argv[2] current = path.lstat() if ( not stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) or current.st_nlink != 1 ): raise SystemExit(1) assignment = re.compile(rf"^{re.escape(key)}=") lines = path.read_text().splitlines() payload = ("\n".join(line for line in lines if not assignment.match(line)) + "\n").encode() fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent)) try: os.fchmod(fd, stat.S_IMODE(current.st_mode)) os.fchown(fd, current.st_uid, current.st_gid) with os.fdopen(fd, "wb", closefd=True) as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) temporary = "" directory_fd = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY2 } set_env_value_atomic() { local file="$1" local key="$2" local value="$3" [[ -f "${file}" && ! -L "${file}" ]] || return 1 [[ "${key}" =~ ^[A-Z][A-Z0-9_]*$ ]] || return 1 [[ "${value}" != *$'\n'* && "${value}" != *$'\r'* \ && "${value}" != *'\'* && "${value}" != *'"'* \ && "${value}" != *"'"* \ && "${value}" == "${value#"${value%%[![:space:]]*}"}" \ && "${value}" == "${value%"${value##*[![:space:]]}"}" ]] || return 1 parse_strict_systemd_environment_file "${file}" || return 1 # Send the value over stdin rather than argv so secrets never appear in the # process list. The replace+directory fsync keeps a killed installer from # leaving a partially written production environment. printf '%s' "${value}" | python3 /dev/fd/3 "${file}" "${key}" 3<<'PY2' import os import re import sys import tempfile from pathlib import Path path = Path(sys.argv[1]) key = sys.argv[2] value = sys.stdin.read() if path.is_symlink() or not path.is_file() or any(char in value for char in "\r\n\x00"): raise SystemExit(1) lines = path.read_text().splitlines() output = [] replaced = False assignment = re.compile(rf"^{re.escape(key)}=") for line in lines: if assignment.match(line): if not replaced: output.append(f"{key}={value}") replaced = True continue output.append(line) if not replaced: output.append(f"{key}={value}") payload = ("\n".join(output) + "\n").encode() fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent)) try: os.fchmod(fd, 0o600) with os.fdopen(fd, "wb", closefd=True) as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) directory_fd = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) except BaseException: try: os.close(fd) except OSError: pass try: os.unlink(temporary) except FileNotFoundError: pass raise PY2 } valid_project_egress_token_secret() { local value="${1:-}" [[ "${value}" =~ ^[A-Za-z0-9_-]{43,256}$ ]] } ensure_project_egress_token_secret() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local existing="" existing="$(read_env_value "${env_file}" "PROJECT_EGRESS_TOKEN_SECRET" 2>/dev/null || true)" if [[ -n "${existing}" ]]; then valid_project_egress_token_secret "${existing}" \ || fail "PROJECT_EGRESS_TOKEN_SECRET is malformed; refusing to rotate an active Project credential root automatically." PROJECT_EGRESS_TOKEN_SECRET="${existing}" else PROJECT_EGRESS_TOKEN_SECRET="$(rand_hex 32)" valid_project_egress_token_secret "${PROJECT_EGRESS_TOKEN_SECRET}" \ || fail "Could not generate the Project egress credential secret." fi set_env_value_atomic "${env_file}" "PROJECT_EGRESS_TOKEN_SECRET" "${PROJECT_EGRESS_TOKEN_SECRET}" \ || fail "Could not persist the Project egress credential secret safely." [[ "$(read_env_value "${env_file}" "PROJECT_EGRESS_TOKEN_SECRET" 2>/dev/null || true)" == "${PROJECT_EGRESS_TOKEN_SECRET}" ]] \ || fail "Project egress credential secret verification failed." ok "Project egress credential secret (verified)" } release_base_url_is_trusted() { python3 - "${RELEASE_BASE_URL}" <<'PY2' import sys from urllib.parse import urlparse parsed = urlparse(sys.argv[1]) if parsed.username or parsed.password or parsed.query or parsed.fragment: raise SystemExit(1) if parsed.scheme == "https" and parsed.hostname: raise SystemExit(0) if parsed.scheme == "http" and parsed.hostname in {"127.0.0.1", "::1", "localhost"}: raise SystemExit(0) raise SystemExit(1) PY2 } write_release_verification_key() { local destination="$1" cat > "${destination}" <<'KEY' -----BEGIN PUBLIC KEY----- MCowBQYDK2VwAyEAF0Mqi+e9entO6AacPZbQ4lBQ+hModVr2tqb/P3tkQD4= -----END PUBLIC KEY----- KEY chmod 0600 "${destination}" } verify_release_bundle() { local bundle_dir="$1" local artifact="${bundle_dir}/portal.tar.gz" local manifest="${bundle_dir}/portal-release.manifest" local signature="${bundle_dir}/portal-release.sig" local public_key="" local verify_log="/dev/null" [[ -s "${artifact}" && -s "${manifest}" && -s "${signature}" ]] || return 1 (( $(stat -c '%s' "${manifest}") <= 16384 )) || return 1 (( $(stat -c '%s' "${signature}") <= 4096 )) || return 1 [[ -d "$(dirname "${LOG_FILE}")" && -w "$(dirname "${LOG_FILE}")" ]] && verify_log="${LOG_FILE}" public_key="$(mktemp "${bundle_dir}/.release-key.XXXXXX")" || return 1 if ! write_release_verification_key "${public_key}" \ || [[ "$(openssl pkey -pubin -in "${public_key}" -outform DER 2>/dev/null | sha256sum | awk '{print $1}')" != "${RELEASE_PUBLIC_KEY_SHA256}" ]] \ || ! openssl pkeyutl -verify -pubin -inkey "${public_key}" -rawin \ -in "${manifest}" -sigfile "${signature}" >> "${verify_log}" 2>&1; then rm -f -- "${public_key}" return 1 fi if ! python3 - "${manifest}" "${artifact}" "${VERSION}" "${public_key}" <<'PY2' import base64 import datetime import hashlib import json import os import re import stat import subprocess import sys import tarfile import tempfile from pathlib import PurePosixPath manifest_path, artifact_path, expected_version, public_key_path = sys.argv[1:5] raw = open(manifest_path, "rb").read() if b"\x00" in raw or b"\r" in raw: raise SystemExit("invalid manifest encoding") values = {} for line in raw.decode("utf-8").splitlines(): if not line or "=" not in line: raise SystemExit("invalid manifest line") key, value = line.split("=", 1) if key in values: raise SystemExit("duplicate manifest key") values[key] = value base_required = {"schema", "version", "artifact", "sha256", "size"} metadata_required = {"released", "release_class", "highlights"} if values.get("schema") == "1": if set(values) != base_required: raise SystemExit("unsupported manifest schema") elif values.get("schema") == "2": if set(values) != base_required | metadata_required: raise SystemExit("unsupported manifest schema") else: raise SystemExit("unsupported manifest schema") if values["version"] != expected_version or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", values["version"]): raise SystemExit("release version mismatch") if values["artifact"] != "portal.tar.gz": raise SystemExit("unexpected artifact name") if not re.fullmatch(r"[0-9a-f]{64}", values["sha256"]): raise SystemExit("invalid artifact digest") size = os.path.getsize(artifact_path) if size <= 0 or size > 536_870_912 or str(size) != values["size"]: raise SystemExit("artifact size mismatch") hasher = hashlib.sha256() with open(artifact_path, "rb") as artifact: for chunk in iter(lambda: artifact.read(1024 * 1024), b""): hasher.update(chunk) digest = hasher.hexdigest() if digest != values["sha256"]: raise SystemExit("artifact digest mismatch") manifest_highlights = None if values["schema"] == "2": if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", values["released"]): raise SystemExit("invalid release date") try: if datetime.date.fromisoformat(values["released"]).isoformat() != values["released"]: raise ValueError except ValueError: raise SystemExit("invalid release date") if values["release_class"] not in {"hotfix", "security", "feature", "maintenance"}: raise SystemExit("invalid release class") encoded_highlights = values["highlights"] if not encoded_highlights or len(encoded_highlights) > 8192 or not re.fullmatch(r"[A-Za-z0-9_-]+", encoded_highlights): raise SystemExit("invalid release highlights") try: padding = "=" * (-len(encoded_highlights) % 4) highlight_bytes = base64.urlsafe_b64decode(encoded_highlights + padding) if base64.urlsafe_b64encode(highlight_bytes).rstrip(b"=").decode("ascii") != encoded_highlights: raise ValueError manifest_highlights = json.loads(highlight_bytes.decode("utf-8")) except (ValueError, UnicodeDecodeError, json.JSONDecodeError): raise SystemExit("invalid release highlights") if not isinstance(manifest_highlights, list) or not (1 <= len(manifest_highlights) <= 5): raise SystemExit("invalid release highlights") for highlight in manifest_highlights: if (not isinstance(highlight, str) or highlight != highlight.strip() or not (1 <= len(highlight) <= 200) or any(ord(character) < 32 or ord(character) == 127 for character in highlight)): raise SystemExit("invalid release highlight") if len(set(manifest_highlights)) != len(manifest_highlights): raise SystemExit("duplicate release highlight") required_current_migrations = { "portal/backend/prisma/migrations/0000_initial/migration.sql", "portal/backend/prisma/migrations/20260319_add_mail_signature_forward/migration.sql", "portal/backend/prisma/migrations/20260408_registration_request_password_hash/migration.sql", "portal/backend/prisma/migrations/20260718_auth_identity_transactions/migration.sql", "portal/backend/prisma/migrations/20260718_mailbox_primary_invariant/migration.sql", "portal/backend/prisma/migrations/20260718_project_chat_provider_bindings/migration.sql", "portal/backend/prisma/migrations/20260719_mailbox_reconciliation_outbox/migration.sql", "portal/backend/prisma/migrations/20260719_project_chat_message_request_identity/migration.sql", "portal/backend/prisma/migrations/20260719_project_identity_base/migration.sql", "portal/backend/prisma/migrations/20260719_project_identity_lifecycle/migration.sql", "portal/backend/prisma/migrations/20260719_project_turn_leases/migration.sql", "portal/backend/prisma/migrations/20260720_project_chat_message_presentation/migration.sql", "portal/backend/prisma/migrations/20260721_project_identity_rename_lifecycle/migration.sql", "portal/backend/prisma/migrations/20260721_legacy_openclaw_project_import/migration.sql", "portal/backend/prisma/migrations/20260721_project_chat_destructive_reset_journal/migration.sql", "portal/backend/prisma/migrations/20260722_project_identity_current_openclaw_status/migration.sql", "portal/backend/prisma/migrations/20260722_project_identity_owner_restrict/migration.sql", "portal/backend/prisma/migrations/20260723_ollama_backend_binding/migration.sql", "portal/backend/prisma/migrations/20260723_user_authorization_version/migration.sql", "portal/backend/prisma/migrations/20260726_native_ollama_backend_binding/migration.sql", "portal/backend/prisma/migrations/20260729_project_authorization_transition/migration.sql", "portal/backend/prisma/migrations/20260808_share_link_rate_limits/migration.sql", "portal/backend/prisma/migrations/20260809_project_runtime_recovery_replay/migration.sql", "portal/backend/prisma/migrations/20260812_project_dependency_promotion_decision/migration.sql", "portal/backend/prisma/migrations/20260812_project_dependency_repair_force_forward/migration.sql", } required_current_skill = { "portal/skills/bridgesllm-portal/SKILL.md", "portal/skills/bridgesllm-portal/references/admin.md", "portal/skills/bridgesllm-portal/references/agent-chat.md", "portal/skills/bridgesllm-portal/references/automations.md", "portal/skills/bridgesllm-portal/references/email.md", "portal/skills/bridgesllm-portal/references/files-and-projects.md", "portal/skills/bridgesllm-portal/references/remote-desktop.md", "portal/skills/bridgesllm-portal/references/remote-gpu.md", "portal/skills/bridgesllm-portal/scripts/bridges-rd-openclaw-ui.sh", "portal/skills/bridgesllm-portal/scripts/bridges-rd-shared-chrome.sh", "portal/skills/bridgesllm-portal/scripts/cdp-client.mjs", "portal/skills/bridgesllm-portal/scripts/shared-browser.sh", } required_baseline_members = { "portal/RELEASE-CONTENTS.sha256", "portal/RELEASE-LINEAGE", "portal/CHANGELOG.md", "portal/README.md", "portal/RELEASE-METADATA.json", "portal/backend/dist/cli/projectRuntimeUninstallPreflight.js", "portal/backend/dist/agents/providers/agentZero/AgentZeroProjectCleanup.js", "portal/backend/dist/agents/providers/agentZero/AgentZeroProjectEgress.js", "portal/backend/dist/agents/providers/agentZero/AgentZeroProjectModelBridge.js", "portal/backend/dist/agents/providers/agentZero/AgentZeroProjectModelBridgeCredential.js", "portal/backend/dist/agents/providers/agentZero/AgentZeroProjectImage.js", "portal/backend/dist/agents/providers/agentZero/AgentZeroProjectProvider.js", "portal/backend/dist/agents/providers/agentZero/AgentZeroProjectSandbox.js", "portal/backend/dist/agents/providers/native/adapters/gemini.js", "portal/backend/dist/agents/providers/native/grok/GrokAcpBroker.js", "portal/backend/dist/agents/providers/native/projectSandbox/CodexProjectEgressRuntime.js", "portal/backend/dist/agents/providers/native/projectSandbox/CodexProjectSandbox.js", "portal/backend/dist/agents/providers/native/projectSandbox/AntigravityProjectBridge.js", "portal/backend/dist/agents/providers/native/projectSandbox/AntigravityProjectSandbox.js", "portal/backend/dist/agents/providers/native/projectSandbox/ClaudeCodeProjectSandbox.js", "portal/backend/dist/agents/providers/native/projectSandbox/NativeCliProjectEgressRuntime.js", "portal/backend/dist/agents/providers/native/projectSandbox/NativeCliProjectManagedState.js", "portal/backend/dist/routes/projects.js", "portal/backend/dist/server.js", "portal/backend/dist/services/app-process.service.js", "portal/backend/dist/services/accessTokenAuthorization.js", "portal/backend/dist/services/openclawProjectSandbox.js", "portal/backend/dist/services/openclawProjectQualification.js", "portal/backend/dist/services/portalTransportAuthorization.js", "portal/backend/dist/services/projectChatTurnLease.js", "portal/backend/dist/services/projectDeletionLock.js", "portal/backend/dist/services/projectDependencyInstall.js", "portal/backend/dist/services/projectDependencyPromotionWriterFence.js", "portal/backend/dist/services/projectDependencyPromotionDecision.js", "portal/backend/dist/services/projectDependencyPromotionManifest.js", "portal/backend/dist/services/projectDependencyPromotionStartupRecovery.js", "portal/backend/dist/services/projectDependencyRepair.js", "portal/backend/dist/services/projectChatDependencyPromotionQuiescence.js", "portal/backend/dist/services/projectEgressCleanupAdapter.js", "portal/backend/dist/services/projectEgressCredentials.js", "portal/backend/dist/services/projectEgressPlane.js", "portal/backend/dist/services/projectEgressPolicy.js", "portal/backend/dist/services/projectEgressProxy.js", "portal/backend/dist/services/projectIdentity.js", "portal/backend/dist/services/project-git.service.js", "portal/backend/dist/services/project-lifecycle.service.js", "portal/backend/dist/services/projectNativeRunBroker.js", "portal/backend/dist/services/projectRuntimeAuthorizationPolicy.js", "portal/backend/dist/services/projectRuntimeCleanup.js", "portal/backend/dist/services/projectRuntimeCleanupAdapters.js", "portal/backend/dist/services/projectRuntimeConfinement.js", "portal/backend/dist/services/projectStoragePaths.js", "portal/backend/dist/services/projectWorkloadRuntime.js", "portal/backend/dist/services/releaseUpdateDetails.js", "portal/backend/dist/services/sessionRevocationBus.js", "portal/backend/dist/services/startupStatusServer.js", "portal/backend/dist/version.js", "portal/backend/package-lock.json", "portal/backend/package.json", "portal/backend/prisma/migrations/migration_lock.toml", "portal/backend/prisma/schema.prisma", "portal/frontend/dist/index.html", "portal/frontend/package.json", "portal/installer/agent-zero-project-model-bridge.sh", "portal/installer/agent-zero-project-sandbox.Dockerfile", "portal/installer/agent-zero-runtime.sh", "portal/installer/antigravity-runtime.sh", "portal/installer/bridgesllm-codex-project-runtime-v1.apparmor", "portal/installer/bridgesllm-codex-project-runtime-v1.seccomp.json", "portal/installer/bridgesllm-project-runtime-v1.apparmor", "portal/installer/bridgesllm-project-runtime-v1.seccomp.json", "portal/installer/caddy-managed-config.py", "portal/installer/project-runtime-image-repair-launcher.py", "portal/installer/update-transaction-state.py", "portal/installer/update-validation-protocol-v1", "portal/installer/grok-build-runtime.sh", "portal/installer/install.sh", "portal/installer/install.sh.sig", "portal/installer/Setup-OllamaTailnet.ps1", "portal/installer/Start-Here.cmd", "portal/installer/ollama-tailnet-README.txt", "portal/installer/release-required-members.txt", "portal/installer/release-signing-ed25519.pub.pem", "portal/installer/scripts/bridges-rd-xtigervnc-start.sh", "portal/installer/scripts/bridges-rd-session-guard.sh", "portal/installer/scripts/bridges-rd-healthcheck.sh", "portal/installer/scripts/bridges-rd-window-fit.sh", "portal/static/icons/bridges-ai-agent-zero.svg", "portal/static/icons/bridges-ai-antigravity.svg", "portal/static/icons/bridges-ai-claude-code.svg", "portal/static/icons/bridges-ai-codex.svg", "portal/static/icons/bridges-ai-grok-build.svg", "portal/static/icons/bridges-ai-ollama.svg", "portal/static/scripts/bridges-rd-ai-launchers.sh", "portal/static/scripts/bridges-rd-openclaw-ui.sh", "portal/static/scripts/bridges-rd-shared-chrome.sh", } | required_current_migrations | required_current_skill seen = set() member_index = {} expanded = 0 with tarfile.open(artifact_path, "r:gz") as archive: members = archive.getmembers() if not members or len(members) > 100_000: raise SystemExit("invalid archive member count") for member in members: name = member.name.rstrip("/") path = PurePosixPath(name) if not name or path.is_absolute() or ".." in path.parts or path.parts[0] != "portal": raise SystemExit("archive path escapes portal root") if name in seen: raise SystemExit("duplicate archive member") seen.add(name) member_index[name] = member if member.issym() or member.islnk() or member.isdev() or member.isfifo(): raise SystemExit("archive contains links or special files") relative_parts = tuple(part.lower() for part in path.parts[1:]) if any(part in { ".data", ".git", ".ssh", "app-zips", "apps", "coverage", "node_modules", "projects", "reports", "src", "uploads", } for part in relative_parts): raise SystemExit("archive contains private or non-runtime paths") lowered = name.lower() if lowered.endswith(( ".bak", ".db", ".d.ts", ".log", ".map", ".orig", ".rej", ".sqlite", ".sqlite3", ".tar.gz", ".tgz", )): raise SystemExit("archive contains private or generated residue") basename = path.name.lower() if basename == ".env" or basename.startswith(".env."): raise SystemExit("archive contains environment state") if lowered.endswith((".pem", ".key")) and name != "portal/installer/release-signing-ed25519.pub.pem": raise SystemExit("archive contains key material") if member.isfile() and len(path.parts) > 1 and path.parts[1] == "scripts" \ and name not in { "portal/scripts/patch-openclaw-codex-pending-input-hotfix.sh", "portal/scripts/patch-openclaw-long-run-relay-hotfix.sh", }: raise SystemExit("archive contains non-runtime scripts") if member.size < 0 or member.size > 536_870_912: raise SystemExit("invalid archive member size") expanded += member.size if expanded > 1_073_741_824: raise SystemExit("archive expands beyond safety limit") if not required_baseline_members.issubset(seen): raise SystemExit("release is missing required runtime artifacts") def read_member(name, maximum=4 * 1024 * 1024): info = member_index.get(name) if info is None or not info.isfile() or info.size > maximum: raise SystemExit(f"invalid required member: {name}") handle = archive.extractfile(info) if handle is None: raise SystemExit(f"unreadable required member: {name}") value = handle.read(maximum + 1) if len(value) != info.size: raise SystemExit(f"truncated required member: {name}") return value inventory_raw = read_member("portal/installer/release-required-members.txt", 1024 * 1024) if b"\x00" in inventory_raw or b"\r" in inventory_raw: raise SystemExit("invalid release inventory encoding") inventory = inventory_raw.decode("utf-8").splitlines() if not inventory or len(inventory) > 4096 or len(inventory) != len(set(inventory)): raise SystemExit("invalid release inventory") for required_name in inventory: required_path = PurePosixPath(required_name) if (not required_name or required_name != required_name.strip() or required_path.is_absolute() or ".." in required_path.parts or not required_path.parts or required_path.parts[0] != "portal"): raise SystemExit("unsafe release inventory path") inventory_set = set(inventory) if not required_baseline_members.issubset(inventory_set) or not inventory_set.issubset(seen): raise SystemExit("release inventory does not cover required archive members") archive_migrations = { name for name in seen if name.startswith("portal/backend/prisma/migrations/") and name.endswith("/migration.sql") } inventory_migrations = { name for name in inventory_set if name.startswith("portal/backend/prisma/migrations/") and name.endswith("/migration.sql") } if (archive_migrations != required_current_migrations or inventory_migrations != required_current_migrations): raise SystemExit("release migration set does not match its required inventory") archive_skill = { name for name, info in member_index.items() if info.isfile() and name.startswith("portal/skills/bridgesllm-portal/") } inventory_skill = { name for name in inventory_set if name.startswith("portal/skills/bridgesllm-portal/") } if archive_skill != required_current_skill or inventory_skill != required_current_skill: raise SystemExit("bundled Portal skill does not match its required inventory") contents_raw = read_member("portal/RELEASE-CONTENTS.sha256", 16 * 1024 * 1024) if b"\x00" in contents_raw or b"\r" in contents_raw: raise SystemExit("invalid release content manifest encoding") content_hashes = {} for line in contents_raw.decode("utf-8").splitlines(): match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) if not match: raise SystemExit("invalid release content manifest line") relative = match.group(2) content_path = PurePosixPath(relative) if (content_path.is_absolute() or ".." in content_path.parts or not relative or relative in content_hashes): raise SystemExit("unsafe or duplicate release content path") content_hashes[relative] = match.group(1) regular_members = { name for name, info in member_index.items() if info.isfile() and name != "portal/RELEASE-CONTENTS.sha256" } manifested_members = {f"portal/{relative}" for relative in content_hashes} if regular_members != manifested_members: raise SystemExit("release content manifest does not exactly cover the archive") for relative, expected_hash in content_hashes.items(): name = f"portal/{relative}" info = member_index[name] handle = archive.extractfile(info) if handle is None: raise SystemExit("release content could not be hashed") hasher = hashlib.sha256() for chunk in iter(lambda: handle.read(1024 * 1024), b""): hasher.update(chunk) if hasher.hexdigest() != expected_hash: raise SystemExit(f"release content digest mismatch: {name}") launcher_name = "portal/installer/project-runtime-image-repair-launcher.py" installer_name = "portal/installer/install.sh" signature_name = "portal/installer/install.sh.sig" public_key_name = "portal/installer/release-signing-ed25519.pub.pem" for name, required_mode in ( (launcher_name, 0o644), (installer_name, 0o755), (signature_name, 0o644), (public_key_name, 0o644), ): info = member_index[name] if info.uid != 0 or info.gid != 0 or stat.S_IMODE(info.mode) != required_mode: raise SystemExit(f"invalid release authority metadata: {name}") launcher_payload = read_member(launcher_name, 128 * 1024) bundled_public_key = read_member(public_key_name, 16 * 1024) with open(public_key_path, "rb") as public_key_handle: trusted_public_key = public_key_handle.read(16 * 1024 + 1) if bundled_public_key != trusted_public_key: raise SystemExit("release bundle public key does not match installer trust") launcher_keys = re.findall( rb'^RELEASE_PUBLIC_KEY = b"""(-----BEGIN PUBLIC KEY-----\n' rb'[A-Za-z0-9+/=\n]+-----END PUBLIC KEY-----\n)"""$', launcher_payload, re.MULTILINE, ) if launcher_keys != [trusted_public_key]: raise SystemExit("runtime repair launcher public key does not match installer trust") installer_payload = read_member(installer_name, 2 * 1024 * 1024) installer_signature = read_member(signature_name, 64) if len(installer_signature) != 64: raise SystemExit("Project runtime repair installer signature is malformed") current_launcher_matches = re.findall( rb'^PORTAL_RUNTIME_REPAIR_LAUNCHER_CURRENT = ' rb'\(([0-9]+), "([0-9a-f]{64})"\)$', installer_payload, re.MULTILINE, ) launcher_identity = ( len(launcher_payload), hashlib.sha256(launcher_payload).hexdigest() ) if len(current_launcher_matches) != 1 or ( int(current_launcher_matches[0][0]), current_launcher_matches[0][1].decode("ascii") ) != launcher_identity: raise SystemExit("runtime repair launcher identity is stale") launcher_releases = re.search( rb"PORTAL_RUNTIME_REPAIR_LAUNCHER_RELEASES = \{(.*?)^\}", installer_payload, re.MULTILINE | re.DOTALL, ) if ( launcher_releases is None or len(re.findall( rb"^\s*PORTAL_RUNTIME_REPAIR_LAUNCHER_CURRENT,\s*$", launcher_releases.group(1), re.MULTILINE, )) != 1 ): raise SystemExit("runtime repair launcher is not scanner-admitted") with tempfile.TemporaryDirectory( prefix=".repair-signature-", dir=os.path.dirname(artifact_path) ) as verification_dir: installer_copy = os.path.join(verification_dir, "install.sh") signature_copy = os.path.join(verification_dir, "install.sh.sig") for target, payload in ( (installer_copy, installer_payload), (signature_copy, installer_signature), ): descriptor = os.open( target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600, ) try: view = memoryview(payload) while view: written = os.write(descriptor, view) if written <= 0: raise SystemExit("Project runtime repair signature staging stalled") view = view[written:] os.fsync(descriptor) finally: os.close(descriptor) verification = subprocess.run( [ "/usr/bin/openssl", "pkeyutl", "-verify", "-pubin", "-inkey", public_key_path, "-rawin", "-in", installer_copy, "-sigfile", signature_copy, ], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env={ "HOME": "/root", "LANG": "C", "LC_ALL": "C", "OPENSSL_CONF": "/dev/null", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin", }, timeout=15, check=False, ) if verification.returncode != 0: raise SystemExit("Project runtime repair installer signature is invalid") def decoded(name, maximum=4 * 1024 * 1024): return read_member(name, maximum).decode("utf-8") package_versions = { "backend": json.loads(decoded("portal/backend/package.json")).get("version"), "frontend": json.loads(decoded("portal/frontend/package.json")).get("version"), } compiled_match = re.search( r"PORTAL_VERSION\s*=\s*['\"]([^'\"]+)['\"]", decoded("portal/backend/dist/version.js"), ) installer_match = re.search( r"^readonly\s+VERSION=['\"]([^'\"]+)['\"]\s*$", decoded("portal/installer/install.sh"), re.MULTILINE, ) readme_match = re.search( r"^Read the complete \[([0-9]+\.[0-9]+\.[0-9]+) changelog\]" r"\(CHANGELOG\.md#([0-9]+)---(\d{4}-\d{2}-\d{2})\)", decoded("portal/README.md"), re.MULTILINE, ) changelog_match = re.search( r"^## \[([0-9]+\.[0-9]+\.[0-9]+)\] - (\d{4}-\d{2}-\d{2})\s*$", decoded("portal/CHANGELOG.md"), re.MULTILINE, ) release_versions = list(package_versions.values()) + [ compiled_match.group(1) if compiled_match else None, installer_match.group(1) if installer_match else None, readme_match.group(1) if readme_match else None, changelog_match.group(1) if changelog_match else None, ] if any(version != expected_version for version in release_versions): raise SystemExit("release source, compiled runtime, docs, and manifest versions disagree") try: release_metadata = json.loads(decoded("portal/RELEASE-METADATA.json", 16 * 1024)) except json.JSONDecodeError: raise SystemExit("release metadata is invalid") metadata_keys = {"schema", "version", "releasedAt", "releaseClass", "highlights"} if (not isinstance(release_metadata, dict) or set(release_metadata) != metadata_keys or release_metadata.get("schema") != 1 or release_metadata.get("version") != expected_version): raise SystemExit("release metadata is invalid or version-mismatched") metadata_released = release_metadata.get("releasedAt") if not isinstance(metadata_released, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", metadata_released): raise SystemExit("release metadata date is invalid") try: if datetime.date.fromisoformat(metadata_released).isoformat() != metadata_released: raise ValueError except ValueError: raise SystemExit("release metadata date is invalid") if changelog_match is None or changelog_match.group(2) != metadata_released: raise SystemExit("release metadata date does not match the changelog") if (readme_match is None or readme_match.group(2) != expected_version.replace(".", "") or readme_match.group(3) != metadata_released): raise SystemExit("release metadata date does not match the README") metadata_class = release_metadata.get("releaseClass") metadata_highlights = release_metadata.get("highlights") if metadata_class not in {"hotfix", "security", "feature", "maintenance"}: raise SystemExit("release metadata class is invalid") if not isinstance(metadata_highlights, list) or not (1 <= len(metadata_highlights) <= 5): raise SystemExit("release metadata highlights are invalid") for highlight in metadata_highlights: if (not isinstance(highlight, str) or highlight != highlight.strip() or not (1 <= len(highlight) <= 200) or any(ord(character) < 32 or ord(character) == 127 for character in highlight)): raise SystemExit("release metadata highlight is invalid") if len(set(metadata_highlights)) != len(metadata_highlights): raise SystemExit("release metadata highlights are duplicated") if values["schema"] == "2" and ( values["released"] != metadata_released or values["release_class"] != metadata_class or manifest_highlights != metadata_highlights): raise SystemExit("signed release metadata does not match the artifact") lineage_raw = decoded("portal/RELEASE-LINEAGE", 4096) lineage = {} for line in lineage_raw.splitlines(): if not line or "=" not in line: raise SystemExit("invalid release lineage") key, value = line.split("=", 1) if key in lineage: raise SystemExit("duplicate release lineage key") lineage[key] = value if values["schema"] == "1": if (set(lineage) != {"schema", "source_commit", "source_version"} or lineage.get("schema") != "1" or not re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", lineage.get("source_commit", "")) or lineage.get("source_version") != expected_version): raise SystemExit("legacy release lineage is malformed or version-mismatched") else: if (set(lineage) != {"schema", "source_commit", "source_committed_at", "source_version"} or lineage.get("schema") != "2" or not re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", lineage.get("source_commit", "")) or lineage.get("source_version") != expected_version): raise SystemExit("release lineage is malformed or version-mismatched") source_committed_at = lineage.get("source_committed_at", "") if not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}", source_committed_at): raise SystemExit("release lineage timestamp is malformed") try: committed_at = datetime.datetime.fromisoformat(source_committed_at) except ValueError: raise SystemExit("release lineage timestamp is malformed") if committed_at.tzinfo is None or committed_at.isoformat(timespec="seconds") != source_committed_at: raise SystemExit("release lineage timestamp is malformed") if committed_at.date() > datetime.date.fromisoformat(metadata_released): raise SystemExit("release date predates the source commit") frontend_index = decoded("portal/frontend/dist/index.html", 16 * 1024 * 1024) referenced_assets = set(re.findall(r"(?:src|href)=['\"](/assets/[^'\"?#]+)", frontend_index)) if not referenced_assets or not any(asset.endswith(".js") for asset in referenced_assets): raise SystemExit("frontend entrypoint has no release assets") for asset in referenced_assets: if f"portal/frontend/dist{asset}" not in seen: raise SystemExit(f"frontend entrypoint references a missing asset: {asset}") PY2 then rm -f -- "${public_key}" return 1 fi rm -f -- "${public_key}" } record_verified_release_identity() { local bundle_dir="$1" local manifest="${bundle_dir}/portal-release.manifest" local artifact="${bundle_dir}/portal.tar.gz" local identity=() VERIFIED_RELEASE_VERSION="" VERIFIED_RELEASE_ARTIFACT_SHA256="" VERIFIED_RELEASE_MANIFEST_SHA256="" VERIFIED_RELEASE_MANIFEST_SCHEMA="" mapfile -t identity < <(python3 - "${manifest}" "${artifact}" <<'PY2' import hashlib import os import re import sys manifest_path, artifact_path = sys.argv[1:3] raw = open(manifest_path, "rb").read() if b"\x00" in raw or b"\r" in raw: raise SystemExit(1) values = {} for line in raw.decode("utf-8").splitlines(): if not line or "=" not in line: raise SystemExit(1) key, value = line.split("=", 1) if key in values: raise SystemExit(1) values[key] = value base_required = {"schema", "version", "artifact", "sha256", "size"} metadata_required = {"released", "release_class", "highlights"} if values.get("schema") == "1": if set(values) != base_required: raise SystemExit(1) elif values.get("schema") == "2": if set(values) != base_required | metadata_required: raise SystemExit(1) else: raise SystemExit(1) if values["artifact"] != "portal.tar.gz": raise SystemExit(1) if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", values["version"]): raise SystemExit(1) if not re.fullmatch(r"[0-9a-f]{64}", values["sha256"]): raise SystemExit(1) if str(os.path.getsize(artifact_path)) != values["size"]: raise SystemExit(1) artifact_hash = hashlib.sha256() with open(artifact_path, "rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): artifact_hash.update(chunk) if artifact_hash.hexdigest() != values["sha256"]: raise SystemExit(1) print(values["schema"]) print(values["version"]) print(values["sha256"]) print(hashlib.sha256(raw).hexdigest()) PY2 ) [[ "${#identity[@]}" -eq 4 \ && "${identity[0]}" =~ ^(1|2)$ \ && "${identity[1]}" == "${VERSION}" \ && "${identity[2]}" =~ ^[a-f0-9]{64}$ \ && "${identity[3]}" =~ ^[a-f0-9]{64}$ ]] || return 1 VERIFIED_RELEASE_MANIFEST_SCHEMA="${identity[0]}" VERIFIED_RELEASE_VERSION="${identity[1]}" VERIFIED_RELEASE_ARTIFACT_SHA256="${identity[2]}" VERIFIED_RELEASE_MANIFEST_SHA256="${identity[3]}" } installed_portal_source_version() { local portal_root="${1:-${PORTAL_DIR}}" python3 - "${portal_root}" "${VERSION}" <<'PY2' import json import os import re import sys root, expected = sys.argv[1:3] def json_version(relative): with open(os.path.join(root, relative), encoding="utf-8") as handle: value = json.load(handle).get("version") if not isinstance(value, str): raise SystemExit(1) return value def literal_version(relative, pattern): with open(os.path.join(root, relative), encoding="utf-8") as handle: match = re.search(pattern, handle.read()) if not match: raise SystemExit(1) return match.group(1) versions = { "backend package": json_version("backend/package.json"), "frontend package": json_version("frontend/package.json"), "compiled runtime": literal_version( "backend/dist/version.js", r"PORTAL_VERSION\s*=\s*['\"]([^'\"]+)['\"]", ), "installed installer": literal_version( "installer/install.sh", r"readonly\s+VERSION\s*=\s*['\"]([^'\"]+)['\"]", ), } source_path = os.path.join(root, "backend/src/version.ts") if os.path.isfile(source_path) and not os.path.islink(source_path): versions["source"] = literal_version( "backend/src/version.ts", r"PORTAL_VERSION\s*=\s*['\"]([^'\"]+)['\"]", ) if any(value != expected for value in versions.values()): raise SystemExit(1) print(versions["backend package"]) PY2 } write_portal_deploy_stamp() { local destination="$1" local source_version="$2" local installed_at="$3" [[ "${VERIFIED_RELEASE_MANIFEST_SCHEMA}" =~ ^(1|2)$ \ && "${VERIFIED_RELEASE_VERSION}" == "${VERSION}" \ && "${source_version}" == "${VERSION}" \ && "${VERIFIED_RELEASE_ARTIFACT_SHA256}" =~ ^[a-f0-9]{64}$ \ && "${VERIFIED_RELEASE_MANIFEST_SHA256}" =~ ^[a-f0-9]{64}$ ]] || return 1 python3 - "${destination}" "${source_version}" "${VERIFIED_RELEASE_VERSION}" \ "${VERIFIED_RELEASE_ARTIFACT_SHA256}" "${VERIFIED_RELEASE_MANIFEST_SHA256}" \ "${VERIFIED_RELEASE_MANIFEST_SCHEMA}" "${installed_at}" <<'PY2' import datetime import os import re import stat import sys import tempfile destination, source_version, release_version, artifact_sha, manifest_sha, manifest_schema, installed_at = sys.argv[1:8] if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", source_version): raise SystemExit(1) if source_version != release_version: raise SystemExit(1) if not re.fullmatch(r"[0-9a-f]{64}", artifact_sha) or not re.fullmatch(r"[0-9a-f]{64}", manifest_sha): raise SystemExit(1) if manifest_schema not in {"1", "2"}: raise SystemExit(1) try: parsed_at = datetime.datetime.fromisoformat(installed_at.replace("Z", "+00:00")) except ValueError: raise SystemExit(1) if parsed_at.tzinfo is None: raise SystemExit(1) parent = os.path.dirname(destination) parent_stat = os.lstat(parent) if not stat.S_ISDIR(parent_stat.st_mode) or stat.S_ISLNK(parent_stat.st_mode): raise SystemExit(1) try: current = os.lstat(destination) except FileNotFoundError: current = None if current is not None and (not stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) or current.st_size > 16384): raise SystemExit(1) payload = ( "schema=1\n" f"source_version={source_version}\n" f"release_version={release_version}\n" f"artifact_sha256={artifact_sha}\n" f"manifest_sha256={manifest_sha}\n" f"manifest_schema={manifest_schema}\n" f"installed_at={installed_at}\n" ).encode("ascii") temporary = None try: fd, temporary = tempfile.mkstemp(prefix=".last-portal-deploy.tmp-", dir=parent) os.fchmod(fd, 0o600) with os.fdopen(fd, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, destination) temporary = None os.chmod(destination, 0o600, follow_symlinks=False) directory_fd = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY2 } atomic_restore_portal_deploy_stamp() { local source="$1" destination="$2" [[ -f "${source}" && ! -L "${source}" && "$(stat -c '%s' "${source}")" -le 16384 ]] || return 1 python3 - "${source}" "${destination}" <<'PY2' import os import stat import sys import tempfile source, destination = sys.argv[1:3] parent = os.path.dirname(destination) parent_stat = os.lstat(parent) source_stat = os.lstat(source) if not stat.S_ISDIR(parent_stat.st_mode) or stat.S_ISLNK(parent_stat.st_mode): raise SystemExit(1) if not stat.S_ISREG(source_stat.st_mode) or stat.S_ISLNK(source_stat.st_mode) or source_stat.st_size > 16384: raise SystemExit(1) payload = open(source, "rb").read() temporary = None try: fd, temporary = tempfile.mkstemp(prefix=".last-portal-deploy.restore-", dir=parent) os.fchmod(fd, 0o600) with os.fdopen(fd, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, destination) temporary = None os.chmod(destination, 0o600, follow_symlinks=False) # Provenance manifests pin mtime_ns; the restored stamp must carry the # exact timestamps of the verified backup copy or re-verification of # the restored file can never pass. os.utime( destination, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns), follow_symlinks=False, ) directory_fd = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY2 } remove_portal_deploy_stamp() { local destination="${1:-${PORTAL_DEPLOY_STAMP}}" python3 - "${destination}" <<'PY2' import os import stat import sys destination = sys.argv[1] parent = os.path.dirname(destination) parent_stat = os.lstat(parent) if not stat.S_ISDIR(parent_stat.st_mode) or stat.S_ISLNK(parent_stat.st_mode): raise SystemExit(1) try: current = os.lstat(destination) except FileNotFoundError: raise SystemExit(0) if stat.S_ISDIR(current.st_mode): raise SystemExit(1) os.unlink(destination) directory_fd = os.open(parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY2 } capture_portal_deploy_stamp_for_update() { local backup_dir="$1" stamp_path="${2:-${PORTAL_DEPLOY_STAMP}}" local backup_path="${backup_dir}/last-portal-deploy.previous" UPDATE_RECOVERY_DEPLOY_STAMP_CAPTURED=false UPDATE_RECOVERY_DEPLOY_STAMP_EXISTED=false UPDATE_RECOVERY_DEPLOY_STAMP_BACKUP="${backup_path}" rm -f -- "${backup_path}" if [[ -e "${stamp_path}" || -L "${stamp_path}" ]]; then [[ -f "${stamp_path}" && ! -L "${stamp_path}" && "$(stat -c '%s' "${stamp_path}")" -le 16384 ]] \ || return 1 install -m 0600 -- "${stamp_path}" "${backup_path}" || return 1 UPDATE_RECOVERY_DEPLOY_STAMP_EXISTED=true fi UPDATE_RECOVERY_DEPLOY_STAMP_CAPTURED=true } restore_portal_deploy_stamp_after_failed_update() { local stamp_path="${1:-${PORTAL_DEPLOY_STAMP}}" ${UPDATE_RECOVERY_DEPLOY_STAMP_CAPTURED:-false} || return 0 if ${UPDATE_RECOVERY_DEPLOY_STAMP_EXISTED:-false}; then atomic_restore_portal_deploy_stamp "${UPDATE_RECOVERY_DEPLOY_STAMP_BACKUP}" "${stamp_path}" else remove_portal_deploy_stamp "${stamp_path}" fi } commit_portal_deploy_provenance() { local stamp_path="${1:-${PORTAL_DEPLOY_STAMP}}" local portal_root="${2:-${PORTAL_DIR}}" local source_version installed_at $DRY_RUN && return 0 if ${UNVERIFIED_LOCAL_SOURCE_USED:-false}; then remove_portal_deploy_stamp "${stamp_path}" || return 1 warn "Unverified local source installed; signed release provenance remains deliberately unavailable." return 0 fi source_version="$(installed_portal_source_version "${portal_root}")" || return 1 [[ "${source_version}" == "${VERIFIED_RELEASE_VERSION}" \ && "${source_version}" == "${VERSION}" ]] || return 1 installed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" write_portal_deploy_stamp "${stamp_path}" "${source_version}" "${installed_at}" } stage_verified_release() { local bundle_dir="$1" VERIFIED_RELEASE_VERSION="" VERIFIED_RELEASE_ARTIFACT_SHA256="" VERIFIED_RELEASE_MANIFEST_SHA256="" VERIFIED_RELEASE_MANIFEST_SCHEMA="" if ! release_base_url_is_trusted; then warn "Refusing untrusted release base URL: ${RELEASE_BASE_URL}" return 1 fi install -d -m 0700 "${bundle_dir}" curl --fail --silent --show-error --location --proto '=https,http' --tlsv1.2 \ --max-filesize 536870912 --output "${bundle_dir}/portal.tar.gz" "${RELEASE_URL}" \ || return 1 curl --fail --silent --show-error --location --proto '=https,http' --tlsv1.2 \ --max-filesize 16384 --output "${bundle_dir}/portal-release.manifest" "${RELEASE_MANIFEST_URL}" \ || return 1 curl --fail --silent --show-error --location --proto '=https,http' --tlsv1.2 \ --max-filesize 4096 --output "${bundle_dir}/portal-release.sig" "${RELEASE_SIGNATURE_URL}" \ || return 1 verify_release_bundle "${bundle_dir}" || return 1 record_verified_release_identity "${bundle_dir}" || return 1 tar --extract --gzip --file "${bundle_dir}/portal.tar.gz" --directory "${bundle_dir}" \ --no-same-owner --no-same-permissions } new_release_stage_dir() { local transaction_id="${1:-}" if [[ -z "${transaction_id}" ]]; then mktemp -d /tmp/bridgesllm-release-stage.XXXXXX return fi [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 local stage_root stage_root="$(update_transaction_state_path "${UPDATE_STAGE_ROOT}")" || return 1 install -d -m 0700 -- "${stage_root}" || return 1 [[ -d "${stage_root}" && ! -L "${stage_root}" \ && "$(stat -c '%u:%g:%a' "${stage_root}")" == "$(id -u):$(id -g):700" ]] \ || return 1 local stage_dir="${stage_root}/update-${transaction_id}" mkdir -m 0700 -- "${stage_dir}" || return 1 printf '%s\n' "${stage_dir}" } cleanup_release_stage_dir() { local stage_dir="${1:-}" local transaction_stage_root="" transaction_stage_root="$( update_transaction_state_path "${UPDATE_STAGE_ROOT}" 2>/dev/null || true )" case "${stage_dir}" in /tmp/bridgesllm-release-stage.*) rm -rf -- "${stage_dir}" ;; "${transaction_stage_root}"/update-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) rm -rf -- "${stage_dir}" ;; "") return 0 ;; *) warn "Refusing to remove unexpected release staging path: ${stage_dir}"; return 1 ;; esac } prepare_portal_operation_lock() { local lock_path="$1" python3 - "${lock_path}" <<'PY' import errno import os import stat import sys path = sys.argv[1] if not os.path.isabs(path) or path != os.path.normpath(path): raise SystemExit("Portal operation lock path must be canonical and absolute") if os.geteuid() != 0: raise SystemExit("Portal operation lock must be acquired by root") # Every existing directory component must be root-owned and either not # group/world-writable or protected by the sticky bit. This permits the normal # 01777 /run/lock directory without trusting a user-created final inode. parent = os.path.dirname(path) current = os.path.sep for component in parent.strip(os.path.sep).split(os.path.sep): if not component: continue current = os.path.join(current, component) info = os.lstat(current) if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0: raise SystemExit("Portal operation lock directory boundary is unsafe") if info.st_mode & 0o022 and not info.st_mode & stat.S_ISVTX: raise SystemExit("Portal operation lock directory is writable without sticky-bit protection") flags = os.O_RDWR if hasattr(os, "O_CLOEXEC"): flags |= os.O_CLOEXEC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600) except FileExistsError: fd = os.open(path, flags) except OSError as error: if error.errno != errno.EEXIST: raise fd = os.open(path, flags) try: info = os.fstat(fd) if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or info.st_size != 0 or info.st_mode & 0o022): raise SystemExit("Portal operation lock inode is unsafe") os.fchmod(fd, 0o600) os.fsync(fd) info = os.fstat(fd) print(f"{info.st_dev}:{info.st_ino}:{info.st_uid}:{info.st_gid}:{stat.S_IMODE(info.st_mode):o}:{info.st_nlink}:{info.st_size}") finally: os.close(fd) PY } acquire_portal_operation_lock() { local lock_path="${1:-${PORTAL_OPERATION_LOCK_PATH}}" local expected_inode actual_inode active_journal cutover_journal local backup_journal restore_journal expected_inode="$(prepare_portal_operation_lock "${lock_path}")" \ || fail "Portal operation lock could not be prepared safely." exec 9<>"${lock_path}" \ || fail "Portal operation lock could not be opened safely." actual_inode="$(stat -Lc '%d:%i:%u:%g:%a:%h:%s' /proc/$$/fd/9 2>/dev/null)" \ || fail "Portal operation lock descriptor could not be attested." [[ "${actual_inode}" == "${expected_inode}" ]] \ || fail "Portal operation lock changed while it was being opened." flock -n 9 || fail "Another BridgesLLM install, update, uninstall, or backup operation is already running." active_journal="$(update_transaction_state_path "${UPDATE_ACTIVE_JOURNAL}")" \ || fail "The update transaction fixture root is invalid." cutover_journal="$(update_transaction_state_path "${UPDATE_CUTOVER_JOURNAL}")" \ || fail "The update transaction fixture root is invalid." if [[ -e "${active_journal}" || -L "${active_journal}" \ || -e "${cutover_journal}" || -L "${cutover_journal}" ]]; then declare -F recover_pending_update_transaction >/dev/null 2>&1 \ || fail "An interrupted update transaction exists, but this installer cannot recover it." recover_pending_update_transaction \ || fail "The interrupted update could not be recovered safely. Portal remains boot-fenced; recovery artifacts were preserved." else release_update_disk_reserves \ || fail "A stale update recovery reserve could not be released safely." fi recover_pending_uninstall_transaction \ || fail "The interrupted uninstall could not be resumed safely. Its boot fence and root-only transaction artifacts were preserved." backup_journal="$(update_transaction_state_path "${BACKUP_QUIESCENCE_JOURNAL}")" \ || fail "The backup recovery fixture root is invalid." [[ ! -e "${backup_journal}" && ! -L "${backup_journal}" ]] \ || fail "An interrupted backup must recover its quiesced runtime state before install, update, or uninstall can mutate the host." restore_journal="$(update_transaction_state_path "${RESTORE_ACTIVE_JOURNAL}")" \ || fail "The restore recovery fixture root is invalid." [[ ! -e "${restore_journal}" && ! -L "${restore_journal}" ]] \ || fail "An interrupted restore must recover before install, update, or uninstall can mutate the host." sweep_stale_release_stage_dirs } acquire_project_runtime_image_repair_lock() { # This narrow repair shares the installer/update/uninstall lock, but it must # never recover, resume, sweep, or otherwise join those transactions. Any # durable operation state therefore blocks repair without being changed. local lock_path="${1:-${PORTAL_OPERATION_LOCK_PATH}}" local expected_inode actual_inode journal resolved expected_inode="$(prepare_portal_operation_lock "${lock_path}")" \ || fail "Portal operation lock could not be prepared safely." exec 9<>"${lock_path}" \ || fail "Portal operation lock could not be opened safely." actual_inode="$(stat -Lc '%d:%i:%u:%g:%a:%h:%s' /proc/$$/fd/9 2>/dev/null)" \ || fail "Portal operation lock descriptor could not be attested." [[ "${actual_inode}" == "${expected_inode}" ]] \ || fail "Portal operation lock changed while it was being opened." flock -n 9 \ || fail "Another BridgesLLM install, update, uninstall, backup, restore, or repair operation is already running." for journal in \ "${UPDATE_ACTIVE_JOURNAL}" \ "${UPDATE_CUTOVER_JOURNAL}" \ "${UNINSTALL_ACTIVE_JOURNAL}" \ "${BACKUP_QUIESCENCE_JOURNAL}" \ "${RESTORE_ACTIVE_JOURNAL}"; do resolved="$(update_transaction_state_path "${journal}")" \ || fail "A durable Portal operation path is invalid." [[ ! -e "${resolved}" && ! -L "${resolved}" ]] \ || fail "A durable install, update, uninstall, backup, or restore operation must settle before Project runtime image repair." done } sweep_stale_release_stage_dirs() { # Holding the operation lock proves no other installer is running, so any # existing staging tree is residue from a hard-aborted earlier run (traps # cannot fire on SIGKILL/panic). Observed accumulating one tree per aborted # deploy on a long-lived box. local stale for stale in /tmp/bridgesllm-release-stage.*; do [[ -e "${stale}" ]] || continue cleanup_release_stage_dir "${stale}" >/dev/null 2>&1 || true done local active_journal cutover_journal stage_root active_journal="$(update_transaction_state_path "${UPDATE_ACTIVE_JOURNAL}")" || return 1 cutover_journal="$(update_transaction_state_path "${UPDATE_CUTOVER_JOURNAL}")" || return 1 [[ ! -e "${active_journal}" && ! -L "${active_journal}" \ && ! -e "${cutover_journal}" && ! -L "${cutover_journal}" ]] \ || return 1 stage_root="$(update_transaction_state_path "${UPDATE_STAGE_ROOT}")" || return 1 if [[ -d "${stage_root}" && ! -L "${stage_root}" ]]; then for stale in "${stage_root}"/update-*; do [[ -e "${stale}" ]] || continue local stale_name transaction_id cleanup_policy stale_name="$(basename "${stale}")" if [[ "${stale_name}" =~ ^update-([a-f0-9]{32})$ \ && -d "${stale}" && ! -L "${stale}" \ && "$(stat -c '%u:%g' "${stale}")" == "$(id -u):$(id -g)" ]]; then transaction_id="${BASH_REMATCH[1]}" cleanup_policy="$( prepared_update_project_runtime_cleanup_policy_from_stage \ "${stale}" "${transaction_id}" 2>/dev/null || true )" cleanup_prepared_update_project_runtime_tags \ "${transaction_id}" "${cleanup_policy}" \ || return 1 rm -rf -- "${stale}" fi done fi } detect_wsl() { [[ -n "${WSL_DISTRO_NAME:-}" || -n "${WSL_INTEROP:-}" ]] && return 0 grep -qiE '(microsoft|wsl)' /proc/sys/kernel/osrelease 2>/dev/null && return 0 grep -qiE '(microsoft|wsl)' /proc/version 2>/dev/null && return 0 return 1 } detect_runtime_profile() { if detect_wsl; then IS_WSL=true INSTALL_PROFILE="local" fi } use_local_profile() { [[ "${INSTALL_PROFILE}" == "local" ]] } use_tailnet_profile() { [[ "${ORIGIN_MODE}" == "tailnet" ]] } systemd_ready() { command -v systemctl &>/dev/null && [[ -d /run/systemd/system ]] } portal_primary_origin() { if use_local_profile; then echo "http://localhost:4001" elif use_tailnet_profile && [[ -n "${TAILNET_DNS_NAME}" ]]; then echo "https://${TAILNET_DNS_NAME}" elif [[ -n "$DOMAIN" ]]; then echo "https://${DOMAIN}" else # A server without proven TLS stays loopback-only. Remote operators reach # this origin through the encrypted SSH tunnel printed after installation. echo "http://localhost:4001" fi } portal_cors_origins() { if use_local_profile; then echo "http://localhost:4001,http://127.0.0.1:4001" elif use_tailnet_profile && [[ -n "${TAILNET_DNS_NAME}" ]]; then # Tailscale Serve terminates TLS on the tailnet name and proxies to # loopback; the loopback origins keep the SSH-tunnel escape hatch working. echo "https://${TAILNET_DNS_NAME},http://localhost:4001,http://127.0.0.1:4001" elif [[ -n "$DOMAIN" ]]; then echo "https://${DOMAIN},https://www.${DOMAIN}" else echo "http://localhost:4001,http://127.0.0.1:4001" fi } app_content_domain_from_origin() { local origin="${1:-}" python3 - "${origin}" <<'PY2' import sys from urllib.parse import urlparse try: parsed = urlparse(sys.argv[1]) if parsed.scheme == "https" and parsed.hostname and not parsed.username and not parsed.password: print(parsed.hostname.lower()) except Exception: pass PY2 } configure_app_content_identity() { if use_local_profile; then APP_CONTENT_DOMAIN="apps.localhost" APP_CONTENT_ORIGIN="http://apps.localhost:4001" APP_CONTENT_DNS_MODE="local" return 0 fi if use_tailnet_profile; then # Tailscale grants one DNS name per machine, and app content must live on a # cookie-distinct host (alternate ports share cookies and are unsafe). # Hosted app content therefore stays disabled in tailnet mode until a # second tailnet identity (Tailscale Services) is wired in; the backend # fails closed without an APP_CONTENT_ORIGIN. APP_CONTENT_DOMAIN="" APP_CONTENT_ORIGIN="" APP_CONTENT_DNS_MODE="tailnet-disabled" return 0 fi [[ "${PUBLIC_IP}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] \ || fail "A public IPv4 address is required for isolated app-content hosting." if [[ -z "${APP_CONTENT_DOMAIN}" ]]; then APP_CONTENT_DOMAIN="app-content.${PUBLIC_IP}.sslip.io" APP_CONTENT_DNS_MODE="sslip" elif [[ -z "${APP_CONTENT_DNS_MODE}" ]]; then APP_CONTENT_DNS_MODE="custom" fi APP_CONTENT_DOMAIN="${APP_CONTENT_DOMAIN,,}" APP_CONTENT_ORIGIN="https://${APP_CONTENT_DOMAIN}" local validation_error="" validation_error="$(python3 - "${DOMAIN}" "${PUBLIC_IP}" "${APP_CONTENT_DOMAIN}" <<'PY2' import ipaddress import re import sys portal_domain, public_ip, app_domain = (value.strip().lower() for value in sys.argv[1:4]) try: ipaddress.IPv4Address(public_ip) except ValueError: print("invalid public IPv4 address") raise SystemExit if len(app_domain) > 253 or not re.fullmatch(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", app_domain): print("invalid app-content hostname") raise SystemExit if ".." in app_domain or "." not in app_domain: print("invalid app-content hostname") raise SystemExit def site_key(hostname: str) -> str: labels = [label for label in hostname.split(".") if label] return ".".join(labels[-2:]) if portal_domain and (app_domain == portal_domain or site_key(app_domain) == site_key(portal_domain)): print("app-content hostname shares the Portal cookie site") PY2 )" [[ -z "${validation_error}" ]] \ || fail "Unsafe APP_CONTENT_DOMAIN (${APP_CONTENT_DOMAIN}): ${validation_error}. Use a separate registrable domain, not a sibling hostname or alternate port." local resolved_ips="" resolved_ips="$(python3 - "${APP_CONTENT_DOMAIN}" <<'PY2' import socket import sys try: addresses = sorted({item[4][0] for item in socket.getaddrinfo(sys.argv[1], 443, socket.AF_INET, socket.SOCK_STREAM)}) print("\n".join(addresses)) except OSError: pass PY2 )" if ! grep -Fxq "${PUBLIC_IP}" <<<"${resolved_ips}"; then fail "APP_CONTENT_DOMAIN ${APP_CONTENT_DOMAIN} must have an A record pointing to ${PUBLIC_IP} before installation." fi if [[ "${APP_CONTENT_DNS_MODE}" == "sslip" ]]; then info "Using DNS-proven isolated app origin ${APP_CONTENT_ORIGIN} (sslip.io fallback)" warn "This fallback depends on external sslip.io DNS. Use --app-content-domain with a separate domain to remove that dependency." else info "Using isolated app origin ${APP_CONTENT_ORIGIN}" fi } render_app_content_caddy_block() { cat < 1: raise SystemExit("malformed or duplicate app-content Caddy markers") if begin_count == 1: start = content.index(begin) finish = content.index(end, start) + len(end) if finish < len(content) and content[finish] == "\n": finish += 1 updated = content[:start] + block + content[finish:] else: separator = "" if not content or content.endswith("\n") else "\n" updated = content + separator + block with open(candidate, "w", encoding="utf-8") as handle: handle.write(updated) handle.flush() os.fsync(handle.fileno()) PY2 chmod --reference="${caddy_path}" "${candidate}" 2>/dev/null || chmod 0644 "${candidate}" chown --reference="${caddy_path}" "${candidate}" 2>/dev/null || true if cmp -s "${candidate}" "${caddy_path}"; then rm -f "${candidate}" return 0 fi if ! caddy validate --config "${candidate}" --adapter caddyfile >> "$LOG_FILE" 2>&1; then rm -f "${candidate}" fail "Isolated app-content Caddy configuration is invalid; existing configuration was left unchanged." fi local rollback_copy="" rollback_copy="$(mktemp /etc/caddy/.Caddyfile.app-content-rollback.XXXXXX)" cp -a "${caddy_path}" "${rollback_copy}" mv -f "${candidate}" "${caddy_path}" if [[ "${reload_caddy}" == "true" ]]; then if ! systemctl reload caddy >> "$LOG_FILE" 2>&1; then mv -f "${rollback_copy}" "${caddy_path}" systemctl reload caddy >> "$LOG_FILE" 2>&1 || true fail "Caddy reload rejected the app-content site; the previous configuration was restored." fi UPDATE_RECOVERY_CADDY_CHANGED=true fi rm -f "${rollback_copy}" } portal_https_resolve_targets() { # curl --resolve pins an HTTPS probe to this machine's own Caddy while curl # still performs normal hostname and certificate validation. Loopback is the # common case, but Caddy can be bound to specific addresses (default_bind), # in which case nothing listens on 127.0.0.1:443 and a loopback-only probe # can never connect. Probe every address Caddy actually serves HTTPS on. local host="$1" address [[ -n "${host}" ]] || return 1 local -a addresses=(127.0.0.1) while read -r address; do case "${address}" in ''|'*'|'0.0.0.0'|'::') continue ;; esac addresses+=("${address}") done < <( ss -H -ltnp 2>/dev/null \ | awk '/"caddy"/ && $4 ~ /:443$/ {print $4}' \ | sed -E 's/:443$//' \ | tr -d '[]' \ | sort -u ) if [[ -n "${PUBLIC_IP:-}" ]]; then addresses+=("${PUBLIC_IP}") fi printf '%s\n' "${addresses[@]}" \ | awk 'NF && !seen[$0]++' \ | while read -r address; do if [[ "${address}" == *:* ]]; then printf '%s:443:[%s]\n' "${host}" "${address}" else printf '%s:443:%s\n' "${host}" "${address}" fi done } verify_app_content_caddy_ready() { use_local_profile && return 0 # Tailnet mode ships without isolated app-content hosting (one DNS name per # machine; alternate ports share cookies and are unsafe). use_tailnet_profile && return 0 local waited=0 status="" share_probe="" target while (( waited < 90 )); do while read -r target; do status="$(curl -sS --max-time 5 --resolve "${target}" \ -o /dev/null -w '%{http_code}' "https://${APP_CONTENT_DOMAIN}/__bridgesllm_isolation_probe" 2>/dev/null || true)" share_probe="$(curl -sSI --max-time 5 --resolve "${target}" \ "https://${APP_CONTENT_DOMAIN}/share/__bridgesllm_invalid_probe__" 2>/dev/null || true)" if [[ "${status}" == "404" ]] \ && grep -qi '^HTTP/.* 404' <<<"${share_probe}" \ && grep -qi '^x-robots-tag:.*noindex' <<<"${share_probe}"; then ok "Isolated app-content TLS host is ready" return 0 fi done < <(portal_https_resolve_targets "${APP_CONTENT_DOMAIN}") sleep 3 waited=$((waited + 3)) done return 1 } verify_portal_tls_ready() { [[ -n "${DOMAIN}" ]] || return 1 local waited=0 status="" target while (( waited < 90 )); do while read -r target; do # --resolve pins the probe to this Caddy instance while curl still # performs normal hostname and certificate validation. Never use -k here. status="$(curl -sS --max-time 5 --resolve "${target}" \ -o /dev/null -w '%{http_code}' "https://${DOMAIN}/api/setup/status" 2>/dev/null || true)" if [[ "${status}" == "200" ]]; then ok "Portal HTTPS identity is ready" return 0 fi done < <(portal_https_resolve_targets "${DOMAIN}") sleep 3 waited=$((waited + 3)) done return 1 } portal_setup_url() { if [[ -z "${SETUP_TOKEN}" ]]; then printf '%s/login\n' "$(portal_primary_origin)" elif use_tailnet_profile && [[ -n "${TAILNET_DNS_NAME}" ]]; then printf 'https://%s/setup#bootstrap=%s\n' "${TAILNET_DNS_NAME}" "${SETUP_TOKEN}" elif [[ -n "$DOMAIN" ]] && ! use_local_profile; then printf 'https://%s/setup#bootstrap=%s\n' "${DOMAIN}" "${SETUP_TOKEN}" else # URL fragments are not sent in the initial HTTP request or Referer. The # wizard strips this fragment before exchanging it once on loopback. printf 'http://localhost:4001/setup#bootstrap=%s\n' "${SETUP_TOKEN}" fi } portal_setup_ssh_user() { local candidate="${SUDO_USER:-root}" [[ "${candidate}" =~ ^[a-z_][a-z0-9_-]*$ ]] || candidate="root" printf '%s\n' "${candidate}" } resolve_caddy_config_helper() { local candidate="" if [[ -n "${UPDATE_RELEASE_STAGE_DIR}" ]]; then candidate="${UPDATE_RELEASE_STAGE_DIR}/portal/installer/caddy-managed-config.py" if [[ -f "${candidate}" && ! -L "${candidate}" ]]; then printf '%s\n' "${candidate}" return 0 fi fi if [[ -f "${CADDY_CONFIG_HELPER}" && ! -L "${CADDY_CONFIG_HELPER}" ]]; then printf '%s\n' "${CADDY_CONFIG_HELPER}" return 0 fi return 1 } normalize_portal_domain() { [[ -n "${DOMAIN}" ]] || return 0 local helper normalized helper="$(resolve_caddy_config_helper)" \ || fail "The signed release is missing the managed Caddy configuration helper." normalized="$(python3 "${helper}" normalize-domain --value "${DOMAIN}")" \ || fail "DOMAIN is not a valid canonical hostname." DOMAIN="${normalized}" } # Defense-in-depth: if no domain was provided or preserved in the Portal # environment, recover it only from one BridgesLLM-owned Caddy block. Foreign # sites on a shared Caddy host are never candidates. recover_domain_from_caddyfile() { [[ -z "${DOMAIN}" ]] || { normalize_portal_domain; return 0; } use_local_profile && return 0 use_tailnet_profile && return 0 [[ -e /etc/caddy/Caddyfile || -L /etc/caddy/Caddyfile ]] || return 0 local helper recovered_domain helper="$(resolve_caddy_config_helper)" \ || fail "The signed release is missing the managed Caddy configuration helper." recovered_domain="$( python3 "${helper}" recover-domain --caddy-path /etc/caddy/Caddyfile )" || fail "The existing Caddyfile has unsafe or ambiguous BridgesLLM ownership markers." if [[ -n "${recovered_domain}" ]]; then DOMAIN="${recovered_domain}" info "Recovered the Portal domain from its owned Caddy block: ${DOMAIN}" fi } tailnet_status_json() { { tailscale status --json 2>/dev/null || true; } } tailnet_backend_state() { tailnet_status_json | python3 -c ' import json, sys try: print(str(json.load(sys.stdin).get("BackendState") or "")) except Exception: pass ' 2>/dev/null || true } tailnet_dns_name_from_status() { tailnet_status_json | python3 -c ' import json, sys try: data = json.load(sys.stdin) name = str((data.get("Self") or {}).get("DNSName") or "").strip().rstrip(".") if name: print(name) except Exception: pass ' 2>/dev/null || true } tailnet_auth_url_from_status() { tailnet_status_json | python3 -c ' import json, sys try: url = str(json.load(sys.stdin).get("AuthURL") or "").strip() if url.startswith("https://"): print(url) except Exception: pass ' 2>/dev/null || true } tailnet_cert_domains_ready() { tailnet_status_json | python3 -c ' import json, sys try: domains = json.load(sys.stdin).get("CertDomains") or [] sys.exit(0 if isinstance(domains, list) and len(domains) > 0 else 1) except Exception: sys.exit(1) ' } # Private-origin mode: the portal is served at https://..ts.net # via Tailscale Serve, TLS included, with zero public ports. Runs early so the # operator can approve the machine in their browser while the install proceeds. setup_tailnet_origin() { step_header "Joining your private Tailscale network" CURRENT_STEP="tailnet-origin" if ! command -v tailscale &>/dev/null; then info "Installing Tailscale (official installer)..." local ts_installer ts_installer="$(mktemp /tmp/bridgesllm-tailscale-install.XXXXXX.sh)" curl -fsSL https://tailscale.com/install.sh -o "${ts_installer}" \ || fail "Could not download the Tailscale installer. Check outbound connectivity and rerun." head -c 200 "${ts_installer}" | grep -q '^#!/bin/sh' \ || fail "The downloaded Tailscale installer did not look like the official script. Aborting without running it." sh "${ts_installer}" >> "$LOG_FILE" 2>&1 || fail "Tailscale installation failed — see ${LOG_FILE}" rm -f "${ts_installer}" fi systemctl enable --now tailscaled >> "$LOG_FILE" 2>&1 || true ok "Tailscale $(tailscale version 2>/dev/null | head -1 || echo installed)" local state state="$(tailnet_backend_state)" if [[ "${state}" != "Running" ]]; then if [[ -n "${TS_AUTHKEY}" ]]; then info "Connecting this server to your tailnet with the provided auth key..." local key_file key_file="$(mktemp /root/.bridgesllm-tskey.XXXXXX)" chmod 600 "${key_file}" printf '%s' "${TS_AUTHKEY}" > "${key_file}" if ! timeout 90 tailscale up --reset --hostname="${TS_HOSTNAME}" --auth-key="file:${key_file}" >> "$LOG_FILE" 2>&1; then rm -f "${key_file}" fail "Tailscale rejected the auth key (expired or invalid?). Generate a fresh one at https://login.tailscale.com/admin/settings/keys and rerun." fi rm -f "${key_file}" else info "Starting Tailscale sign-in — a browser approval link will appear below." (tailscale up --reset --hostname="${TS_HOSTNAME}" >> "$LOG_FILE" 2>&1 &) local waited=0 auth_url="" while (( waited < 30 )); do auth_url="$(tailnet_auth_url_from_status)" [[ -n "${auth_url}" ]] && break state="$(tailnet_backend_state)" [[ "${state}" == "Running" ]] && break sleep 2 waited=$((waited + 2)) done if [[ -n "${auth_url}" ]]; then echo "" print_kv "APPROVE THIS SERVER" "${auth_url}" "$GREEN" info "Open that link in any browser, sign in to YOUR Tailscale account (the same one your other devices use), and approve this machine. Waiting up to 10 minutes..." fi waited=0 while (( waited < 600 )); do state="$(tailnet_backend_state)" [[ "${state}" == "Running" ]] && break sleep 3 waited=$((waited + 3)) done [[ "$(tailnet_backend_state)" == "Running" ]] \ || fail "The server did not join the tailnet in time. Approve the link above, then rerun this installer — it resumes safely." fi fi TAILNET_DNS_NAME="$(tailnet_dns_name_from_status)" [[ -n "${TAILNET_DNS_NAME}" ]] \ || fail "Tailscale is connected but reported no MagicDNS name. Enable MagicDNS at https://login.tailscale.com/admin/dns and rerun." ok "Joined tailnet as ${TAILNET_DNS_NAME}" # ts.net TLS requires the tailnet-wide HTTPS Certificates toggle (one time). if ! tailnet_cert_domains_ready; then warn "HTTPS certificates are not enabled for your tailnet yet." info "Enable them once at https://login.tailscale.com/admin/dns → 'HTTPS Certificates' → Enable. Waiting up to 10 minutes..." local cert_waited=0 while (( cert_waited < 600 )); do tailnet_cert_domains_ready && break sleep 5 cert_waited=$((cert_waited + 5)) done tailnet_cert_domains_ready \ || fail "HTTPS certificates are still disabled for the tailnet. Enable them at https://login.tailscale.com/admin/dns, then rerun this installer." fi ok "Tailnet HTTPS certificates are enabled" # Tailscale terminates TLS on the tailnet name and proxies to the loopback # portal. Serve config persists across reboots; the upstream comes alive # when the portal service starts. tailscale serve --bg --https=443 http://127.0.0.1:4001 >> "$LOG_FILE" 2>&1 \ || fail "tailscale serve could not be configured — see ${LOG_FILE}" ok "Portal origin: https://${TAILNET_DNS_NAME}/ (your tailnet only — no public ports)" } verify_tailnet_tls_ready() { # First request triggers the ts.net certificate mint; allow for that. local waited=0 while (( waited < 180 )); do if curl -fsS --max-time 10 "https://${TAILNET_DNS_NAME}/health" -o /dev/null 2>> "$LOG_FILE"; then return 0 fi sleep 5 waited=$((waited + 5)) done return 1 } write_caddy_config() { recover_domain_from_caddyfile local helper transaction_snapshot="" local -a snapshot_args=() helper="$(resolve_caddy_config_helper)" \ || fail "The signed release is missing the managed Caddy configuration helper." if [[ "${UPDATE_RECOVERY_ARMED:-false}" == "true" ]]; then local backup_dir="" backup_dir="$(read_update_transaction_field active backup_dir 2>/dev/null || true)" [[ -n "${backup_dir}" ]] \ || fail "The update transaction has no attested Caddy snapshot directory." transaction_snapshot="${backup_dir}/Caddyfile.updated" snapshot_args=(--installed-snapshot-path "${transaction_snapshot}") fi if ! python3 "${helper}" apply \ --caddy-path /etc/caddy/Caddyfile \ --domain "${DOMAIN}" \ --public-ip "${PUBLIC_IP}" \ --app-domain "${APP_CONTENT_DOMAIN}" \ "${snapshot_args[@]}" >> "$LOG_FILE" 2>&1; then fail "Caddy configuration could not be converged safely. Existing non-Portal sites were left unchanged or restored; see ${LOG_FILE}." fi } # ═══════════════════════════════════════════════════════════════ # Args # ═══════════════════════════════════════════════════════════════ usage() { cat << 'EOF' BridgesLLM Portal Installer Usage: curl -fsSL https://bridgesllm.ai/install.sh | sudo bash Options: --domain DOMAIN Pre-set domain (enables HTTPS immediately) --tailnet EXPERIMENTAL. Private mode: serve the portal on your Tailscale network at https://..ts.net — no domain purchase, no DNS, no public ports. The installer installs Tailscale, shows a browser approval link (or use --ts-authkey), and wires HTTPS via Tailscale Serve. Mail and hosted app content need a public domain and stay disabled in this mode. This path is new and still under field validation; the --domain path remains the recommended production install. --ts-authkey KEY Tailscale auth key for non-interactive --tailnet joins (generate at login.tailscale.com under Settings -> Keys) --ts-hostname NAME Tailnet machine name for --tailnet (default: bridgesllm-portal) --app-content-domain DOMAIN Separate-site hostname for hosted/share app content. Its A record must already point to this server. Sibling Portal subdomains and alternate ports are rejected. --local Experimental local install profile for Windows / WSL testing (used automatically on WSL) --skip-ollama Don't install Ollama --skip-openclaw Don't install OpenClaw --skip-project-runtimes Install with Project Chat runtimes disabled. For hosts where kernel and Docker AppArmor support disagree (LXC/OpenVZ/nested containers) and confined runtimes cannot be attested. The rest of the Portal installs and runs normally. --maintain-tools Explicitly update optional AI tools during an update (Ollama, ClawHub, Codex, Claude Code, Antigravity, Grok Build) --update Force update of an existing installation (re-running the installer on an existing install routes into the signed update flow, so this flag is usually optional) --reinstall Force a fresh install over an existing one --uninstall Remove BridgesLLM portal --repair-project-runtime-image Repair only the installed Portal's canonical full-stack Project runtime image. This Owner-triggered maintenance mode accepts no other option and restarts only Portal. --residue-policy MODE With --uninstall Clean slate, decide leftover managed runtime residue from an earlier partial cleanup without a prompt: 'safe' finishes uninstalling but leaves the leftovers (never touches the host firewall); 'wipe' also deletes the leftover Docker resources and Portal-created firewall entries after saving a full firewall backup. --dry-run Print a zero-side-effect plan. Never writes, downloads, sends telemetry, changes services/network state, or prompts. -h, --help Show this help Requirements: - Ubuntu 22.04/24.04 or Debian 12+ - Root access - 3.5GB+ RAM, 35GB+ disk, 2+ CPUs recommended - Ports 80, 443 available EOF } parse_args() { local original_count=$# repair_argument_count=0 argument for argument in "$@"; do [[ "${argument}" == "--repair-project-runtime-image" ]] \ && repair_argument_count=$((repair_argument_count + 1)) done if (( repair_argument_count > 0 )); then REPAIR_PROJECT_RUNTIME_IMAGE=true (( repair_argument_count == 1 && original_count == 1 )) \ || fail "--repair-project-runtime-image is a mutually exclusive maintenance operation and accepts no other option." fi while [[ $# -gt 0 ]]; do case "$1" in --domain) [[ $# -ge 2 && -n "${2:-}" && "${2:-}" != --* ]] \ || fail "--domain requires a non-empty hostname." DOMAIN="$2" ORIGIN_SELECTION_EXPLICIT=true shift 2 ;; --app-content-domain) [[ $# -ge 2 && -n "${2:-}" && "${2:-}" != --* ]] \ || fail "--app-content-domain requires a non-empty hostname." APP_CONTENT_DOMAIN="$2" APP_CONTENT_DNS_MODE="custom" APP_CONTENT_SELECTION_EXPLICIT=true shift 2 ;; --tailnet) ORIGIN_MODE="tailnet"; ORIGIN_SELECTION_EXPLICIT=true; shift ;; --ts-authkey) TS_AUTHKEY="${2:-}"; shift 2 ;; --ts-hostname) TS_HOSTNAME="${2:-}"; shift 2 ;; --local) INSTALL_PROFILE="local"; ORIGIN_SELECTION_EXPLICIT=true; shift ;; --skip-ollama) SKIP_OLLAMA=true; shift ;; --skip-openclaw) SKIP_OPENCLAW=true; shift ;; --skip-project-runtimes) SKIP_PROJECT_RUNTIMES=true; shift ;; --maintain-tools) MAINTAIN_TOOLS=true; shift ;; --update) UPDATE_MODE=true; shift ;; --reinstall) FORCE_FRESH=true; shift ;; --uninstall) UNINSTALL_MODE=true; shift ;; --repair-project-runtime-image) REPAIR_PROJECT_RUNTIME_IMAGE=true; shift ;; --residue-policy) [[ "${2:-}" == "safe" || "${2:-}" == "wipe" ]] \ || fail "--residue-policy requires 'safe' or 'wipe'." RESIDUE_POLICY="$2" shift 2 ;; --dry-run) DRY_RUN=true; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1"; usage; exit 1 ;; esac done if [[ "${ORIGIN_MODE}" == "tailnet" ]]; then [[ -z "${DOMAIN}" ]] || fail "--tailnet and --domain are mutually exclusive. Pick one origin." [[ "${INSTALL_PROFILE}" != "local" ]] || fail "--tailnet and --local are mutually exclusive." [[ "${TS_HOSTNAME}" =~ ^[A-Za-z0-9][A-Za-z0-9-]{0,62}$ ]] \ || fail "--ts-hostname may contain only letters, digits, and hyphens (max 63 characters)." if [[ -n "${TS_AUTHKEY}" && ! "${TS_AUTHKEY}" =~ ^tskey-[A-Za-z0-9_-]{5,200}$ ]]; then fail "--ts-authkey does not look like a Tailscale auth key (they start with tskey-)." fi fi local operation_count=0 $UPDATE_MODE && operation_count=$((operation_count + 1)) $FORCE_FRESH && operation_count=$((operation_count + 1)) $UNINSTALL_MODE && operation_count=$((operation_count + 1)) $REPAIR_PROJECT_RUNTIME_IMAGE && operation_count=$((operation_count + 1)) (( operation_count <= 1 )) \ || fail "--update, --reinstall, --uninstall, and --repair-project-runtime-image are mutually exclusive operations." } load_existing_origin_for_forced_reinstall() { $FORCE_FRESH || return 0 $IS_WSL && return 0 local existing_env="${1:-${PORTAL_DIR}/backend/.env.production}" [[ -f "${existing_env}" && ! -L "${existing_env}" ]] || return 0 local existing_profile existing_origin existing_domain existing_profile="$(read_env_value "${existing_env}" "INSTALL_PROFILE" || true)" existing_origin="$(read_env_value "${existing_env}" "ORIGIN_MODE" || true)" existing_domain="$(read_env_value "${existing_env}" "DOMAIN" || true)" if ! $ORIGIN_SELECTION_EXPLICIT; then case "${existing_profile}" in ""|server) INSTALL_PROFILE="server" ;; local) INSTALL_PROFILE="local" ;; *) fail "The existing Portal has an unknown INSTALL_PROFILE; choose --domain, --tailnet, or --local explicitly." ;; esac case "${existing_origin}" in "") ORIGIN_MODE="" ;; tailnet) ORIGIN_MODE="tailnet" ;; *) fail "The existing Portal has an unknown ORIGIN_MODE; choose --domain, --tailnet, or --local explicitly." ;; esac if use_local_profile && use_tailnet_profile; then fail "The existing Portal claims both local and Tailnet origin modes; choose --domain, --tailnet, or --local explicitly." fi if use_local_profile || use_tailnet_profile; then DOMAIN="" else DOMAIN="${existing_domain}" fi if use_tailnet_profile; then TAILNET_DNS_NAME="$(read_env_value "${existing_env}" "TAILNET_DNS_NAME" || true)" fi fi if ! use_local_profile && ! use_tailnet_profile && ! $APP_CONTENT_SELECTION_EXPLICIT; then local existing_app_domain existing_app_mode existing_app_origin existing_public_ip existing_app_domain="$(read_env_value "${existing_env}" "APP_CONTENT_DOMAIN" || true)" existing_app_mode="$(read_env_value "${existing_env}" "APP_CONTENT_DNS_MODE" || true)" existing_public_ip="$(read_env_value "${existing_env}" "PUBLIC_IP" || true)" if [[ -z "${existing_app_domain}" ]]; then existing_app_origin="$(read_env_value "${existing_env}" "APP_CONTENT_ORIGIN" || true)" existing_app_domain="$(app_content_domain_from_origin "${existing_app_origin}" || true)" fi case "${existing_app_mode}" in custom) [[ -n "${existing_app_domain}" ]] \ || fail "The existing Portal's custom app-content hostname is missing; choose --app-content-domain explicitly." APP_CONTENT_DOMAIN="${existing_app_domain}" APP_CONTENT_DNS_MODE="custom" ;; "") APP_CONTENT_DOMAIN="${existing_app_domain}" APP_CONTENT_DNS_MODE="" ;; sslip) if ! python3 - "${existing_public_ip}" "${existing_app_domain}" <<'PY2' import ipaddress import sys try: public_ip = str(ipaddress.IPv4Address(sys.argv[1])) except ValueError: raise SystemExit(1) expected = f"app-content.{public_ip}.sslip.io" raise SystemExit(0 if sys.argv[2].lower() == expected else 1) PY2 then fail "The existing Portal's automatic app-content hostname is inconsistent; choose --app-content-domain explicitly." fi # The automatic hostname embeds the public IP. Let normal identity # configuration regenerate it from the currently detected address. APP_CONTENT_DOMAIN="" APP_CONTENT_DNS_MODE="sslip" ;; local) [[ "${existing_profile}" == "local" && -z "${existing_origin}" ]] \ || fail "The existing Portal's app-content mode conflicts with its origin profile; choose --app-content-domain explicitly." # An explicit migration from a private profile to a public domain must # derive a new isolated public origin rather than reusing private state. APP_CONTENT_DOMAIN="" APP_CONTENT_DNS_MODE="" ;; tailnet-disabled) [[ "${existing_profile:-server}" == "server" && "${existing_origin}" == "tailnet" ]] \ || fail "The existing Portal's app-content mode conflicts with its origin profile; choose --app-content-domain explicitly." APP_CONTENT_DOMAIN="" APP_CONTENT_DNS_MODE="" ;; *) fail "The existing Portal has an unknown APP_CONTENT_DNS_MODE; choose --app-content-domain explicitly." ;; esac fi } validate_selected_origin() { if use_local_profile && use_tailnet_profile; then fail "Local and Tailnet origin modes are mutually exclusive." fi if use_local_profile && [[ -n "${DOMAIN}" ]]; then fail "--local and --domain are mutually exclusive. Pick one origin." fi if use_tailnet_profile && [[ -n "${DOMAIN}" ]]; then fail "--tailnet and --domain are mutually exclusive. Pick one origin." fi if $APP_CONTENT_SELECTION_EXPLICIT && use_local_profile; then fail "--app-content-domain is unavailable with --local." fi if $APP_CONTENT_SELECTION_EXPLICIT && use_tailnet_profile; then fail "--app-content-domain is unavailable with --tailnet." fi } # ═══════════════════════════════════════════════════════════════ # Step 1: Preflight # ═══════════════════════════════════════════════════════════════ # Retire the one known Portal-created all-images prune job and fail closed on # literal unsafe prune commands in effective installed root scheduler # definitions and their bounded, strict absolute helper-script chain. Opaque # executables, arbitrary shell variables, relative helpers, and external # program construction remain outside a static installer inspection. # The optional root argument is only for isolated installer fixtures. converge_unsafe_docker_prune_automation() { local host_root="${1:-/}" command -v python3 >/dev/null 2>&1 \ || { warn "Python 3 is required to inspect scheduled Docker cleanup jobs safely."; return 1; } python3 - \ "${host_root}" \ "${LEGACY_DOCKER_PRUNE_CRON_PATH}" \ "${LEGACY_DOCKER_PRUNE_QUARANTINE_PATH}" <<'PY2' import ctypes import errno import hashlib import os import re import shlex import stat import sys LIMIT = 1024 * 1024 HELPER_FILE_LIMIT = 128 * 1024 HELPER_TOTAL_LIMIT = 1024 * 1024 HELPER_FILE_COUNT_LIMIT = 96 HELPER_LINE_LIMIT = 16384 HELPER_DEPTH_LIMIT = 4 WRAPPER_DEPTH_LIMIT = 16 SCHEDULER_ENTRY_LIMIT = 16384 HELPER_CANDIDATE_FILE_LIMIT = 256 HELPER_CANDIDATE_TOTAL_LIMIT = 4 * 1024 * 1024 HELPER_CANDIDATE_LINE_LIMIT = 65536 PORTAL_AUDITED_HELPER_LIMIT = 512 * 1024 # Exact backup-full.sh payloads retained from installable Portal 4.0 releases. # installer/backup-helper-identities.tsv is the append-only, version-labelled # source ledger; the release gate validates every historical Git blob and # requires this set to match that ledger exactly. The backup service # legitimately schedules this larger audited entrypoint. Keep arbitrary # operator helpers on the generic 128 KiB per-file, 96-file, 1 MiB aggregate, # and 16,384 executable-line work bounds. PORTAL_BACKUP_HELPER_RELEASES = { (223518, "4988ba5be75e7e78e9c0f0981684c0a9203c7175f353eb07f890d337df8e1b1b"), (224956, "fc6751d6844589e081911e425c551afb317a3ded4b9d465b754dc7113c3c9454"), (227908, "a25a3d3562dfc2b51798916b893070f811e91074ef9c40430f296f22afd5a736"), (229463, "cba9826305c93de11f99699437937d6f7f89e1ae9869f415df903b047c5f2816"), (248156, "5b42a047fa07e3b5af2752b85f6d3e22e5944778c660799cf2eaade18f868143"), (263283, "337d15eb7289a63f77429aee7677ebb8c1aa4a037698111c591565a3496a1a85"), (268494, "384e21ade857380bdced2855671184f0a624338dcc20da3f21bea09cdc6178c9"), # 4.0.15 shipped a rewritten helper (degraded-backup preservation) without # adding it here, so 4.0.15's own guard rejected the helper 4.0.15 installs # and every host that reached 4.0.15 was blocked from updating again. # scripts/validation/backup-helper-allowlist-static.py now fails the build # when this set does not cover the helper the release actually ships. (271864, "c68881be5af61d0ba778b37fbea65068f650b74ab96703f15cafe73420b34f6c"), (288193, "04309ddec3ab7f2ae5c18d68a6e07b09809924f94a3f77b9694da90aa8c48305"), } PORTAL_RUNTIME_REPAIR_LAUNCHER = ( "/opt/bridgesllm/portal/installer/" "project-runtime-image-repair-launcher.py" ) PORTAL_RUNTIME_REPAIR_UNIT = ( "/run/systemd/transient/" "bridgesllm-project-runtime-image-repair.service" ) PORTAL_RUNTIME_REPAIR_LAUNCHER_CURRENT = (8641, "9bb962ad51725c2025d4813bb84c1887c115f47ca891134a5f3e70abfa8bc9ee") PORTAL_RUNTIME_REPAIR_LAUNCHER_RELEASES = { PORTAL_RUNTIME_REPAIR_LAUNCHER_CURRENT, } LEGACY = b'13 0 * * * root docker image prune -af --filter "until=24h"\n' SEPARATORS = {";", "&", "&&", "|", "||"} EXEC_KEYS = { "ExecCondition", "ExecReload", "ExecStart", "ExecStartPost", "ExecStartPre", "ExecStop", "ExecStopPost", } UNIT_SUFFIXES = { ".automount", ".path", ".scope", ".service", ".slice", ".socket", ".swap", ".target", ".timer", } SHELL_NAMES = {"bash", "dash", "ksh", "sh", "zsh"} PERL_NAMES = {"perl"} RUBY_NAMES = {"ruby"} NODE_NAMES = {"node", "nodejs"} BUILTIN_WRAPPERS = {"command", "exec"} SHELL_CONTROL_PREFIXES = {"!", "{", "coproc", "do", "elif", "else", "if", "then", "until", "while"} DOCKER_TRUE_VALUES = {"1", "t", "true"} DOCKER_FALSE_VALUES = {"0", "f", "false"} def fail(message): print(f"Unsafe Docker prune guard: {message}", file=sys.stderr) raise SystemExit(1) root = os.path.abspath(sys.argv[1]) source_logical, quarantine_logical = sys.argv[2:4] def host_path(logical): value = os.path.normpath(os.path.join(root, logical.lstrip("/"))) if root != "/" and value != root and not value.startswith(root + os.sep): fail(f"mapped path escapes the fixture root: {logical}") return value def display(path): if root == "/": return path relative = os.path.relpath(path, root) return "/" if relative == "." else "/" + relative def lstat_or_none(path): try: return os.lstat(path) except FileNotFoundError: return None except OSError as exc: fail(f"could not inspect {display(path)}: {exc}") def read_root_file(path, modes=None, single_link=False, require_root_group=False): before = lstat_or_none(path) if before is None: return None, None mode = stat.S_IMODE(before.st_mode) if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode): fail(f"{display(path)} is not a regular file; inspect it manually") if before.st_uid != 0 or (require_root_group and before.st_gid != 0): expected_owner = "root:root" if require_root_group else "root" fail(f"{display(path)} is not owned by {expected_owner}; inspect it manually") if single_link and before.st_nlink != 1: fail(f"{display(path)} has unexpected hard links; inspect it manually") if modes is not None and mode not in modes: expected = ", ".join(f"{item:04o}" for item in sorted(modes)) fail(f"{display(path)} has mode {mode:04o}, expected {expected}") if before.st_size > LIMIT: fail(f"{display(path)} exceeds the 1 MiB job inspection limit") flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) try: descriptor = os.open(path, flags) except OSError as exc: fail(f"could not open {display(path)} without following links: {exc}") try: opened = os.fstat(descriptor) before_id = ( before.st_dev, before.st_ino, before.st_mode, before.st_uid, before.st_gid, before.st_nlink, before.st_size, before.st_mtime_ns, before.st_ctime_ns, ) opened_id = ( opened.st_dev, opened.st_ino, opened.st_mode, opened.st_uid, opened.st_gid, opened.st_nlink, opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns, ) if opened_id != before_id: fail(f"{display(path)} changed during inspection") payload = b"" while len(payload) <= LIMIT: chunk = os.read(descriptor, min(65536, LIMIT + 1 - len(payload))) if not chunk: break payload += chunk if len(payload) > LIMIT: fail(f"{display(path)} exceeds the 1 MiB job inspection limit") after = os.fstat(descriptor) after_id = ( after.st_dev, after.st_ino, after.st_mode, after.st_uid, after.st_gid, after.st_nlink, after.st_size, after.st_mtime_ns, after.st_ctime_ns, ) if after_id != opened_id: fail(f"{display(path)} changed while it was read") return payload, before finally: os.close(descriptor) def require_secure_directory( path, metadata, context, *, allow_standard_crontab_spool=False, require_root_group=True, ): mode = stat.S_IMODE(metadata.st_mode) if ( allow_standard_crontab_spool and display(path) == "/var/spool/cron/crontabs" and stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_uid == 0 and mode == 0o1730 ): # Debian/Ubuntu deliberately use root:crontab 1730 here so the # setgid crontab helper can install per-user spools. Its sticky bit, # lack of world access, and root ownership are the supported secure # exception to the otherwise root:root/non-writable chain. return if ( not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != 0 or (require_root_group and metadata.st_gid != 0) or mode & 0o022 ): fail( f"{context} directory {display(path)} is linked, non-root-owned, " "or writable by another account" ) def resolve_secure_path( path, *, allow_missing=False, allow_mask=False, final_kind, require_root_group=True, require_root_group_directories=True, require_owner_readable=False, allow_standard_crontab_spool=False, ): normalized = os.path.normpath(path) if root != "/" and normalized != root and not normalized.startswith(root + os.sep): fail(f"secure path escapes the host root: {path}") relative = os.path.relpath(normalized, root) pending = [] if relative == "." else relative.split(os.sep) current = root root_stat = lstat_or_none(root) if root_stat is None: fail("host root disappeared during scheduler inspection") require_secure_directory( root, root_stat, "scheduler", allow_standard_crontab_spool=allow_standard_crontab_spool, ) symlink_count = 0 visited_links = set() required_link_target_components = 0 while pending: component = pending.pop(0) if component in {"", "."}: continue if component == "..": fail(f"secure path escapes the host root: {path}") candidate = os.path.join(current, component) metadata = lstat_or_none(candidate) if metadata is None: if allow_missing and required_link_target_components == 0: return None, False fail(f"scheduled path {display(candidate)} is missing or a link target is broken") if stat.S_ISLNK(metadata.st_mode): if metadata.st_uid != 0 or metadata.st_gid != 0: fail(f"scheduled path link {display(candidate)} is not owned by root:root") symlink_count += 1 if symlink_count > 32: fail(f"scheduled path {display(path)} has too many link indirections") link_identity = (metadata.st_dev, metadata.st_ino) if link_identity in visited_links: fail(f"scheduled path {display(path)} contains a link cycle") visited_links.add(link_identity) try: link_target = os.readlink(candidate) except OSError as exc: fail(f"could not read scheduled path link {display(candidate)}: {exc}") target = ( host_path(link_target) if os.path.isabs(link_target) else os.path.normpath(os.path.join(os.path.dirname(candidate), link_target)) ) if root != "/" and target != root and not target.startswith(root + os.sep): fail(f"scheduled path link {display(candidate)} escapes the host root") if allow_mask and not pending and target == host_path("/dev/null"): return None, True target_relative = os.path.relpath(target, root) target_parts = [] if target_relative == "." else target_relative.split(os.sep) required_link_target_components = ( len(target_parts) + max(required_link_target_components - 1, 0) ) pending = target_parts + pending current = root continue if required_link_target_components > 0: required_link_target_components -= 1 if pending: require_secure_directory( candidate, metadata, "scheduler", allow_standard_crontab_spool=allow_standard_crontab_spool, require_root_group=require_root_group_directories, ) current = candidate metadata = lstat_or_none(current) if metadata is None: if allow_missing and symlink_count == 0: return None, False fail(f"scheduled path {display(current)} disappeared during inspection") if final_kind == "directory": require_secure_directory( current, metadata, "scheduler", allow_standard_crontab_spool=allow_standard_crontab_spool, require_root_group=require_root_group_directories, ) elif final_kind == "file": if ( not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0 or (require_root_group and metadata.st_gid != 0) or stat.S_IMODE(metadata.st_mode) & 0o022 or (require_owner_readable and not metadata.st_mode & stat.S_IRUSR) ): if not require_owner_readable: fail( f"scheduled definition or helper {display(current)} is non-regular, " "non-root-owned, or writable by another account" ) fail( f"scheduled definition or helper {display(current)} is non-regular, " "unreadable, non-root-owned, or writable by another account" ) elif final_kind == "either": if stat.S_ISDIR(metadata.st_mode): require_secure_directory( current, metadata, "scheduler", allow_standard_crontab_spool=allow_standard_crontab_spool, require_root_group=require_root_group_directories, ) elif ( not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0 or (require_root_group and metadata.st_gid != 0) or stat.S_IMODE(metadata.st_mode) & 0o022 or (require_owner_readable and not metadata.st_mode & stat.S_IRUSR) ): if not require_owner_readable: fail( f"scheduled definition or helper {display(current)} is non-regular, " "non-root-owned, or writable by another account" ) fail( f"scheduled definition or helper {display(current)} is non-regular, " "unreadable, non-root-owned, or writable by another account" ) else: fail(f"unsupported secure-path kind: {final_kind}") return current, False def tokens(value): try: lexer = shlex.shlex(value, posix=True, punctuation_chars=";&|") lexer.whitespace_split = True lexer.commenters = "#" return list(lexer) except ValueError: return re.findall(r"&&|\|\||[;&|]|[^\s;&|]+", value) def word(value): cleaned = value.strip("$(){}[]<>,\"'") prefix_match = re.match(r"^[-+!:@]+", cleaned) if prefix_match: prefix = prefix_match.group(0) return prefix + os.path.basename(cleaned[len(prefix):]) return os.path.basename(cleaned) def is_known_legacy_payload(payload): """Recognize the retired Portal cron job plus inert output redirects.""" if b"\0" in payload: return False try: text = payload.decode("utf-8") except UnicodeDecodeError: return False stripped = text.strip() if not stripped or len(stripped.splitlines()) != 1: return False fields = stripped.split(None, 6) if len(fields) != 7 or fields[:6] != ["13", "0", "*", "*", "*", "root"]: return False try: lexer = shlex.shlex(fields[6], posix=True, punctuation_chars="><;&|") lexer.whitespace_split = True lexer.commenters = "" command = list(lexer) except ValueError: return False base = [ "docker", "image", "prune", "-af", "--filter", "until=24h", ] if command[:len(base)] != base: return False suffix = command[len(base):] seen_stdout, seen_stderr = False, False while suffix: if suffix[:2] == [">", "/dev/null"]: if seen_stdout: return False seen_stdout = True suffix = suffix[2:] continue if suffix[:3] == ["1", ">", "/dev/null"]: if seen_stdout: return False seen_stdout = True suffix = suffix[3:] continue if suffix[:3] == ["2", ">&", "1"]: if seen_stderr: return False seen_stderr = True suffix = suffix[3:] continue return False return True def systemd_effective_argv(arguments): argv = list(arguments) if not argv: return argv prefix_match = re.match(r"^[-+!:@]+", argv[0]) prefix = prefix_match.group(0) if prefix_match else "" executable = argv[0][len(prefix):] if prefix else argv[0] if not executable: return [] if "@" not in prefix: return [executable, *argv[1:]] # systemd's @ executable prefix consumes the next word as the overridden # argv[0]. It is not part of the called program's argument vector. The # prefix has no such meaning in cron's ordinary shell grammar. if len(argv) < 2: return [] return [executable, *argv[2:]] def command_segments(value): result = [] start = 0 index = 0 quote = None end = len(value) while index < len(value): character = value[index] if quote == "'": if character == "'": quote = None index += 1 continue if quote == '"': if character == "\\": index += 2 continue if character == '"': quote = None index += 1 continue if character == "\\": index += 2 continue if character in {"'", '"'}: quote = character index += 1 continue if quote is None and character == "#" and ( index == 0 or value[index - 1].isspace() or value[index - 1] in ";&|()" ): end = index break separator_length = 0 if quote is None and character in {"(", ")"} and not ( character == "(" and index > 0 and value[index - 1] == "$" ): separator_length = 1 elif quote is None and character in {";", "\n"}: separator_length = 1 elif quote is None and character in {"&", "|"}: if ( (index > 0 and value[index - 1] in {">", "<"}) or ( character == "&" and index + 1 < len(value) and value[index + 1] == ">" ) ): index += 1 continue separator_length = ( 2 if index + 1 < len(value) and value[index + 1] in {character, "&"} else 1 ) if separator_length: segment = value[start:index].strip() if segment: result.append(tokens(segment)) index += separator_length start = index continue index += 1 segment = value[start:end].strip() if segment: result.append(tokens(segment)) return result def systemd_command_segments(value): # systemd supports multiple Exec*= commands only when a semicolon appears # as its own unquoted, unescaped word. Pipes, ampersands, substitutions, # and escaped/quoted semicolons are ordinary argv data, not shell syntax. result = [] start = 0 index = 0 quote = None while index < len(value): character = value[index] if character == "\\": index += 2 continue if quote is not None: if character == quote: quote = None index += 1 continue if character in {"'", '"'}: quote = character index += 1 continue if character == ";" and ( index == 0 or value[index - 1].isspace() ) and ( index + 1 == len(value) or value[index + 1].isspace() ): segment = value[start:index].strip() if segment: result.append(tokens(segment)) start = index + 1 index += 1 segment = value[start:].strip() if segment: result.append(tokens(segment)) return result def shell_segment_plan(arguments): argv = [] stdin_source = None raw = list(arguments) index = 0 redirection = re.compile( r"^(?:(?P[0-9]+))?" r"(?P<<<|<<|<>|>>|>|<)(?P.*)$" ) while index < len(raw): item = raw[index] # shlex separates the `&` in &> and descriptor duplication. Keep # shell redirections out of the executed argv without treating the # duplication ampersand as a command-list separator. if item == "&" and index + 1 < len(raw): following = redirection.fullmatch(raw[index + 1]) if following and following.group("operator").startswith(">"): item = raw[index + 1] index += 1 else: argv.append(item) index += 1 continue match = redirection.fullmatch(item) if match: descriptor = match.group("descriptor") operator = match.group("operator") suffix = match.group("suffix") target = suffix descriptor_duplication = target.startswith("&") if descriptor_duplication: target = target[1:] index += 1 if not suffix and index < len(raw): if raw[index] in {"&", "|"}: descriptor_duplication = raw[index] == "&" index += 1 if index < len(raw): target = raw[index] index += 1 else: target = raw[index] index += 1 effective_descriptor = ( int(descriptor) if descriptor is not None else (0 if operator.startswith("<") else 1) ) if effective_descriptor == 0 and operator.startswith("<"): if operator == "<<<": stdin_source = ( ("here-string", target) if target != "" else ("here-string", "") ) elif operator == "<<": # The body of a here-document is outside this bounded # single-command tokenizer. A stdin-driven shell must not # be admitted on an unknowable command source. stdin_source = ("opaque", "here-document") elif descriptor_duplication: stdin_source = ( ("closed", None) if target == "-" else ("opaque", "descriptor duplication") ) elif target: stdin_source = ("file", target) else: stdin_source = ("opaque", "missing redirection target") continue argv.append(item) index += 1 while argv and argv[0] in SHELL_CONTROL_PREFIXES: argv.pop(0) return argv, stdin_source def shell_segment_argv(arguments): return shell_segment_plan(arguments)[0] def shell_invocation_plan(arguments, stdin_source): """Resolve whether a shell reads -c, a script, or its effective stdin.""" index = 0 noexec = False stdin_mode = False command = None terminal = False while index < len(arguments): item = arguments[index] if item == "--": index += 1 break if item in {"--help", "--version"}: terminal = True break if item == "--noexec": noexec = True index += 1 continue if item in {"-O", "+O", "-o", "+o", "--init-file", "--rcfile"}: if index + 1 >= len(arguments): terminal = True break option_value = arguments[index + 1] if item in {"-o", "+o"} and option_value == "noexec": noexec = item == "-o" index += 2 continue if item.startswith(("-", "+")) and item not in {"-", "+"}: sign = item[0] cluster = item[1:] if not cluster.isalpha(): # An unmodelled shell option can change how the remaining # argv is interpreted. Reject an stdin-driven invocation # later rather than guessing that it is harmless. return { "kind": "opaque", "value": "unsupported shell option", "noexec": noexec, } if "n" in cluster: noexec = sign == "-" if "s" in cluster: stdin_mode = sign == "-" if "c" in cluster: if index + 1 >= len(arguments): terminal = True else: command = arguments[index + 1] index = len(arguments) break index += 1 continue break if noexec or terminal: return {"kind": "nonexecuting", "value": None, "noexec": noexec} if command is not None: return {"kind": "command", "value": command, "noexec": False} if not stdin_mode and index < len(arguments) and arguments[index] != "-": return {"kind": "script", "value": arguments[index], "noexec": False} if stdin_source is None or stdin_source[0] == "closed": return {"kind": "empty-stdin", "value": None, "noexec": False} return {"kind": stdin_source[0], "value": stdin_source[1], "noexec": False} def env_command_argv(arguments): index = 0 while index < len(arguments): item = arguments[index] if item == "--": index += 1 break if item in {"-u", "--unset", "-C", "--chdir"}: if index + 1 >= len(arguments): return [] index += 2 continue split_value = None split_tail = [] if item in {"-S", "--split-string"}: if index + 1 >= len(arguments): return [] split_value = arguments[index + 1] split_tail = arguments[index + 2:] elif item.startswith("-S") and item != "-S": split_value = item[2:] split_tail = arguments[index + 1:] elif item.startswith("--split-string="): split_value = item.split("=", 1)[1] split_tail = arguments[index + 1:] if split_value is not None: try: split_arguments = shlex.split( split_value, comments=False, posix=True ) except ValueError: # GNU env rejects an unterminated split string before it can # execute a command. Treat that invocation as non-executing. return [] return split_arguments + list(split_tail) if item.startswith("-") or re.fullmatch( r"[A-Za-z_][A-Za-z0-9_]*=.*", item ): index += 1 continue break return list(arguments[index:]) def docker_all_value(argument): if argument == "--all": return True if argument.startswith("--all="): explicit = argument.split("=", 1)[1].strip().lower() if explicit in DOCKER_TRUE_VALUES: return True if explicit in DOCKER_FALSE_VALUES: return False return None if not argument.startswith("-") or argument.startswith("--"): return None shorthand = argument[1:] flags, separator, explicit = shorthand.partition("=") if "a" not in flags: return None # pflag evaluates a compact Boolean shorthand from left to right. An # explicit value belongs to the final shorthand; preceding `a` flags have # already taken their no-option default of true. if not separator or flags.index("a") < len(flags) - 1: return True explicit = explicit.strip().lower() if explicit in DOCKER_TRUE_VALUES: return True if explicit in DOCKER_FALSE_VALUES: return False return None def docker_prune_kind(arguments): index = 0 global_value_options = { "-H", "--config", "--context", "--host", "-l", "--log-level", "--tlscacert", "--tlscert", "--tlskey", } global_boolean_options = {"-D", "--debug", "--tls", "--tlsverify"} global_nonexecuting_options = {"-h", "--help", "-v", "--version"} while index < len(arguments): argument = arguments[index] if argument == "--": index += 1 break if argument in global_nonexecuting_options: return None if argument in global_value_options: if index + 1 >= len(arguments): return None index += 2 continue if any(argument.startswith(option + "=") for option in global_value_options): index += 1 continue if ( (argument.startswith("-H") and argument != "-H") or (argument.startswith("-l") and argument != "-l") ): index += 1 continue if argument in global_boolean_options: index += 1 continue if any(argument.startswith(option + "=") for option in global_boolean_options): explicit = argument.split("=", 1)[1].strip().lower() if explicit not in (DOCKER_TRUE_VALUES | DOCKER_FALSE_VALUES): return None index += 1 continue if argument.startswith("-"): return None break if index + 1 >= len(arguments): return None kind = word(arguments[index]) if kind not in {"image", "system"} or word(arguments[index + 1]) != "prune": return None index += 2 all_enabled = False while index < len(arguments): argument = arguments[index] if argument == "--": if index + 1 < len(arguments): return None break if argument == "--filter": if index + 1 >= len(arguments): return None index += 2 continue if argument.startswith("--filter="): index += 1 continue if argument == "--all" or argument.startswith("--all="): all_value = docker_all_value(argument) if all_value is None: return None all_enabled = all_value index += 1 continue if argument in {"--force", "-f"}: index += 1 continue if argument.startswith("--force="): if argument.split("=", 1)[1].strip().lower() not in ( DOCKER_TRUE_VALUES | DOCKER_FALSE_VALUES ): return None index += 1 continue if kind == "system" and argument == "--volumes": index += 1 continue if kind == "system" and argument.startswith("--volumes="): if argument.split("=", 1)[1].strip().lower() not in ( DOCKER_TRUE_VALUES | DOCKER_FALSE_VALUES ): return None index += 1 continue if argument.startswith("-") and not argument.startswith("--"): shorthand, separator, explicit = argument[1:].partition("=") if not shorthand or any(flag not in {"a", "f"} for flag in shorthand): return None if separator and explicit.strip().lower() not in ( DOCKER_TRUE_VALUES | DOCKER_FALSE_VALUES ): return None if "a" in shorthand: all_value = docker_all_value(argument) if all_value is None: return None all_enabled = all_value index += 1 continue return None return kind if all_enabled else None def shell_command_substitutions(value): """Return statically delimited substitutions that a shell will execute.""" substitutions = [] index = 0 quote = None while index < len(value): character = value[index] if quote == "'": if character == "'": quote = None index += 1 continue if character == "\\": if ( quote == '"' and index + 1 < len(value) and value[index + 1] not in {'$', '`', '"', "\\", "\n"} ): index += 1 else: index += 2 continue if quote is None and character == "'": quote = "'" index += 1 continue if character == '"': quote = None if quote == '"' else '"' index += 1 continue if quote is None and character == "#" and ( index == 0 or value[index - 1].isspace() or value[index - 1] in ";&|()" ): break if character == "`": end = index + 1 while end < len(value): if value[end] == "\\": end += 2 continue if value[end] == "`": substitutions.append(value[index + 1:end]) index = end + 1 break end += 1 else: return substitutions continue if character == "$" and index + 1 < len(value) and value[index + 1] == "(": start = index + 2 end = start depth = 1 inner_quote = None while end < len(value): inner = value[end] if inner_quote == "'": if inner == "'": inner_quote = None end += 1 continue if inner == "\\": if ( inner_quote == '"' and end + 1 < len(value) and value[end + 1] not in {'$', '`', '"', "\\", "\n"} ): end += 1 else: end += 2 continue if inner_quote is None and inner == "'": inner_quote = "'" end += 1 continue if inner == '"': inner_quote = None if inner_quote == '"' else '"' end += 1 continue if inner_quote is None and inner == "(": depth += 1 elif inner_quote is None and inner == ")": depth -= 1 if depth == 0: substitutions.append(value[start:end]) index = end + 1 break end += 1 else: return substitutions continue index += 1 return substitutions def language_execution_command_records(value, kind): # These APIs execute one literal string through a command shell. Arbitrary # strings, assignments, logging calls, and dynamically constructed values # are intentionally not interpreted as executable commands. Scan the # complete bounded source rather than one physical line: each supported # language permits whitespace (including newlines) between the call, # opening parenthesis, and literal argument. call_names = { "python": (("os.system",), False, "#"), "perl": (("system", "exec"), True, "#"), "ruby": (("system", "exec"), True, "#"), "node": (("execSync", "exec"), False, "//"), } configuration = call_names.get(kind) if configuration is None: return [] names, optional_parenthesis, comment = configuration commands = [] index = 0 quote = None while index < len(value): character = value[index] if quote is not None: if character == "\\": index += 2 continue if character == quote: quote = None index += 1 continue if character in ({"'", '"', "`"} if kind == "node" else {"'", '"'}): quote = character index += 1 continue if value.startswith(comment, index): newline = value.find("\n", index + len(comment)) if newline < 0: break index = newline + 1 continue matched = next( ( name for name in sorted(names, key=len, reverse=True) if value.startswith(name, index) and (index == 0 or not (value[index - 1].isalnum() or value[index - 1] == "_")) and ( index + len(name) == len(value) or not ( value[index + len(name)].isalnum() or value[index + len(name)] == "_" ) ) ), None, ) if matched is None: index += 1 continue call_line = value.count("\n", 0, index) + 1 cursor = index + len(matched) while cursor < len(value) and value[cursor].isspace(): cursor += 1 if cursor < len(value) and value[cursor] == "(": cursor += 1 while cursor < len(value) and value[cursor].isspace(): cursor += 1 elif not optional_parenthesis: index += len(matched) continue argument_quotes = ( {"'", '"', "`"} if kind == "node" else {"'", '"'} ) if cursor >= len(value) or value[cursor] not in argument_quotes: index += len(matched) continue argument_quote = value[cursor] cursor += 1 argument = [] template_interpolation = False while cursor < len(value): if value[cursor] == "\\" and cursor + 1 < len(value): argument.extend(value[cursor:cursor + 2]) cursor += 2 continue if argument_quote == "`" and value.startswith("${", cursor): template_interpolation = True if value[cursor] == argument_quote: command = "".join(argument) lowered_command = command.lower() if ( template_interpolation and "docker" in lowered_command and "prune" in lowered_command ): fail( "scheduled Node execution template contains " "interpolation and a Docker prune candidate that " "cannot be inspected safely" ) if not template_interpolation: commands.append((command, call_line)) cursor += 1 break argument.append(value[cursor]) cursor += 1 index = max(cursor, index + len(matched)) return commands def language_execution_commands(value, kind): return [ command for command, _ in language_execution_command_records(value, kind) ] def dangerous_argv(arguments, depth, stdin_source=None): if depth > WRAPPER_DEPTH_LIMIT: fail( f"scheduled command wrapper inspection exceeded depth " f"{WRAPPER_DEPTH_LIMIT}" ) argv = list(arguments) while argv and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", argv[0]): argv.pop(0) if not argv: return None name = word(argv[0]) rest = argv[1:] if name == "docker": return docker_prune_kind(rest) if name in BUILTIN_WRAPPERS: if name == "command" and any(item in {"-v", "-V"} for item in rest): return None index = 0 while index < len(rest) and rest[index].startswith("-"): if rest[index] == "--": index += 1 break if name == "exec" and rest[index] == "-a" and index + 1 < len(rest): index += 2 else: index += 1 return dangerous_argv(rest[index:], depth + 1, stdin_source) if name == "eval": return dangerous(" ".join(rest), shell_syntax=True, depth=depth + 1) if name == "env": return dangerous_argv(env_command_argv(rest), depth + 1, stdin_source) if name == "timeout": index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-k", "--kill-after", "-s", "--signal"}: index += 2 continue if item.startswith("-"): index += 1 continue break return ( dangerous_argv(rest[index + 1:], depth + 1, stdin_source) if index < len(rest) else None ) if name == "flock": for index, item in enumerate(rest): if item in {"-c", "--command"} and index + 1 < len(rest): return dangerous(rest[index + 1], shell_syntax=True, depth=depth + 1) index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-w", "--wait", "-E", "--conflict-exit-code"}: index += 2 continue if item.startswith("-"): index += 1 continue break return ( dangerous_argv(rest[index + 1:], depth + 1, stdin_source) if index < len(rest) else None ) if name == "nice": index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-n", "--adjustment"}: index += 2 continue if re.fullmatch(r"-\d+", item) or item.startswith("--adjustment="): index += 1 continue if item.startswith("-"): index += 1 continue break return dangerous_argv(rest[index:], depth + 1, stdin_source) if name == "time": index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-f", "--format", "-o", "--output"}: index += 2 continue if item.startswith("-"): index += 1 continue break return dangerous_argv(rest[index:], depth + 1, stdin_source) if name == "nohup": return dangerous_argv( rest[1:] if rest[:1] == ["--"] else rest, depth + 1, stdin_source, ) if name == "sudo": index = 0 value_options = { "-C", "--close-from", "-D", "--chdir", "-g", "--group", "-h", "--host", "-p", "--prompt", "-R", "--chroot", "-T", "--command-timeout", "-u", "--user", } while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in value_options: index += 2 continue if item.startswith("-") or re.fullmatch( r"[A-Za-z_][A-Za-z0-9_]*=.*", item ): index += 1 continue break return dangerous_argv(rest[index:], depth + 1, stdin_source) kind = interpreter_kind(name) if kind == "shell": plan = shell_invocation_plan(rest, stdin_source) if plan["kind"] == "command": return dangerous(plan["value"], shell_syntax=True, depth=depth + 1) if plan["kind"] == "here-string": return dangerous(plan["value"], shell_syntax=True, depth=depth + 1) if plan["kind"] == "opaque": if plan["value"] == "here-document": # A containing shell helper supplies and inspects this body # with its literal delimiter below. return None fail( "scheduled stdin-driven shell uses a command source that " f"cannot be inspected safely ({plan['value']})" ) return None if kind in {"python", "perl", "ruby", "node"}: plan = interpreter_invocation(kind, rest) for source in plan["inline"]: for command in language_execution_commands(source, kind): finding = dangerous(command, shell_syntax=True, depth=depth + 1) if finding: return finding return None return None def dangerous(value, *, shell_syntax=False, systemd_syntax=False, depth=0): parts = tokens(value) if isinstance(value, str) else list(value) if isinstance(value, str) and shell_syntax: segments = command_segments(value) elif isinstance(value, str) and systemd_syntax: segments = systemd_command_segments(value) else: segments = [parts] for segment in segments: stdin_source = None if shell_syntax: segment, stdin_source = shell_segment_plan(segment) elif systemd_syntax: segment = systemd_effective_argv(segment) finding = dangerous_argv(segment, depth, stdin_source) if finding: return finding if shell_syntax and isinstance(value, str): for substitution in shell_command_substitutions(value): finding = dangerous( substitution, shell_syntax=True, depth=depth + 1 ) if finding: return finding return None def cron_commands(payload, kind): text = payload.decode("utf-8", errors="replace").replace("\\\n", " ") for number, raw in enumerate(text.splitlines(), 1): stripped = raw.strip() if not stripped or stripped.startswith("#"): continue if kind == "run-parts": yield number, raw continue if re.match(r"^[A-Za-z_][A-Za-z0-9_]*\s*=", stripped): continue if kind == "anacron": fields = stripped.split(None, 3) if len(fields) == 4: yield number, fields[3] continue if kind == "root-spool": if stripped.startswith("@"): fields = stripped.split(None, 1) if len(fields) == 2: yield number, fields[1] else: fields = stripped.split(None, 5) if len(fields) == 6: yield number, fields[5] continue if stripped.startswith("@"): fields = stripped.split(None, 2) if len(fields) == 3 and fields[1] == "root": yield number, fields[2] else: fields = stripped.split(None, 6) if len(fields) == 7 and fields[5] == "root": yield number, fields[6] def systemd_service_directives(payload): text = payload.decode("utf-8", errors="replace").replace("\\\n", " ") section = None for number, raw in enumerate(text.splitlines(), 1): stripped = raw.strip() if not stripped or stripped.startswith(("#", ";")): continue if stripped.startswith("[") and stripped.endswith("]"): section = stripped[1:-1].strip() continue if section != "Service" or "=" not in stripped: continue key, command = stripped.split("=", 1) yield number, key.strip(), command def systemd_directives(payload): for number, key, command in systemd_service_directives(payload): if key in EXEC_KEYS: yield number, key, command def systemd_unescaped_reference(value): simple = { "a": "\a", "b": "\b", "f": "\f", "n": "\n", "r": "\r", "s": " ", "t": "\t", "v": "\v", "\\": "\\", '"': '"', "'": "'", } result = [] index = 0 while index < len(value): if value[index] != "\\" or index + 1 >= len(value): result.append(value[index]) index += 1 continue marker = value[index + 1] if marker in simple: result.append(simple[marker]) index += 2 continue widths = {"x": 2, "u": 4, "U": 8} width = widths.get(marker) if width is not None: encoded = value[index + 2:index + 2 + width] if len(encoded) == width and re.fullmatch(r"[0-9A-Fa-f]+", encoded): try: result.append(chr(int(encoded, 16))) index += 2 + width continue except ValueError: pass octal = re.match(r"[0-7]{1,3}", value[index + 1:]) if octal is not None: result.append(chr(int(octal.group(0), 8))) index += 1 + len(octal.group(0)) continue result.extend(("\\", marker)) index += 2 return "".join(result) def secure_cron_file(path, *, allow_missing, periodic, allow_crontab_spool): resolved, masked = resolve_secure_path( path, allow_missing=allow_missing, allow_mask=False, final_kind="file", require_root_group=False, require_owner_readable=True, allow_standard_crontab_spool=allow_crontab_spool, ) if masked or resolved is None: return None metadata = lstat_or_none(resolved) if metadata is None: fail(f"cron scheduler target {display(resolved)} disappeared") if periodic and not stat.S_IMODE(metadata.st_mode) & 0o111: return None return resolved def bounded_directory_entries(directory, entry_budget, context): entries = [] try: with os.scandir(directory) as iterator: for entry in iterator: entry_budget[0] += 1 if entry_budget[0] > SCHEDULER_ENTRY_LIMIT: fail( f"scheduler inspection exceeded its " f"{SCHEDULER_ENTRY_LIMIT}-entry aggregate budget" ) entries.append(entry) except OSError as exc: fail(f"could not enumerate {context} {display(directory)}: {exc}") entries.sort(key=lambda entry: entry.name) return entries def is_unit_name_activation_link(relative, path, metadata): parts = relative.split(os.sep) if ( len(parts) != 2 or not parts[0].endswith((".wants", ".requires")) or os.path.splitext(parts[1])[1] not in UNIT_SUFFIXES or not stat.S_ISLNK(metadata.st_mode) ): return False if metadata.st_uid != 0 or metadata.st_gid != 0: fail(f"scheduled path link {display(path)} is not owned by root:root") try: target = os.readlink(path) except OSError as exc: fail(f"could not read scheduled path link {display(path)}: {exc}") current = lstat_or_none(path) if current is None: fail(f"systemd scheduler entry {display(path)} disappeared") before_identity = ( metadata.st_dev, metadata.st_ino, metadata.st_mode, metadata.st_uid, metadata.st_gid, metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, ) current_identity = ( current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_gid, current.st_nlink, current.st_size, current.st_mtime_ns, current.st_ctime_ns, ) if current_identity != before_identity: fail(f"systemd scheduler entry {display(path)} changed during inspection") # systemd permits .wants/.requires dependency links to name a unit in the # global load path even when the relative filesystem target is dangling. # Defer only the canonical same-name form; a concrete definition elsewhere # in the load path must still win, and a missing definition remains fatal. return target == f"../{parts[1]}" def is_systemd_loadable_directory(relative): parts = relative.split(os.sep) return len(parts) == 1 and parts[0].endswith( (".d", ".wants", ".requires") ) def is_systemd_loadable_leaf(relative): parts = relative.split(os.sep) basename = parts[-1] suffix = os.path.splitext(basename)[1] if len(parts) == 1: return suffix in UNIT_SUFFIXES if len(parts) != 2: return False if parts[0].endswith((".wants", ".requires")): return suffix in UNIT_SUFFIXES return parts[0].endswith(".d") and basename.endswith(".conf") def secure_cron_directory(logical, kind, entry_budget, seen_directories=None): lexical_root = host_path(logical) resolved_root, masked = resolve_secure_path( lexical_root, allow_missing=True, allow_mask=False, final_kind="directory", ) if masked or resolved_root is None: return root_metadata = lstat_or_none(resolved_root) if root_metadata is None: fail(f"cron scheduler root {display(resolved_root)} disappeared") root_identity = (root_metadata.st_dev, root_metadata.st_ino) if seen_directories is not None: if root_identity in seen_directories: return seen_directories.add(root_identity) entries = bounded_directory_entries( resolved_root, entry_budget, "cron scheduler root" ) for entry in entries: lexical = os.path.join(lexical_root, entry.name) resolved, entry_masked = resolve_secure_path( lexical, allow_missing=False, allow_mask=False, final_kind="either", require_root_group=False, require_owner_readable=True, ) if entry_masked or resolved is None: continue metadata = lstat_or_none(resolved) if metadata is None: fail(f"cron scheduler entry {display(resolved)} disappeared") if stat.S_ISDIR(metadata.st_mode): # cron.d and run-parts do not recurse into subdirectories. The # directory was still securely resolved and attested above. continue if kind == "run-parts" and not stat.S_IMODE(metadata.st_mode) & 0o111: continue yield lexical, kind, resolved def collect_systemd_root(logical, resolved_root, entry_budget): records = [] pending = [(resolved_root, "")] visited = set() while pending: directory, relative_prefix = pending.pop() metadata = lstat_or_none(directory) if metadata is None: fail(f"systemd scheduler directory {display(directory)} disappeared") require_secure_directory(directory, metadata, "systemd scheduler") visit_key = (metadata.st_dev, metadata.st_ino, relative_prefix) if visit_key in visited: continue visited.add(visit_key) entries = bounded_directory_entries( directory, entry_budget, "systemd scheduler directory" ) for entry in entries: path = entry.path relative = os.path.join(relative_prefix, entry.name) if relative_prefix else entry.name try: entry_metadata = os.lstat(path) except OSError as exc: fail(f"could not inspect systemd scheduler entry {display(path)}: {exc}") if stat.S_ISDIR(entry_metadata.st_mode): if not is_systemd_loadable_directory(relative): continue require_secure_directory(path, entry_metadata, "systemd scheduler") pending.append((path, relative)) continue if not ( is_systemd_loadable_leaf(relative) or is_systemd_loadable_directory(relative) ): # systemd ignores backups, editor temporaries, and arbitrary # nested files in a unit load path. Do not attest content that # the scheduler itself cannot load. continue if is_unit_name_activation_link(relative, path, entry_metadata): records.append((relative, path, None, False)) continue resolved, entry_masked = resolve_secure_path( path, allow_missing=False, allow_mask=True, final_kind="either", ) if entry_masked: records.append((relative, path, None, True)) continue if resolved is None: fail(f"systemd scheduler entry {display(path)} disappeared") resolved_metadata = lstat_or_none(resolved) if resolved_metadata is None: fail(f"systemd scheduler target {display(resolved)} disappeared") if stat.S_ISDIR(resolved_metadata.st_mode): if not is_systemd_loadable_directory(relative): continue pending.append((resolved, relative)) continue records.append((relative, path, resolved, False)) return records def systemd_dropin_owners(unit_name): stem, suffix = os.path.splitext(unit_name) owners = [unit_name] if "@" in stem: owners.append(stem.split("@", 1)[0] + "@" + suffix) prefix = stem while "-" in prefix: prefix = prefix.rsplit("-", 1)[0] owners.append(prefix + "-" + suffix) owners.append(suffix.lstrip(".")) return list(dict.fromkeys(owners)) def locally_managed_scheduler_path(path): return path.startswith(("/etc/", "/run/", "/root/", "/usr/local/")) def effective_systemd_definitions(logical_roots, entry_budget): selected_main = {} activation_fallback = {} selected_dropins = {} seen_roots = set() for logical in logical_roots: lexical_root = host_path(logical) resolved_root, masked = resolve_secure_path( lexical_root, allow_missing=True, allow_mask=True, final_kind="directory", ) if masked: fail(f"systemd scheduler root {logical} cannot be masked") if resolved_root is None: continue root_metadata = lstat_or_none(resolved_root) if root_metadata is None: fail(f"systemd scheduler root {logical} disappeared") root_identity = (root_metadata.st_dev, root_metadata.st_ino) if root_identity in seen_roots: continue seen_roots.add(root_identity) for relative, definition, target, entry_masked in collect_systemd_root( logical, resolved_root, entry_budget ): parts = relative.split(os.sep) basename = parts[-1] suffix = os.path.splitext(basename)[1] if len(parts) == 1 and suffix in UNIT_SUFFIXES: selected_main.setdefault( basename, (definition, target, entry_masked) ) continue if ( len(parts) == 2 and parts[0].endswith((".wants", ".requires")) and suffix in UNIT_SUFFIXES ): activation_fallback.setdefault( basename, (definition, target, entry_masked) ) continue if ( len(parts) == 2 and parts[0].endswith(".d") and basename.endswith(".conf") ): owner = parts[0][:-2] selected_dropins.setdefault( (owner, basename), (definition, target, entry_masked) ) for unit_name, fallback in activation_fallback.items(): selected_main.setdefault(unit_name, fallback) def definition_priority(item): unit_name, (definition, _, _) = item origin = display(definition) locally_managed = locally_managed_scheduler_path(origin) return (0 if locally_managed else 1, unit_name) # Inspect administrator/runtime definitions before vendor definitions so # ordinary distro units cannot consume generic helper work ahead of the # scheduler surface most likely to contain local policy. for unit_name, selected_record in sorted( selected_main.items(), key=definition_priority ): definition, target, masked = selected_record if masked: continue if target is None: fail(f"effective systemd definition {display(definition)} has no target") definitions = [(definition, target)] owner_rank = { owner: rank for rank, owner in enumerate(systemd_dropin_owners(unit_name)) } effective_dropins = {} for (owner, basename), record in selected_dropins.items(): if owner not in owner_rank: continue previous = effective_dropins.get(basename) if previous is None or owner_rank[owner] < previous[0]: effective_dropins[basename] = (owner_rank[owner], record) for basename in sorted(effective_dropins): _, (dropin, dropin_target, dropin_masked) = effective_dropins[basename] if dropin_masked: continue if dropin_target is None: fail(f"effective systemd drop-in {display(dropin)} has no target") definitions.append((dropin, dropin_target)) yield unit_name, definitions def jobs(scheduler_entry_budget, scheduler_directories): for logical, kind in ( ("/etc/crontab", "system-cron"), ("/etc/anacrontab", "anacron"), ("/var/spool/cron/crontabs/root", "root-spool"), ("/var/spool/cron/root", "root-spool"), ): lexical = host_path(logical) target = secure_cron_file( lexical, allow_missing=True, periodic=False, allow_crontab_spool=kind == "root-spool", ) if target is not None: yield lexical, kind, target for logical, kind in ( ("/etc/cron.d", "system-cron"), ("/etc/cron.hourly", "run-parts"), ("/etc/cron.daily", "run-parts"), ("/etc/cron.weekly", "run-parts"), ("/etc/cron.monthly", "run-parts"), ("/etc/cron.yearly", "run-parts"), ): yield from secure_cron_directory( logical, kind, scheduler_entry_budget, scheduler_directories ) system_roots = ( "/etc/systemd/system.control", "/run/systemd/system.control", "/run/systemd/transient", "/run/systemd/generator.early", "/etc/systemd/system", "/etc/systemd/system.attached", "/run/systemd/system", "/run/systemd/system.attached", "/run/systemd/generator", "/usr/local/lib/systemd/system", "/usr/lib/systemd/system", "/lib/systemd/system", "/run/systemd/generator.late", ) root_user_roots = ( "/root/.config/systemd/user.control", "/run/user/0/systemd/user.control", "/run/user/0/systemd/transient", "/run/user/0/systemd/generator.early", "/root/.config/systemd/user", "/etc/xdg/systemd/user", "/etc/systemd/user", "/run/user/0/systemd/user", "/run/systemd/user", "/root/.local/share/systemd/user", "/usr/local/share/systemd/user", "/usr/share/systemd/user", "/usr/local/lib/systemd/user", "/usr/lib/systemd/user", "/run/user/0/systemd/generator", "/run/user/0/systemd/generator.late", ) for roots in (system_roots, root_user_roots): for unit_name, definitions in effective_systemd_definitions( roots, scheduler_entry_budget ): yield unit_name, "systemd", definitions def is_audited_portal_backup_helper(logical_path, path, metadata): expected_path = "/opt/bridgesllm/portal/backup-full.sh" if ( os.path.normpath(logical_path) != expected_path or display(path) != expected_path ): return False if metadata.st_size > PORTAL_AUDITED_HELPER_LIMIT: fail( "scheduled Portal backup helper exceeds its 512 KiB audited " "release limit" ) if metadata.st_size not in { release_size for release_size, _ in PORTAL_BACKUP_HELPER_RELEASES }: fail( "scheduled Portal backup helper does not match a known shipped " "BridgesLLM release; restore it through a signed update before " "retrying" ) payload, current = read_root_file( path, single_link=True, require_root_group=True, ) expected_identity = ( metadata.st_dev, metadata.st_ino, metadata.st_mode, metadata.st_uid, metadata.st_gid, metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, ) current_identity = ( current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_gid, current.st_nlink, current.st_size, current.st_mtime_ns, current.st_ctime_ns, ) if current_identity != expected_identity: fail( "scheduled Portal backup helper changed before release " "attestation" ) digest = hashlib.sha256(payload).hexdigest() if ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and root != "/" and os.environ.get("BRIDGESLLM_DOCKER_PRUNE_TEST_HOOK", "") == "portal-helper-post-read-replacement" ): replacement = path + ".test-replacement" displaced = path + ".test-old" if os.path.lexists(displaced) or not os.path.isfile(replacement): fail("Portal helper replacement fixture is incomplete") os.rename(path, displaced) os.rename(replacement, path) final = lstat_or_none(path) if final is None: fail("scheduled Portal backup helper disappeared after attestation") final_identity = ( final.st_dev, final.st_ino, final.st_mode, final.st_uid, final.st_gid, final.st_nlink, final.st_size, final.st_mtime_ns, final.st_ctime_ns, ) if final_identity != expected_identity: fail( "scheduled Portal backup helper changed during release " "attestation" ) if (metadata.st_size, digest) not in PORTAL_BACKUP_HELPER_RELEASES: fail( "scheduled Portal backup helper does not match a known shipped " "BridgesLLM release; restore it through a signed update before " "retrying" ) return True def is_audited_portal_runtime_repair_launcher(logical_path, path, metadata): if ( os.path.normpath(logical_path) != PORTAL_RUNTIME_REPAIR_LAUNCHER or display(path) != PORTAL_RUNTIME_REPAIR_LAUNCHER ): return False strict_path, strict_masked = resolve_secure_path( host_path(PORTAL_RUNTIME_REPAIR_LAUNCHER), allow_missing=False, allow_mask=False, final_kind="file", require_root_group=True, require_root_group_directories=True, ) if strict_masked or strict_path != path: fail("scheduled Project runtime repair launcher path is unsafe") if metadata.st_size not in { release_size for release_size, _ in PORTAL_RUNTIME_REPAIR_LAUNCHER_RELEASES }: fail( "scheduled Project runtime repair launcher does not match a " "known shipped BridgesLLM release" ) payload, current = read_root_file( path, modes={0o600, 0o644}, single_link=True, require_root_group=True, ) expected_identity = ( metadata.st_dev, metadata.st_ino, metadata.st_mode, metadata.st_uid, metadata.st_gid, metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, ) current_identity = ( current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_gid, current.st_nlink, current.st_size, current.st_mtime_ns, current.st_ctime_ns, ) if current_identity != expected_identity: fail("scheduled Project runtime repair launcher changed before attestation") digest = hashlib.sha256(payload).hexdigest() final = lstat_or_none(path) if final is None: fail("scheduled Project runtime repair launcher disappeared after attestation") final_identity = ( final.st_dev, final.st_ino, final.st_mode, final.st_uid, final.st_gid, final.st_nlink, final.st_size, final.st_mtime_ns, final.st_ctime_ns, ) if final_identity != expected_identity: fail("scheduled Project runtime repair launcher changed during attestation") if (metadata.st_size, digest) not in PORTAL_RUNTIME_REPAIR_LAUNCHER_RELEASES: fail( "scheduled Project runtime repair launcher does not match a " "known shipped BridgesLLM release" ) return True class HelperBudget: def __init__(self, scheduler_entries, scheduler_directories): self.files = 0 self.bytes = 0 self.lines = 0 self.candidate_files = 0 self.candidate_bytes = 0 self.candidate_lines = 0 self.inspected = set() self.warnings = set() self.scheduler_entries = scheduler_entries self.scheduler_directories = scheduler_directories def warn_skip(self, key, message): if key not in self.warnings: print(f"Unsafe Docker prune guard warning: {message}", file=sys.stderr) self.warnings.add(key) def admit(self, logical_path, path, metadata, candidate=False): identity = (metadata.st_dev, metadata.st_ino) is_runtime_repair_launcher = ( os.path.normpath(logical_path) == PORTAL_RUNTIME_REPAIR_LAUNCHER ) if is_runtime_repair_launcher: is_audited_portal_runtime_repair_launcher( logical_path, path, metadata ) if identity in self.inspected: return False if ( metadata.st_size > HELPER_FILE_LIMIT and is_audited_portal_backup_helper( logical_path, path, metadata ) ): # This exact release helper is always attested, even after generic # helper work has reached a soft ceiling. Its exact bytes are # reviewed and shipped by BridgesLLM, so no line parsing follows. self.inspected.add(identity) self.files += 1 self.bytes += metadata.st_size return False if metadata.st_size > HELPER_FILE_LIMIT: fail(f"scheduled helper {display(path)} exceeds 128 KiB") generic_exhausted = ( self.files + 1 > HELPER_FILE_COUNT_LIMIT or self.bytes + metadata.st_size > HELPER_TOTAL_LIMIT ) if candidate and generic_exhausted: if self.files + 1 > HELPER_FILE_COUNT_LIMIT: self.warn_skip( "files", "scheduled helper inspection reached its 96-file work " "budget; prioritized helper candidates continue in the " "bounded reserve", ) else: self.warn_skip( "bytes", "scheduled helper inspection reached its 1 MiB aggregate " "budget; prioritized helper candidates continue in the " "bounded reserve", ) if self.candidate_files + 1 > HELPER_CANDIDATE_FILE_LIMIT: fail( "scheduled helper candidate inspection exceeded its " "256-file reserve" ) if ( self.candidate_bytes + metadata.st_size > HELPER_CANDIDATE_TOTAL_LIMIT ): fail( "scheduled helper candidate inspection exceeded its " "4 MiB reserve" ) self.inspected.add(identity) self.candidate_files += 1 self.candidate_bytes += metadata.st_size return True if self.files + 1 > HELPER_FILE_COUNT_LIMIT: self.warn_skip( "files", "scheduled helper inspection reached its 96-file work budget; " "remaining helper bodies are skipped while literal scheduler " "commands remain inspected", ) return False if self.bytes + metadata.st_size > HELPER_TOTAL_LIMIT: self.warn_skip( "bytes", "scheduled helper inspection reached its 1 MiB aggregate " "budget; remaining helper bodies are skipped while literal " "scheduler commands remain inspected", ) return False if is_runtime_repair_launcher: self.inspected.add(identity) self.files += 1 self.bytes += metadata.st_size return False self.inspected.add(identity) self.files += 1 self.bytes += metadata.st_size return True def admit_line(self, candidate=False): if self.lines >= HELPER_LINE_LIMIT: if candidate: self.warn_skip( "lines", "scheduled helper inspection reached its 16384-line work " "budget; prioritized helper candidates continue in the " "bounded reserve", ) self.candidate_lines += 1 if self.candidate_lines > HELPER_CANDIDATE_LINE_LIMIT: fail( "scheduled helper candidate inspection exceeded its " "65536-line reserve" ) return True self.warn_skip( "lines", "scheduled helper inspection reached its 16384-line work " "budget; remaining helper bodies are skipped while literal " "scheduler commands remain inspected", ) return False self.lines += 1 return True def absolute_token(value): cleaned = value.strip("$(){}[]<>,\"'") return cleaned if cleaned.startswith("/") else None def helper_prune_candidate(payload): # This is only a priority signal, never a finding: false positives spend a # small reserve and are parsed normally. Keeping the scan byte-oriented # also surfaces ASCII shell commands inside otherwise non-UTF-8 scripts. lowered = payload.lower() return b"docker" in lowered and b"prune" in lowered def shell_text(payload): try: text = payload.decode("utf-8") except UnicodeDecodeError: text = "".join( chr(value) if value in {9, 10, 13} or 32 <= value <= 126 else " " for value in payload ) return "".join( character if character in "\t\r\n" or character.isprintable() else " " for character in text ) def interpreter_kind(name): if name in SHELL_NAMES: return "shell" if re.fullmatch(r"python(?:\d+(?:\.\d+)*)?", name): return "python" if name in PERL_NAMES: return "perl" if name in RUBY_NAMES: return "ruby" if name in NODE_NAMES: return "node" return None def interpreter_inline_argument(kind, arguments, index): argument = arguments[index] following = arguments[index + 1] if index + 1 < len(arguments) else None match = None if kind == "python": # Python permits no-value short switches to be clustered before -c, # including the common `-BcCODE` form. match = re.fullmatch(r"-[bBdEiIOPqRsSuvx]*c(.*)", argument) elif kind == "perl": # Perl accepts attached -e/-E code and clusters such as -weCODE. # Its -0 and -l switches can also carry an attached octal value before # a later execution switch (for example `-0777eCODE`). Parse that # cluster deterministically: a nested-regex spelling permits # catastrophic backtracking on a long, safe nonmatch. if argument.startswith("-") and not argument.startswith("--"): body = argument[1:] cursor = 0 simple = set("wWTtnpsSUdD") while cursor < len(body): option = body[cursor] if option in {"e", "E"}: source = body[cursor + 1:] if source: return source, 1 return (following, 2) if following is not None else (None, 1) if option in simple: cursor += 1 continue if option == "0": cursor += 1 octal_digits = 0 while ( cursor < len(body) and octal_digits < 3 and body[cursor] in "01234567" ): cursor += 1 octal_digits += 1 continue if option == "l": cursor += 1 octal_digits = 0 while ( cursor < len(body) and octal_digits < 4 and body[cursor] in "01234567" ): cursor += 1 octal_digits += 1 continue break return None elif kind == "ruby": # Ruby accepts attached -e code and no-value clusters such as -weCODE. match = re.fullmatch(r"-[wWvdlnpsa]*e(.*)", argument) elif kind == "node": if argument.startswith("--eval="): return argument.split("=", 1)[1], 1 if argument in {"-e", "-p", "--eval", "--print"}: return (following, 2) if following is not None else (None, 1) if re.fullmatch(r"-[ep].+", argument): # Shipped Node rejects attached short-form source (`-eCODE` and # `-pCODE`) before it can execute a main script. return None, 1 if match is None: return None attached = match.group(1) if attached: return attached, 1 return (following, 2) if following is not None else (None, 1) def interpreter_invocation(kind, arguments): inline = [] preloads = [] node_print_mode = False index = 0 while index < len(arguments): argument = arguments[index] if argument == "--": index += 1 break if kind == "node" and argument.startswith("--print="): # Node treats the equals suffix as an ignored value for the print # switch. The first later non-option is the expression, if any; # with no such argument this invocation prints `undefined` and # does not execute the suffix. node_print_mode = True index += 1 continue if kind == "node" and ( re.fullmatch(r"-r.+", argument) or ( argument.startswith("--require") and argument != "--require" and not argument.startswith("--require=") ) or ( argument.startswith("--import") and argument != "--import" and not argument.startswith("--import=") ) ): # These attached spellings are rejected by shipped Node before a # main script can run. Only -r VALUE, --require[=]VALUE, and # --import[=]VALUE are executable preload contracts. return { "inline": inline, "preloads": preloads, "script": None, "terminal": True, } parsed_inline = interpreter_inline_argument(kind, arguments, index) if parsed_inline is not None: source, consumed = parsed_inline if source is None: return { "inline": inline, "preloads": preloads, "script": None, "terminal": True, } inline.append(source) index += consumed if kind in {"python", "node"}: return { "inline": inline, "preloads": preloads, "script": None, "terminal": True, } continue if kind == "python" and ( argument == "-m" or (argument.startswith("-m") and argument != "-m") ): return { "inline": inline, "preloads": preloads, "script": None, "terminal": True, } if argument.startswith("-"): preload_value = None preload_consumed = 1 if kind in {"node", "ruby"}: preload_options = ( {"-r", "--require", "--import"} if kind == "node" else {"-r", "--require"} ) if argument in preload_options: if index + 1 >= len(arguments): return { "inline": inline, "preloads": preloads, "script": None, "terminal": True, } preload_value = arguments[index + 1] preload_consumed = 2 elif ( argument.startswith("--require=") or (kind == "node" and argument.startswith("--import=")) ): preload_value = argument.split("=", 1)[1] elif ( kind == "ruby" and argument.startswith("-r") and argument != "-r" ): preload_value = argument[2:] if preload_value is not None: preload = absolute_token(preload_value) if preload is not None: preloads.append(preload) index += preload_consumed continue value_options = { "python": {"-W", "-X", "--check-hash-based-pycs"}, "perl": {"-I", "-M", "-m"}, "ruby": {"-I", "-r"}, "node": {"-r", "--require", "--import"}, }.get(kind, set()) if argument in value_options: if index + 1 >= len(arguments): return { "inline": inline, "preloads": preloads, "script": None, "terminal": True, } index += 2 continue if any( argument.startswith(option + "=") for option in value_options if option.startswith("--") ): index += 1 continue if any( argument.startswith(option) and argument != option for option in value_options if option.startswith("-") and not option.startswith("--") ): index += 1 continue index += 1 continue break if kind == "node" and node_print_mode: if index < len(arguments): inline.append(arguments[index]) return { "inline": inline, "preloads": preloads, "script": None, "terminal": True, } script = absolute_token(arguments[index]) if index < len(arguments) else None return { "inline": inline, "preloads": preloads, "script": script, "terminal": bool(inline), } def recurse_argv(arguments, depth, direct_invocation, stdin_source=None): if depth > WRAPPER_DEPTH_LIMIT: fail( f"scheduled command wrapper inspection exceeded depth " f"{WRAPPER_DEPTH_LIMIT}" ) return helper_candidates_from_argv( arguments, depth, direct_invocation, stdin_source ) def interpreter_candidates( kind, arguments, depth, direct_invocation, stdin_source=None ): if kind == "shell": plan = shell_invocation_plan(arguments, stdin_source) if plan["kind"] in {"command", "here-string"}: return helper_candidates( plan["value"], depth + 1, "shell-fallback" ) if plan["kind"] == "script": script = absolute_token(plan["value"]) if script is None: fail( "scheduled shell script path is not a literal absolute path" ) return [(script, "interpreter:shell")] if plan["kind"] == "file": script = absolute_token(plan["value"]) if script is None: fail( "scheduled shell stdin path is not a literal absolute path" ) return [(script, "shell-source")] if plan["kind"] == "opaque": if plan["value"] == "here-document": return [] fail( "scheduled stdin-driven shell uses a command source that " f"cannot be inspected safely ({plan['value']})" ) return [] plan = interpreter_invocation(kind, arguments) candidates = [ (preload, f"interpreter:{kind}") for preload in plan["preloads"] ] for source in plan["inline"]: for command in language_execution_commands(source, kind): candidates.extend( helper_candidates(command, depth + 1, "shell-fallback") ) if plan["terminal"]: return candidates if plan["script"]: candidates.append((plan["script"], f"interpreter:{kind}")) return candidates def run_parts_directory(arguments): index = 0 value_options = {"--arg", "--regex", "--umask"} nonexecuting_options = {"--help", "--list", "--test", "--version"} options_end = ( arguments.index("--") if "--" in arguments else len(arguments) ) if any(argument in nonexecuting_options for argument in arguments[:options_end]): return None while index < len(arguments): argument = arguments[index] if argument == "--": index += 1 break if argument in value_options: index += 2 continue if any(argument.startswith(option + "=") for option in value_options): index += 1 continue if argument.startswith("-"): index += 1 continue break if index >= len(arguments): return None return absolute_token(arguments[index]) def helper_candidates_from_argv( arguments, depth, direct_invocation, stdin_source=None ): if depth > WRAPPER_DEPTH_LIMIT: fail( f"scheduled command wrapper inspection exceeded depth " f"{WRAPPER_DEPTH_LIMIT}" ) argv = list(arguments) while argv and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", argv[0]): argv.pop(0) if not argv: return [] if any(PORTAL_RUNTIME_REPAIR_LAUNCHER in item for item in argv): if ( direct_invocation != "systemd" or len(argv) != 6 or argv[:4] != [ "/usr/bin/python3", "-I", "-S", PORTAL_RUNTIME_REPAIR_LAUNCHER, ] or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", argv[4]) is None or re.fullmatch( r"/opt/bridgesllm/logs/" r"project-runtime-image-repair-[A-Za-z0-9_.:-]+\.log", argv[5], ) is None ): fail("scheduled Project runtime repair launcher command is malformed") return [ (PORTAL_RUNTIME_REPAIR_LAUNCHER, "portal-runtime-repair-launcher") ] name = word(argv[0]) rest = argv[1:] if direct_invocation == "shell-fallback" and argv[0] in {".", "source"}: # Shell source builtins execute only their first positional operand; # later values become the sourced file's positional arguments. Limit # recursion to a literal absolute file so dynamic PATH lookup remains # outside this bounded static inspection. sourced = absolute_token(rest[0]) if rest else None return [(sourced, "shell-source")] if sourced is not None else [] if name in BUILTIN_WRAPPERS: if name == "command" and any(item in {"-v", "-V"} for item in rest): return [] index = 0 while index < len(rest) and rest[index].startswith("-"): if rest[index] == "--": index += 1 break if name == "exec" and rest[index] == "-a" and index + 1 < len(rest): index += 2 else: index += 1 return recurse_argv( rest[index:], depth + 1, direct_invocation, stdin_source ) if name == "env": return recurse_argv( env_command_argv(rest), depth + 1, "shell-fallback", stdin_source ) if name == "time": index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-f", "--format", "-o", "--output"}: index += 2 continue if item.startswith("-"): index += 1 continue break return recurse_argv( rest[index:], depth + 1, "shell-fallback", stdin_source ) if name == "timeout": index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-k", "--kill-after", "-s", "--signal"}: index += 2 continue if item.startswith("-"): index += 1 continue break if index >= len(rest): return [] return recurse_argv( rest[index + 1:], depth + 1, "shell-fallback", stdin_source ) if name == "flock": for index, item in enumerate(rest): if item in {"-c", "--command"} and index + 1 < len(rest): return helper_candidates( rest[index + 1], depth + 1, "shell-fallback" ) index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-w", "--wait", "-E", "--conflict-exit-code"}: index += 2 continue if item.startswith("-"): index += 1 continue break if index >= len(rest): return [] return recurse_argv( rest[index + 1:], depth + 1, "shell-fallback", stdin_source ) if name == "nice": index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-n", "--adjustment"}: index += 2 continue if re.fullmatch(r"-\d+", item) or item.startswith("--adjustment="): index += 1 continue if item.startswith("-"): index += 1 continue break return recurse_argv( rest[index:], depth + 1, "shell-fallback", stdin_source ) if name == "nohup": return recurse_argv( rest[1:] if rest[:1] == ["--"] else rest, depth + 1, "shell-fallback", stdin_source, ) if name == "sudo": index = 0 value_options = { "-C", "--close-from", "-D", "--chdir", "-g", "--group", "-h", "--host", "-p", "--prompt", "-R", "--chroot", "-T", "--command-timeout", "-u", "--user", } while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in value_options: index += 2 continue if item.startswith("-") or re.fullmatch( r"[A-Za-z_][A-Za-z0-9_]*=.*", item ): index += 1 continue break return recurse_argv( rest[index:], depth + 1, "shell-fallback", stdin_source ) kind = interpreter_kind(name) if kind: return interpreter_candidates( kind, rest, depth, direct_invocation, stdin_source ) if name == "run-parts": directory = run_parts_directory(rest) candidates = [] executable = absolute_token(argv[0]) if executable is not None: candidates.append((executable, direct_invocation)) if directory is not None: candidates.append((directory, "run-parts")) return candidates executable = absolute_token(argv[0]) if executable is None: return [] return [(executable, direct_invocation)] def helper_candidates(command, depth=0, direct_invocation="direct"): candidates = [] if direct_invocation == "shell-fallback": segments = command_segments(command) elif direct_invocation == "systemd": segments = systemd_command_segments(command) else: segments = [tokens(command)] for segment in segments: stdin_source = None if direct_invocation == "shell-fallback": segment, stdin_source = shell_segment_plan(segment) elif direct_invocation == "systemd": segment = systemd_effective_argv(segment) candidates.extend( helper_candidates_from_argv( segment, depth, direct_invocation, stdin_source ) ) return candidates def read_root_prefix(path, expected, maximum=4096): flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) try: descriptor = os.open(path, flags) except OSError as exc: fail(f"could not open scheduled helper {display(path)} safely: {exc}") try: opened = os.fstat(descriptor) if ( opened.st_dev, opened.st_ino, opened.st_mode, opened.st_uid, opened.st_gid, opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns, ) != ( expected.st_dev, expected.st_ino, expected.st_mode, expected.st_uid, expected.st_gid, expected.st_size, expected.st_mtime_ns, expected.st_ctime_ns, ): fail(f"scheduled helper {display(path)} changed before type inspection") payload = os.read(descriptor, maximum) after = os.fstat(descriptor) if ( after.st_dev, after.st_ino, after.st_mode, after.st_uid, after.st_gid, after.st_size, after.st_mtime_ns, after.st_ctime_ns, ) != ( opened.st_dev, opened.st_ino, opened.st_mode, opened.st_uid, opened.st_gid, opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns, ): fail(f"scheduled helper {display(path)} changed during type inspection") return payload finally: os.close(descriptor) def supported_shebang_invocation(payload): first_line = payload.splitlines()[0] if payload.splitlines() else b"" if not first_line.startswith(b"#!"): return None try: declaration = first_line[2:].decode("utf-8").strip() except UnicodeDecodeError: return None if not declaration: return None # Linux passes everything after the interpreter pathname as one optional # argv item. Quotes and spaces in that tail are not shell syntax. GNU env # is the deliberate exception: -S/--split-string parses its tail into the # final interpreter argv. shebang_match = re.fullmatch(r"(\S+)(?:[ \t]+(.*))?", declaration) if shebang_match is None: return None interpreter = shebang_match.group(1) optional_tail = shebang_match.group(2) arguments = [optional_tail] if optional_tail else [] name = word(interpreter) if name == "env": remaining = env_command_argv(arguments) if not remaining: return None name = word(remaining[0]) arguments = remaining else: arguments = [interpreter, *arguments] kind = interpreter_kind(name) return (kind, arguments[1:]) if kind is not None else None def shell_heredoc_lex(command): """Mask arithmetic shifts and describe each real unquoted heredoc operator.""" result = [] masked = list(command) mask_marker = "\ue000" while mask_marker in command: mask_marker += "\ue001" quote = None escaped = False arithmetic_kind = None arithmetic_depth = 0 index = 0 while index < len(command): character = command[index] if escaped: if character == "<": # The shell treats this byte as literal. Mask it in the shlex # input too; otherwise `\\<<` or `\\<\\<` can be reassembled # into a false heredoc token even though the operator counter # correctly ignored the escaped byte. masked[index] = mask_marker escaped = False index += 1 continue if quote == "'": if character == "<": masked[index] = mask_marker if character == "'": quote = None index += 1 continue if quote == '"': if character == "<": masked[index] = mask_marker if character == "\\": escaped = True elif character == '"': quote = None index += 1 continue if character == "\\": escaped = True index += 1 continue if character in "'\"": quote = character index += 1 continue if arithmetic_kind is not None: if command[index:index + 2] == "<<": # shlex otherwise exposes a Bash arithmetic left shift as the # same punctuation token used by a here-document. Preserve # token width while keeping it out of the heredoc stream. masked[index:index + 2] = [mask_marker, mask_marker] index += 2 continue if arithmetic_kind == "paren": if character == "(": arithmetic_depth += 1 elif character == ")": arithmetic_depth -= 1 if arithmetic_depth == 0: arithmetic_kind = None elif character == "[": arithmetic_depth += 1 elif character == "]": arithmetic_depth -= 1 if arithmetic_depth == 0: arithmetic_kind = None index += 1 continue if character == "#" and ( index == 0 or command[index - 1].isspace() or command[index - 1] in ";|&" ): break if command[index:index + 3] == "$((": arithmetic_kind = "paren" arithmetic_depth = 2 index += 3 continue if command[index:index + 2] == "((": arithmetic_kind = "paren" arithmetic_depth = 2 index += 2 continue if command[index:index + 2] == "$[": arithmetic_kind = "bracket" arithmetic_depth = 1 index += 2 continue if ( command[index:index + 2] == "<<" and (index == 0 or command[index - 1] != "<") and command[index:index + 3] != "<<<" ): result.append(command[index + 2:index + 3] == "-") index += 2 continue index += 1 return "".join(masked), result, mask_marker def shell_heredoc_tab_operators(command): """Record whether each real heredoc operator has a tab-strip dash.""" return shell_heredoc_lex(command)[1] def shell_heredoc_specs(command): """Return ordered delimiters with the command that receives each body.""" masked_command, tab_operators, mask_marker = shell_heredoc_lex(command) try: # Non-POSIX mode retains quoting long enough to distinguish a quoted # delimiter (literal body) from an unquoted one (shell substitutions # still execute before the receiving interpreter reads the body). lexer = shlex.shlex(masked_command, posix=False, punctuation_chars="<;&|") lexer.whitespace_split = True lexer.commenters = "#" parts = list(lexer) except ValueError: return [] heredoc_index = 0 result = [] segment = [] index = 0 while index < len(parts): if parts[index] in SEPARATORS: segment = [] index += 1 continue if parts[index] != "<<" or index + 1 >= len(parts): segment.append(parts[index]) index += 1 continue if heredoc_index >= len(tab_operators): return [] strip_tabs = tab_operators[heredoc_index] heredoc_index += 1 delimiter_index = index + 1 delimiter = parts[delimiter_index] if strip_tabs and delimiter == "-": # shlex separates the valid spaced form `<<- 'EOF'` into # `<<`, `-`, and the delimiter word. Consume the operator dash; # it is never itself the terminator. delimiter_index += 1 if delimiter_index >= len(parts): segment.append(parts[index]) index += 1 continue delimiter = parts[delimiter_index] strip_tabs = True elif strip_tabs and delimiter.startswith("-"): delimiter = delimiter[1:] # Non-operator less-than bytes remain masked in every ordinary token, # but a delimiter is a shell word whose exact quoted/escaped bytes are # semantically significant. Restore only this proven delimiter token; # restoring other tokens could recreate a false `<<` punctuation item. delimiter = delimiter.replace(mask_marker, "<") quoted = any(character in delimiter for character in "'\"\\") try: unquoted = shlex.split(delimiter, comments=False, posix=True) except ValueError: unquoted = [] if len(unquoted) == 1 and unquoted[0]: try: receiver_argv = shlex.split( " ".join(segment), comments=False, posix=True ) except ValueError: receiver_argv = [] result.append( ( unquoted[0], strip_tabs, not quoted, wrapped_interpreter_kind(receiver_argv), ) ) index = delimiter_index + 1 return result def wrapped_interpreter_kind(arguments, depth=0): if depth > WRAPPER_DEPTH_LIMIT: return None argv, _ = shell_segment_plan(arguments) while argv and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", argv[0]): argv.pop(0) if not argv: return None name = word(argv[0]) rest = argv[1:] kind = interpreter_kind(name) if kind is not None: return kind if name == "env": return wrapped_interpreter_kind(env_command_argv(rest), depth + 1) if name in BUILTIN_WRAPPERS: index = 0 while index < len(rest) and rest[index].startswith("-"): if rest[index] == "--": index += 1 break if name == "exec" and rest[index] == "-a" and index + 1 < len(rest): index += 2 else: index += 1 return wrapped_interpreter_kind(rest[index:], depth + 1) if name == "timeout": index = 0 while index < len(rest): item = rest[index] if item == "--": index += 1 break if item in {"-k", "--kill-after", "-s", "--signal"}: index += 2 continue if item.startswith("-"): index += 1 continue break return ( wrapped_interpreter_kind(rest[index + 1:], depth + 1) if index < len(rest) else None ) return None def inspect_helper(path_logical, invocation, budget, depth, priority=False): if depth > HELPER_DEPTH_LIMIT: fail(f"scheduled helper recursion exceeded depth {HELPER_DEPTH_LIMIT}") lexical = host_path(path_logical) resolved, masked = resolve_secure_path( lexical, allow_missing=True, allow_mask=False, final_kind="either", # A dedicated service may need read/traverse access to its immutable # root-owned helper. Group identity is not mutation authority when no # traversed directory or final file grants group/world write. Keep # scheduler roots and definitions on the stricter root:root path. require_root_group=False, require_root_group_directories=False, ) if masked or resolved is None: # A missing command cannot execute. If it appears later, a subsequent # installer run will inspect it like every other absolute helper. return None metadata = lstat_or_none(resolved) if metadata is None: fail(f"scheduled helper {path_logical} disappeared") if stat.S_ISDIR(metadata.st_mode): # A directory-shaped command token (including a bare `/`) cannot be # executed as a helper and is not evidence of unsafe cleanup. return None prefix = read_root_prefix( resolved, metadata, min(metadata.st_size, HELPER_FILE_LIMIT) + 1, ) prune_candidate = priority or helper_prune_candidate(prefix) has_shebang = prefix.startswith(b"#!") shebang = supported_shebang_invocation(prefix) shebang_kind = shebang[0] if shebang is not None else None if invocation in {"direct", "systemd"} and shebang_kind is None: # A directly executed custom program is a statically inspectable # script only when its kernel interpreter contract is explicit. ELF, # other binaries, and non-shebang text executables remain opaque. return None if invocation == "shell-fallback" and shebang_kind is None: # Cron invokes its command through a shell, while the supported process # wrappers use execvp-style ENOEXEC fallback. In both cases an # executable non-ELF file without a shebang is interpreted by /bin/sh. # Shell source bytes need not be UTF-8, so preserve ASCII commands and # sanitize other bytes later instead of treating encoding as fatal. if ( has_shebang or not metadata.st_mode & 0o111 or prefix.startswith(b"\x7fELF") ): return None if not budget.admit( path_logical, resolved, metadata, candidate=prune_candidate ): return None payload, _ = read_root_file(resolved, require_root_group=False) if invocation == "shell-source": # The current shell interprets sourced content. A leading shebang is # merely a comment and cannot change the source language. language = "shell" elif invocation.startswith("interpreter:"): language = invocation.split(":", 1)[1] elif shebang_kind is not None: language = shebang_kind else: language = "shell" if b"\0" in payload: if language == "shell": # Bash ignores embedded NUL bytes with a warning. Remove them for # static inspection so the remaining executed shell text is still # checked for a literal unsafe prune. payload = payload.replace(b"\0", b"") elif not prune_candidate: return None if language == "shell": text = shell_text(payload) else: try: text = payload.decode("utf-8") except UnicodeDecodeError: if not prune_candidate: return None # Known interpreter source containing the byte-level Docker/prune # candidate cannot become opaque because of one irrelevant byte. # Preserve its ASCII source and line layout while replacing bytes # whose language semantics cannot be reduced safely. text = "".join( chr(value) if value in {9, 10, 13} or 32 <= value <= 126 else " " for value in payload ) if not all( character in "\t\r\n" or character.isprintable() for character in text ): if not prune_candidate: return None text = "".join( chr(value) if value in {9, 10, 13} or 32 <= value <= 126 else " " for value in payload ) def inspect_literal_commands(commands, line_number): for command in commands: prune_kind = dangerous(command, shell_syntax=True) if prune_kind: return prune_kind, display(resolved), line_number for nested, nested_invocation in helper_candidates( command, direct_invocation="shell-fallback" ): if nested_invocation == "run-parts": finding = inspect_run_parts_directory( nested, budget, depth + 1, priority=prune_candidate ) else: finding = inspect_helper( nested, nested_invocation, budget, depth + 1, priority=prune_candidate, ) if finding: return finding return None if shebang is not None and shebang_kind in {"python", "perl", "ruby", "node"}: shebang_plan = interpreter_invocation( shebang_kind, [*shebang[1], display(resolved)] ) for preload in shebang_plan["preloads"]: finding = inspect_helper( preload, f"interpreter:{shebang_kind}", budget, depth + 1, priority=prune_candidate, ) if finding: return finding for source in shebang_plan["inline"]: if not budget.admit_line(candidate=prune_candidate): return None finding = inspect_literal_commands( language_execution_commands(source, shebang_kind), 1 ) if finding: return finding if shebang_plan["terminal"]: return None logical_text = text.replace("\\\n", " ") if language != "shell": for raw in logical_text.splitlines(): stripped = raw.strip() if ( not stripped or stripped.startswith("#") or (language == "node" and stripped.startswith("//")) ): continue if not budget.admit_line(candidate=prune_candidate): return None for command, line_number in language_execution_command_records( logical_text, language ): finding = inspect_literal_commands([command], line_number) if finding: return finding return None heredocs = [] heredoc_body = [] heredoc_body_start = None for line_number, raw in enumerate(logical_text.splitlines(), 1): if heredocs: delimiter, strip_tabs, expand_body, receiver = heredocs[0] comparison = raw.lstrip("\t") if strip_tabs else raw if comparison == delimiter: if receiver not in {None, "shell"} and heredoc_body: body_text = "\n".join(heredoc_body) for command, relative_line in language_execution_command_records( body_text, receiver ): finding = inspect_literal_commands( [command], heredoc_body_start + relative_line - 1 ) if finding: return finding heredocs.pop(0) heredoc_body = [] heredoc_body_start = None continue if receiver is not None: if not budget.admit_line(candidate=prune_candidate): return None if receiver == "shell": commands = [raw] else: if heredoc_body_start is None: heredoc_body_start = line_number heredoc_body.append(raw) commands = language_execution_commands(raw, receiver) finding = inspect_literal_commands(commands, line_number) if finding: return finding if expand_body: finding = inspect_literal_commands( shell_command_substitutions(raw), line_number ) if finding: return finding continue stripped = raw.strip() if ( not stripped or stripped.startswith("#") or (language == "node" and stripped.startswith("//")) ): continue if not budget.admit_line(candidate=prune_candidate): return None commands = ( [raw] if language == "shell" else language_execution_commands(raw, language) ) finding = inspect_literal_commands(commands, line_number) if finding: return finding if language == "shell": heredocs.extend(shell_heredoc_specs(raw)) return None def inspect_run_parts_directory(path_logical, budget, depth, priority=False): if depth > HELPER_DEPTH_LIMIT: fail(f"scheduled helper recursion exceeded depth {HELPER_DEPTH_LIMIT}") for definition, _, target in secure_cron_directory( path_logical, "run-parts", budget.scheduler_entries, budget.scheduler_directories, ): finding = inspect_helper( display(definition), "direct", budget, depth, priority=priority ) if finding: prune_kind, helper_path, helper_line = finding return ( prune_kind, display(definition) if helper_path == display(target) else helper_path, helper_line, ) return None def inspect_scheduled_command( command, budget, direct_invocation, prioritize_helpers=False ): prune_kind = dangerous( command, shell_syntax=direct_invocation == "shell-fallback", systemd_syntax=direct_invocation == "systemd", ) if prune_kind: return prune_kind, None, None for helper, invocation in helper_candidates( command, direct_invocation=direct_invocation ): if invocation == "run-parts": finding = inspect_run_parts_directory( helper, budget, 1, priority=prioritize_helpers ) else: finding = inspect_helper( helper, invocation, budget, 1, priority=prioritize_helpers ) if finding: return finding return None def unsafe_jobs(excluded): findings, inspected = [], set() scheduler_entry_budget = [0] scheduler_directories = set() helper_budget = HelperBudget(scheduler_entry_budget, scheduler_directories) systemd_cache = {} for definition, kind, resolved_target in jobs( scheduler_entry_budget, scheduler_directories ): if excluded is not None and ( os.path.normpath(definition) == os.path.normpath(excluded) ): continue if kind == "systemd": effective = {key: [] for key in sorted(EXEC_KEYS)} service_assignments = [] for origin, target in resolved_target: metadata = lstat_or_none(target) if metadata is None: fail(f"systemd definition {display(target)} disappeared") if ( not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) & 0o022 ): fail( f"systemd definition {display(target)} is non-regular, " "non-root-owned, or writable by another account" ) identity = ( metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, ) cached = systemd_cache.get(identity) if cached is None: payload, _ = read_root_file( target, require_root_group=True ) assignments = list(systemd_service_directives(payload)) directives = [ item for item in assignments if item[1] in EXEC_KEYS ] cached = (directives, assignments) systemd_cache[identity] = cached directives, assignments = cached service_assignments.extend( (origin, number, key, value) for number, key, value in assignments ) for number, key, command in directives: if not command.strip(): effective[key] = [] else: effective[key].append((origin, number, command)) is_runtime_repair_unit = ( definition == os.path.basename(PORTAL_RUNTIME_REPAIR_UNIT) ) references_runtime_repair = any( PORTAL_RUNTIME_REPAIR_LAUNCHER in systemd_unescaped_reference(value) for _, _, _, value in service_assignments ) if is_runtime_repair_unit or references_runtime_repair: expected_origin = PORTAL_RUNTIME_REPAIR_UNIT if ( not is_runtime_repair_unit or len(resolved_target) != 1 or any( display(origin) != expected_origin or display(target) != expected_origin for origin, target in resolved_target ) ): fail( "scheduled Project runtime repair launcher is outside " "its exact transient unit" ) service_shape = [ (key, value.strip()) for _, _, key, value in service_assignments ] environment_values = [ value for key, value in service_shape if key == "Environment" ] exec_start_values = [ value for key, value in service_shape if key == "ExecStart" ] other_service_keys = [ key for key, _ in service_shape if key not in {"Environment", "ExecStart"} ] nonempty_exec_start = [ value for value in exec_start_values if value ] if ( other_service_keys or len(environment_values) != 1 or tokens(environment_values[0]) != ["HOME=/root"] or len(exec_start_values) != 2 or exec_start_values.count("") != 1 or len(nonempty_exec_start) != 1 ): fail( "scheduled Project runtime repair unit properties " "are malformed" ) repair_candidates = helper_candidates( nonempty_exec_start[0], direct_invocation="systemd" ) if repair_candidates != [ ( PORTAL_RUNTIME_REPAIR_LAUNCHER, "portal-runtime-repair-launcher", ) ]: fail( "scheduled Project runtime repair launcher command " "is malformed" ) unit_found = False for key in sorted(effective): for origin, number, command in effective[key]: finding = inspect_scheduled_command( command, helper_budget, "systemd", prioritize_helpers=locally_managed_scheduler_path( display(origin) ), ) if finding: prune_kind, helper_path, helper_line = finding findings.append( ( helper_path or display(origin), helper_line or number, prune_kind, ) ) unit_found = True break if unit_found: break continue target = resolved_target if target is None: continue metadata = lstat_or_none(target) if metadata is None: continue if ( not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0 or stat.S_IMODE(metadata.st_mode) & 0o022 or not metadata.st_mode & stat.S_IRUSR ): fail( f"cron definition {display(target)} is non-regular, unreadable, " "non-root-owned, or writable by another account" ) identity = (metadata.st_dev, metadata.st_ino, kind) if identity in inspected: continue inspected.add(identity) if kind == "run-parts": finding = inspect_helper( display(definition), "direct", helper_budget, 1, priority=True ) if finding: prune_kind, helper_path, helper_line = finding finding_path = ( display(definition) if helper_path == display(target) else helper_path or display(definition) ) findings.append( (finding_path, helper_line or 1, prune_kind) ) continue payload, _ = read_root_file(target) commands = cron_commands(payload, kind) for number, command in commands: finding = inspect_scheduled_command( command, helper_budget, "shell-fallback", prioritize_helpers=True, ) if finding: prune_kind, helper_path, helper_line = finding findings.append( (helper_path or display(definition), helper_line or number, prune_kind) ) break return findings def ensure_quarantine_parent(path): relative = os.path.relpath(os.path.dirname(path), root) if relative.startswith(".."): fail("quarantine path escapes the host root") current = root components = [] if relative == "." else relative.split(os.sep) for index, component in enumerate(components): current = os.path.join(current, component) metadata = lstat_or_none(current) if metadata is None: os.mkdir(current, 0o700 if index == len(components) - 1 else 0o755) metadata = lstat_or_none(current) if ( metadata is None or not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) & 0o022 ): fail(f"quarantine parent {display(current)} is linked or has unsafe metadata") if index == len(components) - 1 and stat.S_IMODE(metadata.st_mode) != 0o700: fail(f"quarantine directory {display(current)} must have mode 0700") def fsync_dir(path): descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(descriptor) finally: os.close(descriptor) def open_secure_directory_fd(path, context): normalized = os.path.normpath(path) relative = os.path.relpath(normalized, root) if relative == ".." or relative.startswith(".." + os.sep): fail(f"{context} directory escapes the host root") flags = ( os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) ) try: descriptor = os.open(root, flags) except OSError as exc: fail(f"could not bind the host root for {context}: {exc}") current = root try: require_secure_directory(current, os.fstat(descriptor), context) components = [] if relative == "." else relative.split(os.sep) for component in components: try: child = os.open(component, flags, dir_fd=descriptor) except OSError as exc: fail( f"could not bind {context} directory " f"{display(os.path.join(current, component))}: {exc}" ) os.close(descriptor) descriptor = child current = os.path.join(current, component) require_secure_directory(current, os.fstat(descriptor), context) return descriptor except BaseException: os.close(descriptor) raise def open_attested_legacy(parent_fd, name, expected, expected_payload): flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) try: descriptor = os.open(name, flags, dir_fd=parent_fd) except OSError as exc: fail(f"could not open {source_logical} for inode-bound quarantine: {exc}") try: metadata = os.fstat(descriptor) identity = ( metadata.st_dev, metadata.st_ino, metadata.st_mode, metadata.st_uid, metadata.st_gid, metadata.st_nlink, metadata.st_size, metadata.st_mtime_ns, ) expected_identity = ( expected.st_dev, expected.st_ino, expected.st_mode, expected.st_uid, expected.st_gid, expected.st_nlink, expected.st_size, expected.st_mtime_ns, ) if identity != expected_identity: fail(f"{source_logical} changed before inode-bound quarantine") payload = b"" while len(payload) <= LIMIT: chunk = os.read(descriptor, min(65536, LIMIT + 1 - len(payload))) if not chunk: break payload += chunk if payload != expected_payload or not is_known_legacy_payload(payload): fail(f"{source_logical} content changed before inode-bound quarantine") os.lseek(descriptor, 0, os.SEEK_SET) return descriptor, metadata except BaseException: os.close(descriptor) raise def fixture_race_hook( stage, source, source_parent_fd, source_name, quarantine_name, quarantine_dir_fd, ): hook = "" if os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and root != "/": hook = os.environ.get("BRIDGESLLM_DOCKER_PRUNE_TEST_HOOK", "") if not hook: return if hook == "destination-collision" and stage == "before-link": descriptor = os.open( quarantine_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), 0o600, dir_fd=quarantine_dir_fd, ) try: os.write(descriptor, b"operator collision\n") os.fsync(descriptor) finally: os.close(descriptor) os.fsync(quarantine_dir_fd) return if hook in {"source-replacement", "post-check-source-replacement"}: expected_stage = ( "before-link" if hook == "source-replacement" else "post-source-check" ) if stage != expected_stage: return replacement = source + ".test-replacement" displaced = source + ".test-old" if os.path.lexists(displaced): fail("source-replacement test hook found an existing displaced path") replacement_payload, _ = read_root_file( replacement, {0o644}, True, True ) if replacement_payload is None: fail("source-replacement test hook has no replacement fixture") os.rename( source_name, source_name + ".test-old", src_dir_fd=source_parent_fd, dst_dir_fd=source_parent_fd, ) os.rename( source_name + ".test-replacement", source_name, src_dir_fd=source_parent_fd, dst_dir_fd=source_parent_fd, ) os.fsync(source_parent_fd) return if hook == "final-source-reappearance": return if hook and stage != "before-link": return fail(f"unknown Docker prune fixture hook: {hook}") def unlink_destination_if_held_inode(directory_fd, name, held): try: current = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) except FileNotFoundError: return if (current.st_dev, current.st_ino) != (held.st_dev, held.st_ino): fail("quarantine destination changed identity during rollback") os.unlink(name, dir_fd=directory_fd) os.fsync(directory_fd) def rename_noreplace(source_dir_fd, source_name, target_dir_fd, target_name): libc = ctypes.CDLL(None, use_errno=True) renameat2 = getattr(libc, "renameat2", None) if renameat2 is None: fail("this Linux libc does not expose renameat2 for no-replace quarantine") renameat2.argtypes = [ ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint, ] renameat2.restype = ctypes.c_int if renameat2( source_dir_fd, os.fsencode(source_name), target_dir_fd, os.fsencode(target_name), 1, # RENAME_NOREPLACE ) != 0: error_number = ctypes.get_errno() raise OSError(error_number, os.strerror(error_number)) def restore_tombstone(source_parent_fd, tombstone_name, source_name): try: rename_noreplace( source_parent_fd, tombstone_name, source_parent_fd, source_name, ) except OSError as exc: fail( f"source replacement was preserved as {tombstone_name}, but could " f"not be restored to {source_logical}: {exc}" ) os.fsync(source_parent_fd) def quarantine_legacy_inode(source, quarantine, expected, expected_payload): source_directory = os.path.dirname(source) source_name = os.path.basename(source) quarantine_directory = os.path.dirname(quarantine) quarantine_name = os.path.basename(quarantine) source_parent_fd = open_secure_directory_fd( source_directory, "legacy source-parent" ) try: source_fd, held = open_attested_legacy( source_parent_fd, source_name, expected, expected_payload ) except BaseException: os.close(source_parent_fd) raise try: quarantine_dir_fd = open_secure_directory_fd( quarantine_directory, "quarantine" ) except BaseException: os.close(source_fd) os.close(source_parent_fd) raise linked = False tombstone_name = None try: fixture_race_hook( "before-link", source, source_parent_fd, source_name, quarantine_name, quarantine_dir_fd, ) libc = ctypes.CDLL(None, use_errno=True) linkat = getattr(libc, "linkat", None) if linkat is None: fail("this Linux libc does not expose linkat for inode-bound quarantine") linkat.argtypes = [ ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ] linkat.restype = ctypes.c_int if linkat(source_fd, b"", quarantine_dir_fd, os.fsencode(quarantine_name), 0x1000) != 0: error_number = ctypes.get_errno() if error_number == errno.EEXIST: fail( f"{quarantine_logical} appeared during quarantine; " "the active legacy job was left untouched" ) fail( "inode-bound no-replace quarantine was unavailable; " f"the active legacy job was left untouched ({os.strerror(error_number)})" ) linked = True os.fsync(quarantine_dir_fd) source_now = os.stat( source_name, dir_fd=source_parent_fd, follow_symlinks=False ) destination_now = os.stat( quarantine_name, dir_fd=quarantine_dir_fd, follow_symlinks=False ) held_now = os.fstat(source_fd) held_identity = (held.st_dev, held.st_ino) if ( (source_now.st_dev, source_now.st_ino) != held_identity or (destination_now.st_dev, destination_now.st_ino) != held_identity or (held_now.st_dev, held_now.st_ino) != held_identity or held_now.st_nlink != 2 ): unlink_destination_if_held_inode( quarantine_dir_fd, quarantine_name, held ) linked = False fail( f"{source_logical} changed at the final quarantine boundary; " "no recovery copy was installed" ) fixture_race_hook( "post-source-check", source, source_parent_fd, source_name, quarantine_name, quarantine_dir_fd, ) tombstone_name = ( f".{source_name}.bridgesllm-tombstone-{os.getpid()}-" f"{os.urandom(16).hex()}" ) try: rename_noreplace( source_parent_fd, source_name, source_parent_fd, tombstone_name, ) except OSError as exc: unlink_destination_if_held_inode( quarantine_dir_fd, quarantine_name, held ) linked = False fail(f"could not tombstone the attested legacy source: {exc}") tombstone = os.stat( tombstone_name, dir_fd=source_parent_fd, follow_symlinks=False ) if (tombstone.st_dev, tombstone.st_ino) != held_identity: restore_tombstone(source_parent_fd, tombstone_name, source_name) tombstone_name = None unlink_destination_if_held_inode( quarantine_dir_fd, quarantine_name, held ) linked = False fail( f"{source_logical} changed after its final identity check; " "the replacement survived and no recovery copy was installed" ) os.unlink(tombstone_name, dir_fd=source_parent_fd) tombstone_name = None os.fchmod(source_fd, 0o600) os.fsync(source_fd) os.fsync(source_parent_fd) os.fsync(quarantine_dir_fd) final = os.fstat(source_fd) if final.st_nlink != 1 or stat.S_IMODE(final.st_mode) != 0o600: fail("inode-bound quarantine did not settle to one sealed recovery link") except BaseException: if tombstone_name is not None: try: tombstone = os.stat( tombstone_name, dir_fd=source_parent_fd, follow_symlinks=False, ) except FileNotFoundError: tombstone = None if tombstone is not None and ( tombstone.st_dev, tombstone.st_ino ) == (held.st_dev, held.st_ino): restore_tombstone(source_parent_fd, tombstone_name, source_name) tombstone_name = None if linked: try: current = os.stat( source_name, dir_fd=source_parent_fd, follow_symlinks=False, ) except FileNotFoundError: current = None if current is not None: unlink_destination_if_held_inode( quarantine_dir_fd, quarantine_name, held ) raise finally: os.close(quarantine_dir_fd) os.close(source_fd) os.close(source_parent_fd) def attest_secure_directory_chain(path): relative = os.path.relpath(path, root) if relative == ".." or relative.startswith(".." + os.sep): fail(f"scheduler directory {display(path)} escapes the host root") current = root components = [] if relative == "." else relative.split(os.sep) for component in [None, *components]: if component is not None: current = os.path.join(current, component) metadata = lstat_or_none(current) if metadata is None: return False if ( not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) & 0o022 ): fail( f"scheduler directory chain component {display(current)} is " "linked, non-root-owned, or writable by another account" ) return True def fixture_final_scan_hook(source): if not ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and root != "/" and os.environ.get("BRIDGESLLM_DOCKER_PRUNE_TEST_HOOK", "") == "final-source-reappearance" ): return parent = os.path.dirname(source) parent_fd = open_secure_directory_fd(parent, "legacy final-scan source-parent") try: descriptor = os.open( os.path.basename(source), os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), 0o644, dir_fd=parent_fd, ) try: os.write(descriptor, LEGACY) os.fsync(descriptor) finally: os.close(descriptor) os.fsync(parent_fd) finally: os.close(parent_fd) root_metadata = lstat_or_none(root) if ( root_metadata is None or not stat.S_ISDIR(root_metadata.st_mode) or stat.S_ISLNK(root_metadata.st_mode) or root_metadata.st_uid != 0 or root != os.path.realpath(root) ): fail("host root must be an existing canonical root-owned directory") source = host_path(source_logical) quarantine = host_path(quarantine_logical) source_parent_exists = attest_secure_directory_chain(os.path.dirname(source)) source_exists = source_parent_exists and os.path.lexists(source) quarantine_exists = os.path.lexists(quarantine) source_metadata = None if source_exists: payload, source_metadata = read_root_file(source, {0o644}, True, True) if not is_known_legacy_payload(payload): fail( f"{source_logical} differs from the one known legacy job; " "disable or replace it manually, then retry" ) if quarantine_exists: ensure_quarantine_parent(quarantine) payload, _ = read_root_file(quarantine, {0o600}, True, True) if not is_known_legacy_payload(payload): fail(f"{quarantine_logical} is not the known recovery copy; resolve it manually") findings = unsafe_jobs(source) if findings: print( "Unsafe Docker prune guard: refusing install/update because another " "effective installed root scheduler definition contains a literal " "command that can delete retained Docker images:", file=sys.stderr, ) for path, number, kind in findings[:32]: print(f" - {path}:{number} (docker {kind} prune with --all/-a)", file=sys.stderr) if len(findings) > 32: print(f" - and {len(findings) - 32} more", file=sys.stderr) print( "Disable the definition or change it to dangling-only cleanup, such as " "docker image prune --force, then rerun. No unknown job was changed.", file=sys.stderr, ) raise SystemExit(1) if source_exists and quarantine_exists: fail( f"both {source_logical} and {quarantine_logical} exist; " "verify the recovery copy before removing the active file" ) if source_exists: ensure_quarantine_parent(quarantine) payload, current = read_root_file(source, {0o644}, True, True) if not is_known_legacy_payload(payload) or ( current.st_dev, current.st_ino, current.st_mtime_ns ) != ( source_metadata.st_dev, source_metadata.st_ino, source_metadata.st_mtime_ns ): fail(f"{source_logical} changed before quarantine") quarantine_legacy_inode(source, quarantine, current, payload) print(f"Quarantined the retired all-images Docker prune job at {quarantine_logical}.") if os.path.lexists(source): fail(f"{source_logical} still exists after convergence") fixture_final_scan_hook(source) if os.path.lexists(quarantine): payload, _ = read_root_file(quarantine, {0o600}, True, True) if not is_known_legacy_payload(payload): fail(f"{quarantine_logical} failed final attestation") if unsafe_jobs(None): fail( "an unsafe Docker prune definition appeared during convergence; " "inspect installed scheduler definitions and retry" ) if os.path.lexists(source): fail(f"{source_logical} reappeared during final convergence") PY2 } ensure_media_toolchain() { if command -v ffmpeg >/dev/null 2>&1 \ && command -v ffprobe >/dev/null 2>&1; then ok "Animated GIF media tools" return 0 fi command -v apt-get >/dev/null 2>&1 || return 1 info "Installing the FFmpeg media tools required for animated GIF uploads..." run "apt-get update -qq" || return 1 spin_apt "Installing animated GIF media tools" ffmpeg || return 1 command -v ffmpeg >/dev/null 2>&1 \ && command -v ffprobe >/dev/null 2>&1 } preflight() { step_header "Checking system requirements" CURRENT_STEP="preflight" # Root [[ "${EUID:-$(id -u)}" -eq 0 ]] || fail "Must run as root. Use: sudo bash ${SCRIPT_NAME}" # OS if [[ -f /etc/os-release ]]; then OS_ID="$(grep -oP '^ID=\K.*' /etc/os-release | tr -d '"' || echo unknown)" OS_VERSION="$(grep -oP '^VERSION_ID=\K.*' /etc/os-release | tr -d '"' || echo unknown)" fi command -v apt-get &>/dev/null && APT_AVAILABLE=true case "${OS_ID}" in ubuntu|debian) ;; *) warn "Detected ${OS_ID} ${OS_VERSION} — tested on Ubuntu/Debian only" ;; esac # RAM local mem_mb mem_mb=$(awk '/MemTotal/ {printf "%d", $2/1024}' /proc/meminfo) (( mem_mb >= MIN_RAM_MB )) || fail "Need ${MIN_RAM_MB}MB+ RAM (found: ${mem_mb}MB)" # Disk local disk_gb disk_gb=$(df -BG / | awk 'NR==2 {gsub("G",""); print $4}') (( disk_gb >= MIN_DISK_GB )) || fail "Need ${MIN_DISK_GB}GB+ disk (found: ${disk_gb}GB)" # CPUs local cpus cpus=$(nproc 2>/dev/null || echo 1) (( cpus >= 2 )) || warn "Only ${cpus} CPU — 2+ CPUs recommended for best performance" local uptime_min uptime_min=$(awk '{print int($1/60)}' /proc/uptime 2>/dev/null || echo 9999) echo "" print_kv "OS" "${OS_ID^} ${OS_VERSION}" "$WHITE" print_kv "CPUs" "${cpus}" "$WHITE" print_kv "RAM" "${mem_mb} MB" "$WHITE" print_kv "Disk free" "${disk_gb} GB" "$WHITE" if use_local_profile; then print_kv "Profile" "Local beta (WSL / localhost, experimental)" "$GREEN" else print_kv "Profile" "Server / VPS" "$WHITE" fi print_kv "Uptime" "${uptime_min} min" "$WHITE" echo "" if use_local_profile && ! systemd_ready; then fail "WSL detected but systemd is not running. Enable systemd for your Ubuntu WSL distro, restart WSL, then rerun the installer. See docs/WINDOWS_WSL_BETA.md" fi if ! use_local_profile && (( uptime_min < 20 )); then warn "Fresh VPS detected. First-boot package tasks may still be running in the background." info "If package setup pauses later, the installer will now show what it is waiting on and continue automatically." fi # Ports local blocked="" if use_local_profile || use_tailnet_profile; then if ss -tlnp "sport = :4001" 2>/dev/null | grep -q ":4001"; then blocked=" 4001" fi [[ -z "$blocked" ]] && ok "Port 4001 available" || warn "Port${blocked} in use — another local portal or web app may already be running" else for port in 80 443; do if ss -tlnp "sport = :${port}" 2>/dev/null | grep -q ":${port}"; then blocked+=" $port" fi done [[ -z "$blocked" ]] && ok "Ports 80, 443 available" || warn "Ports${blocked} in use — Caddy will take them over" fi # Internet curl -fsSL --max-time 10 https://www.google.com &>/dev/null || fail "No internet connectivity" # Public IP / local access if use_local_profile; then PUBLIC_IP="127.0.0.1" print_kv "Access URL" "http://localhost:4001" "$GREEN" info "WSL local mode serves the portal directly on localhost so Windows users can test before moving to a VPS. This path is experimental, still untested in the field, and under active development." elif use_tailnet_profile; then # A public IP is not required: the portal is reachable only over the # operator's tailnet, and no public port is ever opened. PUBLIC_IP="$(curl -fsSL --max-time 5 https://api.ipify.org 2>/dev/null || echo "0.0.0.0")" print_kv "Origin mode" "Private Tailscale network (no public ports)" "$GREEN" warn "Tailnet origin mode is EXPERIMENTAL — new and still under field validation. Use --domain for the recommended production install." else PUBLIC_IP=$(curl -fsSL --max-time 5 https://api.ipify.org 2>/dev/null || \ curl -fsSL --max-time 5 https://ifconfig.me 2>/dev/null || \ echo "") if [[ -n "$PUBLIC_IP" ]]; then print_kv "Public IP" "${PUBLIC_IP}" "$GREEN" else warn "Could not detect public IP" PUBLIC_IP="0.0.0.0" fi fi if $APT_AVAILABLE; then wait_for_package_manager_ready "system preflight" fi # Core tools — always ensure lsb-release is present (needed by PostgreSQL repo setup) local missing="" for cmd in curl git openssl rsync lsb_release zstd python3; do command -v "$cmd" &>/dev/null || missing+=" $cmd" done if [[ -n "$missing" ]]; then $APT_AVAILABLE || fail "Missing:${missing} — and apt is not available" info "Installing core tools..." run "apt-get update -qq && apt-get install -y -qq curl git openssl rsync ca-certificates gnupg lsb-release ffmpeg python3 python3-venv zstd" fi ensure_media_toolchain \ || fail "Animated GIF uploads require ffmpeg and ffprobe, but the FFmpeg package could not be installed. Check ${LOG_FILE}, repair apt, and rerun the installer." ok "System checks passed" } # ═══════════════════════════════════════════════════════════════ # Step 2: System packages # ═══════════════════════════════════════════════════════════════ ensure_remote_desktop_packages() { # Remote Desktop packages (VNC + XFCE desktop + PulseAudio + X11 clipboard bridge). # Keep this idempotent: update/recovery paths call Remote Desktop setup directly, # so new package requirements must repair existing installs, not just fresh installs. local rd_pkgs=(tigervnc-standalone-server novnc websockify xfce4 xfce4-goodies xfce4-terminal dbus-x11 x11-utils xauth xclip xsel xterm firefox pulseaudio pulseaudio-utils librsvg2-common wmctrl xdotool) local rd_missing=() for pkg in "${rd_pkgs[@]}"; do if ! dpkg -s "$pkg" &>/dev/null; then rd_missing+=("$pkg") fi done if [[ ${#rd_missing[@]} -eq 0 ]]; then ok "Remote Desktop packages" else run "apt-get update -qq" spin_apt "Installing Remote Desktop packages" "${rd_missing[@]}" ok "Remote Desktop packages" fi } install_system_packages() { step_header "Installing system packages" CURRENT_STEP="system packages" # OpenClaw publishes a disjoint engine range with exact patch floors. Node 23 # is not supported even though it is numerically newer than Node 22. ensure_supported_node_runtime # PostgreSQL 16 if command -v psql &>/dev/null; then ok "PostgreSQL $(psql --version | grep -oP '\d+' | head -1)" else run "apt-get install -y -qq wget" run "sh -c 'echo \"deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt \$(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" progress "Refreshing PostgreSQL signing key..." ensure_keyring_from_url "https://www.postgresql.org/media/keys/ACCC4CF8.asc" "/usr/share/keyrings/postgresql-keyring.gpg" spin "Installing PostgreSQL 16" "apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq postgresql-16 postgresql-contrib-16" run "systemctl enable postgresql && systemctl start postgresql" ok "PostgreSQL 16" fi # Caddy if use_local_profile; then info "Skipping Caddy in local beta mode — portal will be served directly on http://localhost:4001" elif command -v caddy &>/dev/null; then ok "Caddy web server" else run "apt-get install -y -qq debian-keyring debian-archive-keyring apt-transport-https" progress "Refreshing Caddy signing key..." ensure_keyring_from_url "https://dl.cloudsmith.io/public/caddy/stable/gpg.key" "/usr/share/keyrings/caddy-stable-archive-keyring.gpg" run "curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list" spin "Installing Caddy web server" "apt-get update -qq && apt-get install -y -qq caddy" ok "Caddy web server" fi # Docker if command -v docker &>/dev/null; then ok "Docker" else spin "Installing Docker" "curl -fsSL https://get.docker.com | sh" run "systemctl enable docker && systemctl start docker" ok "Docker" fi # ClamAV if command -v clamscan &>/dev/null; then ok "ClamAV antivirus" else spin "Installing ClamAV antivirus" "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq clamav clamav-daemon" run "systemctl stop clamav-freshclam 2>/dev/null || true" spin "Updating virus definitions" "freshclam || true" run "systemctl enable clamav-daemon clamav-freshclam 2>/dev/null || true" run "systemctl start clamav-freshclam 2>/dev/null || true" run "systemctl start clamav-daemon 2>/dev/null || true" ok "ClamAV antivirus" fi # UFW if use_local_profile; then info "Skipping UFW in local beta mode — Windows/WSL networking stays local by default" else command -v ufw &>/dev/null || run "apt-get install -y -qq ufw" ok "Firewall (UFW)" fi ensure_remote_desktop_packages # Desktop themes (Greybird + elementary icons) local theme_pkgs=(greybird-gtk-theme elementary-xfce-icon-theme numix-gtk-theme gnome-themes-extra) local themes_missing=() for pkg in "${theme_pkgs[@]}"; do if ! dpkg -s "$pkg" &>/dev/null; then themes_missing+=("$pkg") fi done if [[ ${#themes_missing[@]} -eq 0 ]]; then ok "Desktop themes" else spin_apt "Installing desktop themes" "${themes_missing[@]}" || true ok "Desktop themes" fi # Google Chrome if dpkg -s google-chrome-stable &>/dev/null 2>&1; then ok "Google Chrome" else spin_download "Downloading Google Chrome" "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" "/tmp/google-chrome.deb" spin "Installing Google Chrome" "DEBIAN_FRONTEND=noninteractive apt-get install -y /tmp/google-chrome.deb && rm -f /tmp/google-chrome.deb" || true ok "Google Chrome" fi } # ═══════════════════════════════════════════════════════════════ # Step 3: AI tools # ═══════════════════════════════════════════════════════════════ install_ai_tools() { step_header "Installing AI tools" CURRENT_STEP="AI tools" # Ollama if $SKIP_OLLAMA; then info "Skipping Ollama (--skip-ollama)" else install_or_update_ollama ok "Ollama — models will be configured in the setup wizard" fi # OpenClaw if $SKIP_OPENCLAW; then info "Skipping OpenClaw (--skip-openclaw)" else converge_openclaw_core_package ok "OpenClaw core ${PIN_OPENCLAW_CORE_PACKAGE_VERSION}" fi # Portal-native provider CLIs are exact-pinned. A failed replacement restores # the previous global npm package rather than leaving a half-updated tool. if ! $SKIP_OPENCLAW && command -v npm &>/dev/null; then converge_pinned_npm_cli "ClawHub" "clawhub" "${PIN_CLAWHUB_VERSION}" "clawhub" "--cli-version" \ || warn "ClawHub CLI install failed; Skills marketplace search/install will be unavailable until repaired." fi converge_pinned_npm_cli "Codex CLI" "@openai/codex" "${PIN_CODEX_CLI_VERSION}" "codex" "--version" \ || warn "Codex CLI install failed; the native Codex provider will remain unavailable." converge_pinned_npm_cli "Claude Code" "@anthropic-ai/claude-code" "${PIN_CLAUDE_CODE_VERSION}" "claude" "--version" \ || warn "Claude Code install failed; the native Claude provider will remain unavailable." # Configure OpenClaw gateway with the portal's operator token if ! $SKIP_OPENCLAW && command -v openclaw &>/dev/null; then local oc_dir="${HOME}/.openclaw" local oc_config="${oc_dir}/openclaw.json" mkdir -p "${oc_dir}" # Generate a token early if we don't have one yet [[ -n "${OPENCLAW_TOKEN}" ]] || OPENCLAW_TOKEN="$(rand_hex 24)" if [[ -f "${oc_config}" ]]; then # Read existing token if set, otherwise inject ours local existing_oc_token existing_oc_token="$(python3 -c " import json try: d = json.load(open('${oc_config}')) print(d.get('gateway',{}).get('auth',{}).get('token','')) except: pass " 2>/dev/null || true)" if [[ -n "${existing_oc_token}" ]]; then # Use OpenClaw's existing token for the portal OPENCLAW_TOKEN="${existing_oc_token}" else # Inject our token into OpenClaw's config python3 -c " import json d = json.load(open('${oc_config}')) d.setdefault('gateway', {}).setdefault('auth', {})['token'] = '${OPENCLAW_TOKEN}' d['gateway']['auth']['mode'] = 'token' d['gateway']['port'] = 18789 json.dump(d, open('${oc_config}', 'w'), indent=2) " 2>/dev/null || true fi else # Create minimal OpenClaw config with gateway token cat > "${oc_config}" << OCEOF { "gateway": { "port": 18789, "mode": "local", "bind": "loopback", "auth": { "mode": "token", "token": "${OPENCLAW_TOKEN}" } } } OCEOF chmod 600 "${oc_config}" fi ok "OpenClaw gateway configured" fi } install_native_provider_tools() { step_header "Installing native provider runtimes" CURRENT_STEP="native provider runtimes" if converge_antigravity; then ok "Antigravity ${PIN_ANTIGRAVITY_VERSION} (verified)" else warn "Antigravity could not be installed; its Portal provider will remain unavailable." fi if converge_grok_build; then ok "Grok Build ${PIN_GROK_BUILD_VERSION} (verified)" else warn "Grok Build could not be installed; its Portal provider will remain unavailable." fi } # ═══════════════════════════════════════════════════════════════ # Step 4: Database # ═══════════════════════════════════════════════════════════════ setup_database() { step_header "Setting up database" CURRENT_STEP="database" assert_database_process_environment_safe \ || fail "The installer process environment overrides the attested database runtime. Remove Prisma engine switches, node-postgres PG* fallbacks, NODE_PG_FORCE_NATIVE, and NODE_TLS_REJECT_UNAUTHORIZED before installing." # A forced reinstall over an existing Portal must not create/alter the # installer's default local database when the Portal is configured for a # different PostgreSQL host, port, database, or account. The retained URL is # validated and used unchanged later by run_migrations_safe(). local existing_database_url="" if [[ -f "${PORTAL_DIR}/backend/.env.production" ]]; then assert_prisma_runtime_environment_safe \ "${PORTAL_DIR}/backend/.env.production" \ || fail "The retained backend environment overrides the attested database runtime; repair it before reinstalling." existing_database_url="$(read_env_value "${PORTAL_DIR}/backend/.env.production" "DATABASE_URL" || true)" fi if [[ -n "${existing_database_url}" ]]; then pg_url_component "${existing_database_url}" host >/dev/null \ && pg_url_component "${existing_database_url}" port >/dev/null \ && pg_url_component "${existing_database_url}" database >/dev/null \ && pg_url_component "${existing_database_url}" user >/dev/null \ && pg_url_component "${existing_database_url}" password >/dev/null \ || fail "Existing DATABASE_URL is invalid; refusing to alter a fallback database during reinstall." pg_url_uses_supported_prisma_adapter_options "${existing_database_url}" \ || fail "Existing DATABASE_URL has an ambiguous or unsupported database-driver option. Remote databases must set sslmode=disable, require, verify-ca, or verify-full. An absolute sslrootcert is required for verify-ca/verify-full; sslcert/client identities/keys, sslaccept, and channel_binding are not supported. Only lowercase application_name, fallback_application_name, options, client_encoding, replication, and documented Prisma pool controls may be supplied; literal plus signs must be percent-encoded. Custom connect_timeout and pool_timeout values must match. Repair the URL before reinstalling." info "Preserving the configured PostgreSQL database; local fallback creation is skipped" return 0 fi [[ -n "${DB_PASSWORD}" ]] || DB_PASSWORD="$(rand_pass 24)" # Create user if sudo -u postgres psql -tc "SELECT 1 FROM pg_roles WHERE rolname='blp'" 2>/dev/null | grep -q 1; then run "sudo -u postgres psql -c \"ALTER USER blp WITH PASSWORD '${DB_PASSWORD}';\"" else run "sudo -u postgres psql -c \"CREATE USER blp WITH PASSWORD '${DB_PASSWORD}';\"" fi # Create database if ! sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname='bridgesllm_portal'" 2>/dev/null | grep -q 1; then run "sudo -u postgres psql -c \"CREATE DATABASE bridgesllm_portal OWNER blp;\"" fi run "sudo -u postgres psql -c \"GRANT ALL PRIVILEGES ON DATABASE bridgesllm_portal TO blp;\"" ok "Database ready ${DIM}(bridgesllm_portal)${NC}" } # ═══════════════════════════════════════════════════════════════ # Step 5: Build portal # ═══════════════════════════════════════════════════════════════ attest_portal_app_sources_root() { local root="$1" python3 - "${root}" "${PORTAL_DIR}" "${INSTALL_ROOT}/apps" <<'PY' import os import pathlib import stat import sys raw, portal_root_raw, hosted_root_raw = sys.argv[1:] root = pathlib.Path(raw) portal_root = pathlib.Path(portal_root_raw) hosted_root = pathlib.Path(hosted_root_raw) broad = { pathlib.Path(value) for value in ( "/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib64", "/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/var", ) } if ( not raw or not root.is_absolute() or os.path.normpath(raw) != raw or root in broad or root == hosted_root or hosted_root.is_relative_to(root) or len(raw.encode("utf-8")) > 4096 or any(ord(character) < 32 or ord(character) == 127 for character in raw) ): raise SystemExit(1) current = pathlib.Path("/") for component in root.parts[1:]: current /= component try: info = os.lstat(current) except FileNotFoundError: break if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 ): raise SystemExit(1) # A custom explicit root remains supported, but it may not engulf the Portal # runtime or the hosted deployment root. The two product defaults are exact. if root != portal_root / "apps" and root != pathlib.Path("/portal/apps"): if portal_root == root or portal_root.is_relative_to(root): raise SystemExit(1) print(root) PY } select_portal_app_sources_root() { local env_file="$1" local previous_version="${2:-}" local configured="" if [[ -f "${env_file}" && ! -L "${env_file}" ]]; then configured="$(read_env_value "${env_file}" PORTAL_APPS_ROOT || true)" fi if [[ -z "${configured}" ]]; then if [[ "${previous_version}" == 3.* ]]; then configured="/portal/apps" else configured="${PORTAL_DIR}/apps" fi fi attest_portal_app_sources_root "${configured}" } ensure_portal_app_sources_root() { local root="$1" if [[ ! -e "${root}" && ! -L "${root}" ]]; then install -d -m 0755 -- "${root}" || return 1 fi attest_portal_app_sources_root "${root}" >/dev/null } build_portal() { step_header "Installing portal" CURRENT_STEP="portal build" local pre_overlay_portal_version="" pre_overlay_portal_version="$( attest_existing_portal_for_update "${PORTAL_DIR}" 2>/dev/null || true )" # Create directories mkdir -p "${INSTALL_ROOT}" "${PORTAL_DIR}" "${LOG_DIR}" mkdir -p "${INSTALL_ROOT}/apps" "${INSTALL_ROOT}/uploads" "${INSTALL_ROOT}/backups" mkdir -p "${INSTALL_ROOT}/assets/avatars" "${INSTALL_ROOT}/assets/branding" mkdir -p "${INSTALL_ROOT}/assets/branding" mkdir -p "${PORTAL_DIR}/projects" "${PORTAL_DIR}/upload-temp" chmod 755 "${PORTAL_DIR}/projects" "${PORTAL_DIR}/upload-temp" # Download or copy portal. # Only use the local fallback dir as a SOURCE when it is a different path from # the install target. RELEASE_FALLBACK_DIR defaults to PORTAL_DIR, so on an # existing install this would otherwise rsync the live dir onto itself and # silently deploy nothing — prefer the release tarball in that case. if [[ "${BRIDGESLLM_ALLOW_UNVERIFIED_LOCAL_SOURCE:-0}" == "1" ]] \ && [[ -d "${RELEASE_FALLBACK_DIR}" ]] && [[ -f "${RELEASE_FALLBACK_DIR}/backend/package.json" ]] \ && [[ "$(readlink -f "${RELEASE_FALLBACK_DIR}" 2>/dev/null)" != "$(readlink -f "${PORTAL_DIR}" 2>/dev/null)" ]]; then UNVERIFIED_LOCAL_SOURCE_USED=true VERIFIED_RELEASE_VERSION="" VERIFIED_RELEASE_ARTIFACT_SHA256="" VERIFIED_RELEASE_MANIFEST_SHA256="" VERIFIED_RELEASE_MANIFEST_SCHEMA="" warn "Using explicitly authorized unverified local source; never use this path for a public release." spin "Copying portal files from local source" "rsync -a --delete --exclude='node_modules' --exclude='.git' --exclude='.env' --exclude='.env.production' --exclude='/projects' --exclude='/apps' --exclude='/assets' --exclude='/upload-temp' --exclude='/.data' --exclude='/backend/.data' '${RELEASE_FALLBACK_DIR}/' '${PORTAL_DIR}/'" else local release_stage_dir release_stage_dir="$(new_release_stage_dir)" if ! stage_verified_release "${release_stage_dir}"; then cleanup_release_stage_dir "${release_stage_dir}" || true fail "Portal release signature, version, digest, or archive validation failed." fi # Forced reinstalls converge to the signed payload exactly. Explicit data # exclusions retain user state while --delete removes compiled/routes files # that no longer exist in the release (a common source of ghost handlers). rsync -a --delete \ --exclude='node_modules' \ --exclude='.git' \ --exclude='.env' \ --exclude='.env.production' \ --exclude='/projects' \ --exclude='/apps' \ --exclude='/assets' \ --exclude='/upload-temp' \ --exclude='/.data' \ --exclude='/backend/.data' \ "${release_stage_dir}/portal/" "${PORTAL_DIR}/" cleanup_release_stage_dir "${release_stage_dir}" fi install_backend_runtime_dependencies ok "Portal files ready" local existing_env="${PORTAL_DIR}/backend/.env.production" if ${RETAINED_RECONNECT_MODE}; then # The signed runtime and exact backend dependency tree now exist, while # every retained path is still byte-for-byte identical to the receipt. # Retire the receipt at this recoverable boundary: a later interruption is # an ordinary attested repair/update, not an ambiguous partial fresh host. clear_retained_install_receipt true \ || fail "The retained Portal data changed while the signed runtime was being reconnected. The reconnect was stopped before configuration or migrations." ok "Retained Portal state reconnected to the signed runtime" fi local existing_database_url="" local portal_app_sources_root="" if [[ -f "${existing_env}" ]]; then info "Preserving existing secrets" existing_database_url="$(read_env_value "${existing_env}" "DATABASE_URL" || true)" JWT_SECRET="$(read_env_value "${existing_env}" "JWT_SECRET" || true)" JWT_REFRESH_SECRET="$(read_env_value "${existing_env}" "JWT_REFRESH_SECRET" || true)" PORTAL_UPDATE_PROBE_TOKEN="$(read_env_value "${existing_env}" "PORTAL_UPDATE_PROBE_TOKEN" || true)" PROJECT_EGRESS_TOKEN_SECRET="$(read_env_value "${existing_env}" "PROJECT_EGRESS_TOKEN_SECRET" || true)" # Only read OPENCLAW_TOKEN from env if not already set by install_ai_tools() # (which adopts the token from openclaw.json on fresh installs) if [[ -z "${OPENCLAW_TOKEN}" ]]; then OPENCLAW_TOKEN="$(read_env_value "${existing_env}" "OPENCLAW_GATEWAY_TOKEN" || true)" fi SETUP_TOKEN="$(read_env_value "${existing_env}" "SETUP_TOKEN" || true)" SETUP_TOKEN_EXPIRES_AT="$(read_env_value "${existing_env}" "SETUP_TOKEN_EXPIRES_AT" || true)" SETUP_TOKEN_USED_AT="$(read_env_value "${existing_env}" "SETUP_TOKEN_USED_AT" || true)" SETUP_SESSION_TOKEN_HASH="$(read_env_value "${existing_env}" "SETUP_SESSION_TOKEN_HASH" || true)" SETUP_SESSION_ORIGIN="$(read_env_value "${existing_env}" "SETUP_SESSION_ORIGIN" || true)" SETUP_SESSION_EXPIRES_AT="$(read_env_value "${existing_env}" "SETUP_SESSION_EXPIRES_AT" || true)" SETUP_HANDOFF_TOKEN_HASH="$(read_env_value "${existing_env}" "SETUP_HANDOFF_TOKEN_HASH" || true)" SETUP_HANDOFF_ORIGIN="$(read_env_value "${existing_env}" "SETUP_HANDOFF_ORIGIN" || true)" SETUP_HANDOFF_EXPIRES_AT="$(read_env_value "${existing_env}" "SETUP_HANDOFF_EXPIRES_AT" || true)" fi portal_app_sources_root="$( select_portal_app_sources_root "${existing_env}" "${pre_overlay_portal_version}" )" || fail "The standalone uploaded-App source root is unsafe or ambiguous." ensure_portal_app_sources_root "${portal_app_sources_root}" \ || fail "The standalone uploaded-App source root could not be created safely." [[ -n "${JWT_SECRET}" ]] || JWT_SECRET="$(rand_hex 32)" [[ -n "${JWT_REFRESH_SECRET}" ]] || JWT_REFRESH_SECRET="$(rand_hex 32)" [[ -n "${PORTAL_UPDATE_PROBE_TOKEN}" ]] || PORTAL_UPDATE_PROBE_TOKEN="$(rand_hex 32)" [[ -n "${OPENCLAW_TOKEN}" ]] || OPENCLAW_TOKEN="$(rand_hex 24)" if [[ -n "${PROJECT_EGRESS_TOKEN_SECRET}" ]]; then valid_project_egress_token_secret "${PROJECT_EGRESS_TOKEN_SECRET}" \ || fail "PROJECT_EGRESS_TOKEN_SECRET is malformed; refusing to rotate an active Project credential root automatically." else PROJECT_EGRESS_TOKEN_SECRET="$(rand_hex 32)" fi valid_project_egress_token_secret "${PROJECT_EGRESS_TOKEN_SECRET}" \ || fail "Could not provision a valid independent Project egress credential secret." # Mint a bootstrap secret only for genuinely fresh installs. A reinstall over an # existing account store (preserved env/DB) must not resurrect the setup # wizard: a fresh token there replaces the login page with the # password-reset flow for every logged-out visitor. local setup_now setup_token_rotated=false setup_now="$(date +%s)" if [[ -z "${SETUP_TOKEN}" && -z "${existing_database_url}" ]]; then SETUP_TOKEN="$(rand_hex 32)" SETUP_TOKEN_EXPIRES_AT="$((setup_now + 86400))" setup_token_rotated=true elif [[ -n "${SETUP_TOKEN}" ]] \ && { [[ -n "${SETUP_TOKEN_USED_AT}" ]] \ || [[ ! "${SETUP_TOKEN_EXPIRES_AT}" =~ ^[0-9]+$ ]] \ || (( SETUP_TOKEN_EXPIRES_AT <= setup_now )); }; then # Legacy, expired, or already-consumed bootstrap material is never printed # again. An explicit forced installer rerun rotates it and invalidates the # prior browser session, yielding one new resumable setup link. SETUP_TOKEN="$(rand_hex 32)" SETUP_TOKEN_EXPIRES_AT="$((setup_now + 86400))" setup_token_rotated=true fi if ${setup_token_rotated}; then SETUP_TOKEN_USED_AT="" SETUP_SESSION_TOKEN_HASH="" SETUP_SESSION_ORIGIN="" SETUP_SESSION_EXPIRES_AT="" SETUP_HANDOFF_TOKEN_HASH="" SETUP_HANDOFF_ORIGIN="" SETUP_HANDOFF_EXPIRES_AT="" elif [[ -z "${SETUP_TOKEN}" ]]; then SETUP_TOKEN_EXPIRES_AT="" SETUP_TOKEN_USED_AT="" SETUP_SESSION_TOKEN_HASH="" SETUP_SESSION_ORIGIN="" SETUP_SESSION_EXPIRES_AT="" SETUP_HANDOFF_TOKEN_HASH="" SETUP_HANDOFF_ORIGIN="" SETUP_HANDOFF_EXPIRES_AT="" fi ensure_telemetry_install_id # Build the DATABASE_URL — PRESERVE existing URL on updates to respect # custom port/database name. Only construct a new one for fresh installs. local final_database_url="" if [[ -n "${existing_database_url}" ]]; then final_database_url="${existing_database_url}" info "Preserving existing DATABASE_URL ($(echo "${existing_database_url}" | sed 's#://[^:]*:[^@]*@#://***:***@#'))" else [[ -n "${DB_PASSWORD}" ]] || DB_PASSWORD="$(rand_pass 24)" final_database_url="postgresql://blp:${DB_PASSWORD}@127.0.0.1:5432/bridgesllm_portal" fi # Write .env.production. Recover the served domain first: reinstalls reach # this point with an empty $DOMAIN while Caddy still serves the real vhost, # and DOMAIN/CORS/PORTAL_URL below must match it. recover_domain_from_caddyfile configure_app_content_identity local cors_origin cors_origin="$(portal_cors_origins)" local portal_url portal_url="$(portal_primary_origin)" local portal_public_origin="" if use_tailnet_profile && [[ -n "${TAILNET_DNS_NAME}" ]]; then portal_public_origin="https://${TAILNET_DNS_NAME}" fi cat > "${PORTAL_DIR}/backend/.env.production" << ENVEOF # Generated by BridgesLLM installer v${VERSION} — $(date) NODE_ENV=production PORT=4001 # Loopback bind: external access is served by Caddy over HTTPS. Widen only if # you know you need the portal reachable without the reverse proxy. HOST=127.0.0.1 DATABASE_URL="${final_database_url}" JWT_SECRET=${JWT_SECRET} JWT_REFRESH_SECRET=${JWT_REFRESH_SECRET} PORTAL_UPDATE_PROBE_TOKEN=${PORTAL_UPDATE_PROBE_TOKEN} PORTAL_ROOT=${PORTAL_DIR} PORTAL_APPS_ROOT=${portal_app_sources_root} PORTAL_APP_ZIPS_ROOT=${PORTAL_DIR}/upload-temp/app-zips INSTALL_ROOT=${INSTALL_ROOT} APPS_ROOT=${INSTALL_ROOT}/apps UPLOAD_DIR=${INSTALL_ROOT}/uploads CORS_ORIGIN=${cors_origin} PUBLIC_IP=${PUBLIC_IP} DOMAIN=${DOMAIN} PORTAL_URL=${portal_url} INSTALL_PROFILE=${INSTALL_PROFILE} ORIGIN_MODE=${ORIGIN_MODE} TAILNET_DNS_NAME=${TAILNET_DNS_NAME} PORTAL_PUBLIC_ORIGIN=${portal_public_origin} APP_CONTENT_DOMAIN=${APP_CONTENT_DOMAIN} APP_CONTENT_ORIGIN=${APP_CONTENT_ORIGIN} APP_CONTENT_DNS_MODE=${APP_CONTENT_DNS_MODE} OPENCLAW_API_URL=http://127.0.0.1:18789 OPENCLAW_GATEWAY_TOKEN=${OPENCLAW_TOKEN} OPENCLAW_PROJECT_SANDBOX_IMAGE_ID= CODEX_PROJECT_SANDBOX_IMAGE_ID= CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID= ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID= OLLAMA_PROJECT_SANDBOX_IMAGE_ID= AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID= PORTAL_PROJECT_RUNTIME_IMAGE_ID= PROJECT_RUNTIME_CONFINEMENT_POLICY= # Installer-attested Project egress plane. The credential root is independent # from Portal session/JWT secrets; image IDs are populated only after the exact # runtime recipes and compiled egress artifacts pass verification. PROJECT_EGRESS_PROXY_IMAGE_ID= PROJECT_EGRESS_TOKEN_SECRET=${PROJECT_EGRESS_TOKEN_SECRET} OLLAMA_API_URL=http://127.0.0.1:11434 # One-time setup bootstrap — exchanged only on HTTPS or true loopback and # cleared after wizard completion. Session/handoff values are hashes, not bearer secrets. SETUP_TOKEN=${SETUP_TOKEN} SETUP_TOKEN_EXPIRES_AT=${SETUP_TOKEN_EXPIRES_AT} SETUP_TOKEN_USED_AT=${SETUP_TOKEN_USED_AT} SETUP_SESSION_TOKEN_HASH=${SETUP_SESSION_TOKEN_HASH} SETUP_SESSION_ORIGIN=${SETUP_SESSION_ORIGIN} SETUP_SESSION_EXPIRES_AT=${SETUP_SESSION_EXPIRES_AT} SETUP_HANDOFF_TOKEN_HASH=${SETUP_HANDOFF_TOKEN_HASH} SETUP_HANDOFF_ORIGIN=${SETUP_HANDOFF_ORIGIN} SETUP_HANDOFF_EXPIRES_AT=${SETUP_HANDOFF_EXPIRES_AT} TELEMETRY_INSTALL_ID=${TELEMETRY_INSTALL_ID} ENVEOF chmod 600 "${PORTAL_DIR}/backend/.env.production" assert_prisma_runtime_environment_safe \ "${PORTAL_DIR}/backend/.env.production" \ || fail "The generated backend environment did not preserve the attested database runtime." # Create .env symlink so dotenv.config() finds the production env file ln -sf .env.production "${PORTAL_DIR}/backend/.env" ok "Configuration generated" # Frontend env printf 'VITE_API_URL=/api\n' > "${PORTAL_DIR}/frontend/.env" # Run migrations run_migrations_safe "${final_database_url}" # Build if [[ -f "${PORTAL_DIR}/frontend/dist/index.html" ]] \ && [[ -f "${PORTAL_DIR}/backend/dist/server.js" ]] \ && [[ -f "${PORTAL_DIR}/backend/dist/services/projectEgressPolicy.js" ]] \ && [[ -f "${PORTAL_DIR}/backend/dist/services/projectEgressProxy.js" ]]; then ok "Prebuilt artifacts detected — no compilation needed" else info "Packaged build artifacts missing — building from source" spin "Building frontend" "cd '${PORTAL_DIR}/frontend' && npm run build" spin "Building backend" "cd '${PORTAL_DIR}/backend' && npm run build" ok "Build complete" fi } # ═══════════════════════════════════════════════════════════════ # Step 6: Configure services # ═══════════════════════════════════════════════════════════════ configure_services() { step_header "Configuring services" CURRENT_STEP="service configuration" $DRY_RUN && { ok "[dry-run] Would create systemd service + Caddy config"; return; } # Systemd service cat > /etc/systemd/system/bridgesllm-product.service << SVCEOF [Unit] Description=BridgesLLM Portal After=network.target postgresql.service [Service] Type=simple User=root WorkingDirectory=${PORTAL_DIR}/backend EnvironmentFile=${PORTAL_DIR}/backend/.env.production ExecStart=/usr/bin/node dist/server.js Restart=always RestartSec=5 StandardOutput=journal StandardError=journal [Install] WantedBy=multi-user.target SVCEOF install_portal_update_boot_fence \ || fail "Could not install the Portal update boot-safety fence." systemctl daemon-reload systemctl enable bridgesllm-product >> "$LOG_FILE" 2>&1 ok "Portal service created" # Caddy / direct local access if use_local_profile; then ok "Direct local access configured ${DIM}(http://localhost:4001)${NC}" elif use_tailnet_profile; then # Tailscale Serve owns TLS and routing; Caddy stays out of the picture and # no public listener exists. ok "Tailnet origin configured ${DIM}(https://${TAILNET_DNS_NAME} via Tailscale Serve)${NC}" else write_caddy_config systemctl enable caddy >> "$LOG_FILE" 2>&1 systemctl is-active --quiet caddy \ || systemctl start caddy >> "$LOG_FILE" 2>&1 \ || fail "Caddy could not be started after its configuration was verified." ok "Web server configured" fi # Install both the system-unit fence and the permanent root user-unit # inhibitor even when OpenClaw is deliberately skipped. A later package or # legacy user unit must never acquire unfenced gateway authority. install_openclaw_gateway_authorization_fence_dropin \ || fail "Could not install the OpenClaw gateway authorization fence." # OpenClaw gateway service (if installed) if ! $SKIP_OPENCLAW && command -v openclaw &>/dev/null; then local oc_bin oc_bin="$(which openclaw 2>/dev/null || echo '/usr/bin/openclaw')" cat > /etc/systemd/system/openclaw-gateway.service << OCSVCEOF [Unit] Description=OpenClaw AI Gateway After=network-online.target Wants=network-online.target [Service] Type=simple User=root ExecStart=${oc_bin} gateway --port 18789 Restart=always RestartSec=5 TimeoutStopSec=30 TimeoutStartSec=30 SuccessExitStatus=0 143 KillMode=control-group StandardOutput=journal StandardError=journal [Install] WantedBy=multi-user.target OCSVCEOF systemctl daemon-reload systemctl enable openclaw-gateway >> "$LOG_FILE" 2>&1 || true ok "OpenClaw gateway service configured" fi # Firewall if use_local_profile; then ok "Firewall skipped in local mode" elif use_tailnet_profile; then # No public web ports at all: Tailscale's WireGuard traffic flows over its # own UDP path and the portal is reachable only through the tailnet. ufw allow 22/tcp >> "$LOG_FILE" 2>&1 ufw deny 4001/tcp >> "$LOG_FILE" 2>&1 ufw --force enable >> "$LOG_FILE" 2>&1 || true ok "Firewall configured (no public web ports — tailnet-only access)" else ufw allow 22/tcp >> "$LOG_FILE" 2>&1 ufw allow 80/tcp >> "$LOG_FILE" 2>&1 ufw allow 443/tcp >> "$LOG_FILE" 2>&1 ufw deny 4001/tcp >> "$LOG_FILE" 2>&1 ufw --force enable >> "$LOG_FILE" 2>&1 || true ok "Firewall configured" fi } # ═══════════════════════════════════════════════════════════════ # Step 7: Remote Desktop # ═══════════════════════════════════════════════════════════════ setup_remote_desktop() { step_header "Setting up Remote Desktop" CURRENT_STEP="remote desktop" $DRY_RUN && { ok "[dry-run] Would configure Remote Desktop"; return; } ensure_remote_desktop_packages local RD_USER="bridgesrd" local XDG_DIR="/tmp/bridges-rd-runtime" local LOG_RD="/var/log/bridges-rd" # Create service user if id "$RD_USER" &>/dev/null; then ok "User $RD_USER exists" else useradd -r -m -s /bin/bash "$RD_USER" ok "Created user $RD_USER" fi mkdir -p "$XDG_DIR" "$LOG_RD" chown "$RD_USER:$RD_USER" "$XDG_DIR" "$LOG_RD" chmod 700 "$XDG_DIR" # Deploy the managed XFCE desktop config (Greybird theme, panel layout, # keyboard shortcuts, session settings, and related compatibility defaults). # Only deploy if this is a fresh install or the config dir doesn't exist yet. local xfce_config_dir="/home/$RD_USER/.config/xfce4" local xfce_source="${PORTAL_DIR}/installer/xfce4-config" if [[ -d "$xfce_source" ]]; then mkdir -p "$xfce_config_dir" rsync -a "$xfce_source/" "$xfce_config_dir/" fi chown -R "$RD_USER:$RD_USER" "/home/$RD_USER/.config" ok "Desktop environment configured" # Web browser launcher (for noVNC URL bar) local web_launcher="/usr/local/bin/bridges-rd-web-open.sh" cat > "$web_launcher" << 'WEBEOF' #!/bin/bash URL="${1:-about:blank}" if command -v google-chrome-stable &>/dev/null; then exec google-chrome-stable --start-maximized "$URL" 2>/dev/null elif command -v firefox &>/dev/null; then exec firefox "$URL" 2>/dev/null else echo "No browser found" >&2 exit 1 fi WEBEOF chmod 755 "$web_launcher" # Branded Remote Desktop launchers/icons. Shared Browser keeps the agent-controlled # reset-on-open profile; OpenClaw Web UI gets its own persistent profile so it # does not get confused with normal Chrome or the shared browser. local shared_browser_launcher="/usr/local/bin/bridges-rd-shared-chrome.sh" local openclaw_ui_launcher="/usr/local/bin/bridges-rd-openclaw-ui.sh" local ai_provider_launcher="/usr/local/bin/bridges-rd-ai-launchers.sh" local shared_browser_src="${PORTAL_DIR}/static/scripts/bridges-rd-shared-chrome.sh" local openclaw_ui_src="${PORTAL_DIR}/static/scripts/bridges-rd-openclaw-ui.sh" local ai_provider_src="${PORTAL_DIR}/static/scripts/bridges-rd-ai-launchers.sh" [[ -f "$shared_browser_src" ]] || fail "Signed Shared Browser launcher is missing" [[ -f "$openclaw_ui_src" ]] || fail "Signed OpenClaw UI launcher is missing" [[ -f "$ai_provider_src" ]] || fail "Signed AI provider launcher is missing" if [[ -f "$shared_browser_src" ]]; then install -m 755 "$shared_browser_src" "$shared_browser_launcher" fi if [[ -f "$openclaw_ui_src" ]]; then install -m 755 "$openclaw_ui_src" "$openclaw_ui_launcher" fi if [[ -f "$ai_provider_src" ]]; then install -m 755 "$ai_provider_src" "$ai_provider_launcher" fi mkdir -p /usr/local/share/pixmaps "/home/$RD_USER/Desktop" local shared_browser_icon="/usr/local/share/pixmaps/bridges-shared-browser.svg" local openclaw_ui_icon="/usr/local/share/pixmaps/bridges-openclaw-ui.svg" [[ -f "${PORTAL_DIR}/static/icons/bridges-shared-browser.svg" ]] && install -m 644 "${PORTAL_DIR}/static/icons/bridges-shared-browser.svg" "$shared_browser_icon" [[ -f "${PORTAL_DIR}/static/icons/bridges-openclaw-ui.svg" ]] && install -m 644 "${PORTAL_DIR}/static/icons/bridges-openclaw-ui.svg" "$openclaw_ui_icon" local shared_browser_state_dir="/home/$RD_USER/.config/bridges-agent-browser" local shared_browser_log_dir="$shared_browser_state_dir/logs" local openclaw_ui_profile_dir="/home/$RD_USER/.config/openclaw-control-ui-browser" local openclaw_ui_url_file="$openclaw_ui_profile_dir/dashboard-url" local openclaw_ui_launch_html="$openclaw_ui_profile_dir/launch.html" install -d -o "$RD_USER" -g "$RD_USER" -m 0700 \ "$shared_browser_state_dir" "$shared_browser_log_dir" "$openclaw_ui_profile_dir" OPENCLAW_UI_URL_FILE="$openclaw_ui_url_file" OPENCLAW_UI_LAUNCH_HTML="$openclaw_ui_launch_html" python3 - <<'PY' import html import json import os from pathlib import Path from urllib.parse import quote base = 'http://127.0.0.1:18789/' token = '' config = Path('/root/.openclaw/openclaw.json') try: if config.exists(): token = (json.loads(config.read_text()).get('gateway') or {}).get('auth', {}).get('token') or '' except Exception: token = '' if not token: token_file = Path('/root/.openclaw/gateway.token') try: if token_file.exists(): token = token_file.read_text().strip() except Exception: token = '' url = base + (('#token=' + quote(token, safe='')) if token else '') Path(os.environ['OPENCLAW_UI_URL_FILE']).write_text(url + '\n') Path(os.environ['OPENCLAW_UI_LAUNCH_HTML']).write_text(f''' Opening OpenClaw Web UI…
🦞

Opening OpenClaw Web UI…

''') PY chown -R "$RD_USER:$RD_USER" "$openclaw_ui_profile_dir" chmod 700 "$shared_browser_state_dir" "$shared_browser_log_dir" "$openclaw_ui_profile_dir" chmod 600 "$openclaw_ui_url_file" "$openclaw_ui_launch_html" cat > "/home/$RD_USER/Desktop/Shared Chrome.desktop" < "/home/$RD_USER/Desktop/OpenClaw Web UI.desktop" </dev/null 2>&1 && gio set '/home/$RD_USER/Desktop/Shared Chrome.desktop' metadata::trusted true || true; command -v gio >/dev/null 2>&1 && gio set '/home/$RD_USER/Desktop/OpenClaw Web UI.desktop' metadata::trusted true || true" || true "$ai_provider_launcher" install --assets-dir "${PORTAL_DIR}/static/icons" >> "$LOG_FILE" 2>&1 \ || fail "AI runtime Remote Desktop launchers could not be provisioned." ps -eo pid=,args= | awk '/[O]penClawControlUI/ && /[#]token=/ {print $1}' | xargs -r kill || true ok "Remote Desktop browser and AI runtime launchers configured" # Install the canonical signed Remote Desktop runtime. A release missing any # of these files is invalid; divergent embedded fallbacks are deliberately # not maintained. local vnc_launcher="/usr/local/bin/bridges-rd-xtigervnc-start.sh" local bundled_vnc_launcher="$PORTAL_DIR/installer/scripts/bridges-rd-xtigervnc-start.sh" local session_guard="/usr/local/bin/bridges-rd-session-guard.sh" local bundled_session_guard="$PORTAL_DIR/installer/scripts/bridges-rd-session-guard.sh" local healthcheck="/usr/local/bin/bridges-rd-healthcheck.sh" local bundled_healthcheck="$PORTAL_DIR/installer/scripts/bridges-rd-healthcheck.sh" local bundled_window_fit="$PORTAL_DIR/installer/scripts/bridges-rd-window-fit.sh" if [[ -r "$bundled_window_fit" ]]; then install -o root -g root -m 0755 "$bundled_window_fit" /usr/local/bin/bridges-rd-window-fit.sh else warn "Remote Desktop window-fit helper missing; dynamic desktop resizing will not reposition windows" fi if [[ -r "$bundled_vnc_launcher" ]]; then install -o root -g root -m 0755 "$bundled_vnc_launcher" "$vnc_launcher" else fail "Canonical Remote Desktop launcher is missing from the signed Portal bundle" fi if [[ -r "$bundled_session_guard" ]]; then install -o root -g root -m 0755 "$bundled_session_guard" "$session_guard" else fail "Canonical Remote Desktop session guard is missing from the signed Portal bundle" fi if [[ -r "$bundled_healthcheck" ]]; then install -o root -g root -m 0755 "$bundled_healthcheck" "$healthcheck" else fail "Canonical Remote Desktop healthcheck is missing from the signed Portal bundle" fi chmod 755 "$vnc_launcher" ok "VNC launcher, semantic session guard, and automatic recovery healthcheck written" # Systemd units cat > /etc/systemd/system/bridges-rd-xtigervnc.service << VNCSVCEOF [Unit] Description=Bridges Remote Desktop Xtigervnc :1 After=network.target systemd-tmpfiles-setup.service systemd-user-sessions.service Before=bridges-rd-websockify.service Conflicts=tigervncserver@:1.service tigervncserver@1.service vncserver@:1.service vncserver@1.service RequiresMountsFor=/home/bridgesrd /var/log/bridges-rd StartLimitIntervalSec=600 StartLimitBurst=3 [Service] Type=notify NotifyAccess=main User=root ExecStart=${vnc_launcher} ExecStopPost=-/bin/bash -c 'pkill -f "Xtigervnc :1" 2>/dev/null || true' Restart=on-failure RestartSec=3 WatchdogSec=45 TimeoutStartSec=120 TimeoutStopSec=20 KillMode=control-group Environment=HOME=/root [Install] WantedBy=multi-user.target VNCSVCEOF cat > /etc/systemd/system/bridges-rd-websockify.service << WSSVCEOF [Unit] Description=Bridges Remote Desktop noVNC Websockify After=network.target bridges-rd-xtigervnc.service Requires=bridges-rd-xtigervnc.service [Service] Type=simple User=bridgesrd Group=bridgesrd ExecStart=/usr/bin/python3 /usr/bin/websockify 127.0.0.1:6080 127.0.0.1:5901 Restart=always RestartSec=3 UMask=0077 NoNewPrivileges=true PrivateTmp=true PrivateDevices=true ProtectSystem=full ProtectHome=true ProtectKernelTunables=true ProtectKernelModules=true ProtectControlGroups=true RestrictSUIDSGID=true LockPersonality=true CapabilityBoundingSet= AmbientCapabilities= RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 [Install] WantedBy=multi-user.target WSSVCEOF cat > /etc/systemd/system/bridges-rd-healthcheck.service << HEALTHSVCEOF [Unit] Description=Bridges Remote Desktop semantic health recovery After=bridges-rd-xtigervnc.service bridges-rd-websockify.service [Service] Type=oneshot User=root ExecStart=${healthcheck} TimeoutStartSec=150 UMask=0077 HEALTHSVCEOF cat > /etc/systemd/system/bridges-rd-healthcheck.timer << 'HEALTHTIMEREOF' [Unit] Description=Check and recover Bridges Remote Desktop [Timer] OnBootSec=45s OnUnitActiveSec=30s AccuracySec=5s Unit=bridges-rd-healthcheck.service Persistent=true [Install] WantedBy=timers.target HEALTHTIMEREOF systemctl daemon-reload # Disable every stock TigerVNC display-1 alias. Ubuntu ships # tigervncserver@.service; older images used vncserver@.service. Leaving even # one alias enabled creates a cold-boot race for display :1. systemctl disable --now \ tigervncserver@:1.service tigervncserver@1.service \ vncserver@:1.service vncserver@1.service 2>/dev/null || true local stock_vnc_unit stock_vnc_state for stock_vnc_unit in \ tigervncserver@:1.service tigervncserver@1.service \ vncserver@:1.service vncserver@1.service; do if systemctl is-active --quiet "$stock_vnc_unit"; then fail "Conflicting stock Remote Desktop unit ${stock_vnc_unit} could not be stopped" fi systemctl mask "$stock_vnc_unit" >/dev/null 2>&1 \ || fail "Conflicting stock Remote Desktop unit ${stock_vnc_unit} could not be masked" stock_vnc_state="$(systemctl is-enabled "$stock_vnc_unit" 2>/dev/null || true)" [[ "$stock_vnc_state" == "masked" ]] \ || fail "Conflicting stock Remote Desktop unit ${stock_vnc_unit} is not masked" done # Retire only the complete legacy Bridges rc.local script. Substring edits # are unsafe here: they can mutate an administrator-owned boot script and # leave the legacy root XFCE/Xvfb cgroup alive. The second exact variant is # a Portal-owned variant that commented only its vncserver line; it still # launches the obsolete :99 desktop. Unknown or near-match files remain # byte-for-byte untouched. local rd_legacy_rc_retired rd_legacy_rc_retired="$(python3 - <<'PY' import os from pathlib import Path path = Path('/etc/rc.local') archive = Path('/etc/rc.local.bridgesllm-remote-desktop-legacy') original = '''#!/bin/bash # Start XFCE desktop on system boot export DISPLAY=:99 (/usr/bin/Xvfb :99 -screen 0 1920x1080x24 -ac &) sleep 3 (DISPLAY=:99 /usr/bin/startxfce4 &) sleep 3 (DISPLAY=:99 /usr/bin/vncserver :1 -geometry 1920x1080 -depth 24 -fg 2>/dev/null &) exit 0 ''' interim = '''#!/bin/bash # Start XFCE desktop on system boot export DISPLAY=:99 (/usr/bin/Xvfb :99 -screen 0 1920x1080x24 -ac &) sleep 3 (DISPLAY=:99 /usr/bin/startxfce4 &) sleep 3 # Display :1 is owned by bridges-rd-xtigervnc.service. The legacy vncserver # launch raced the managed Remote Desktop after reboot and could expose an # unusable greeter instead of the Portal desktop. # (DISPLAY=:99 /usr/bin/vncserver :1 -geometry 1920x1080 -depth 24 -fg 2>/dev/null &) exit 0 ''' if not path.exists(): print('false') elif path.is_symlink(): print('false') else: current = path.read_text() if current not in {original, interim}: print('false') else: if archive.exists(): if archive.is_symlink() or archive.read_text() != current: raise SystemExit('legacy rc.local archive already exists with different content') path.unlink() else: os.replace(path, archive) os.chown(archive, 0, 0) os.chmod(archive, 0o600) print('true') PY )" if [[ "$rd_legacy_rc_retired" == "true" ]]; then systemctl stop rc-local.service >/dev/null 2>&1 \ || fail "The retired legacy Remote Desktop rc.local cgroup could not be stopped" local rc_local_state rc_local_state="$(systemctl show --property=ActiveState --value rc-local.service 2>/dev/null || true)" [[ "$rc_local_state" == "inactive" || "$rc_local_state" == "failed" ]] \ || fail "The retired legacy Remote Desktop rc.local cgroup is still active" ok "Retired the exact legacy Remote Desktop rc.local boot stack" elif [[ -e /etc/rc.local ]]; then warn "Preserved an unrecognized /etc/rc.local; no administrator-owned boot commands were changed" fi # Disable legacy service name if present systemctl disable --now bridges-rd-vnc.service 2>/dev/null || true systemctl enable \ bridges-rd-xtigervnc.service \ bridges-rd-websockify.service \ bridges-rd-healthcheck.timer >> "$LOG_FILE" 2>&1 \ || fail "Remote Desktop services and automatic recovery timer could not be enabled" systemctl is-enabled --quiet bridges-rd-healthcheck.timer \ || fail "Remote Desktop automatic recovery timer is not enabled" systemctl restart bridges-rd-xtigervnc.service >> "$LOG_FILE" 2>&1 sleep 2 systemctl restart bridges-rd-websockify.service >> "$LOG_FILE" 2>&1 systemctl restart bridges-rd-healthcheck.timer >> "$LOG_FILE" 2>&1 \ || fail "Remote Desktop automatic recovery timer could not be started" systemctl start bridges-rd-healthcheck.service >> "$LOG_FILE" 2>&1 \ || fail "Remote Desktop automatic health state could not be initialized" local rd_ready=false local rd_deadline=$((SECONDS + 20)) while (( SECONDS < rd_deadline )); do if systemctl is-active --quiet bridges-rd-xtigervnc.service \ && systemctl is-active --quiet bridges-rd-websockify.service \ && systemctl is-active --quiet bridges-rd-healthcheck.timer \ && ss -H -ltn 'sport = :5901' | grep -q . \ && ss -H -ltn 'sport = :6080' | grep -q . \ && "$session_guard" check >/dev/null 2>&1; then rd_ready=true break fi sleep 1 done [[ "$rd_ready" == "true" ]] \ || fail "Remote Desktop services, semantic session, and automatic recovery timer did not become ready" local rd_port for rd_port in 5901 6080; do if ! ss -H -ltn "sport = :$rd_port" | awk ' { address=$4; found=1; if (address !~ /^127\.0\.0\.1:/ && address !~ /^\[::1\]:/ && address !~ /^::1:/) bad=1 } END { exit (!found || bad) } '; then fail "Remote Desktop port $rd_port is missing or exposed beyond loopback" fi done local rd_vnc_process rd_vnc_process="$(ps -eo args= | grep '[X]tigervnc :1' | head -n 1 || true)" if [[ -z "$rd_vnc_process" || "$rd_vnc_process" != *"-localhost=1"* \ || "$rd_vnc_process" != *"-auth /home/bridgesrd/.Xauthority"* \ || " $rd_vnc_process " == *" -ac "* ]]; then fail "Remote Desktop VNC process failed the loopback/Xauthority policy check" fi local rd_websockify_process rd_websockify_process="$(ps -eo user:64=,args= | awk '$1 == "bridgesrd" && $0 ~ /[w]ebsockify 127.0.0.1:6080 127.0.0.1:5901/ { $1=""; sub(/^ +/, ""); print; exit }' || true)" [[ -n "$rd_websockify_process" ]] || fail "Remote Desktop websockify process failed the exact loopback bridge policy check" local rd_private_state for rd_private_state in \ "/home/$RD_USER/.config/bridges-agent-browser" \ "/home/$RD_USER/.config/bridges-agent-browser/logs"; do [[ -d "$rd_private_state" && ! -L "$rd_private_state" ]] \ || fail "Remote Desktop private browser state is missing or linked: $rd_private_state" [[ "$(stat -c '%U:%G:%a' "$rd_private_state")" == "$RD_USER:$RD_USER:700" ]] \ || fail "Remote Desktop private browser state has unsafe ownership or mode: $rd_private_state" done local rd_managed_marker="/home/$RD_USER/.bridgesllm-portal-managed" RD_MANAGED_MARKER="${rd_managed_marker}" python3 - <<'PY' import os path = os.environ["RD_MANAGED_MARKER"] flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW fd = os.open(path, flags, 0o600) try: os.write(fd, b"managed-by=bridgesllm-portal-remote-desktop-v1\n") os.fchmod(fd, 0o600) # This marker authorizes clean-slate account deletion. Keep it outside the # control of the desktop account it attests. os.fchown(fd, 0, 0) finally: os.close(fd) PY ok "Remote Desktop services started and security policy verified" } # ═══════════════════════════════════════════════════════════════ # Step 8: Start # ═══════════════════════════════════════════════════════════════ ensure_openclaw_gateway_boots_cleanly() { if ! systemctl is-enabled openclaw-gateway &>/dev/null 2>&1; then return 0 fi systemctl start openclaw-gateway >> "$LOG_FILE" 2>&1 || true if wait_for_openclaw_gateway_http_ready 60; then return 0 fi warn "OpenClaw gateway did not come up cleanly. Clearing stale gateway processes and retrying once." systemctl stop openclaw-gateway >> "$LOG_FILE" 2>&1 || true pkill -f 'openclaw-gateway|/usr/bin/openclaw gateway|/usr/local/bin/openclaw gateway|openclaw$' >> "$LOG_FILE" 2>&1 || true sleep 2 systemctl start openclaw-gateway >> "$LOG_FILE" 2>&1 || true if wait_for_openclaw_gateway_http_ready 60; then return 0 fi warn "OpenClaw gateway still did not answer on 127.0.0.1:18789 after retry." return 1 } openclaw_cli_version() { command -v openclaw >/dev/null 2>&1 || return 1 OPENCLAW_ALLOW_ROOT=1 openclaw --version 2>/dev/null | head -1 | grep -oP '\d{4}\.\d+\.\d+' || true } openclaw_gateway_version() { command -v openclaw >/dev/null 2>&1 || return 1 OPENCLAW_ALLOW_ROOT=1 openclaw gateway status --deep --json 2>/dev/null | node -e ' let input = ""; process.stdin.on("data", chunk => input += chunk); process.stdin.on("end", () => { try { const data = JSON.parse(input); const raw = (data.gateway && data.gateway.version) || data.runningVersion || ""; const match = String(raw).match(/\d{4}\.\d+\.\d+/); console.log(match ? match[0] : ""); } catch (_) { console.log(""); } }); ' 2>/dev/null || true } ensure_openclaw_gateway_matches_cli() { if $SKIP_OPENCLAW || ! command -v openclaw >/dev/null 2>&1; then return 0 fi if ! systemctl is-enabled openclaw-gateway &>/dev/null 2>&1; then return 0 fi local cli_version gateway_version cli_version="$(openclaw_cli_version)" gateway_version="$(openclaw_gateway_version)" if [[ -n "${cli_version}" && -n "${gateway_version}" && "${cli_version}" == "${gateway_version}" ]]; then if OPENCLAW_ALLOW_ROOT=1 openclaw gateway status --require-rpc --timeout 10000 >> "$LOG_FILE" 2>&1; then ok "OpenClaw gateway ${gateway_version} matches CLI" return 0 fi warn "OpenClaw gateway version matches, but RPC readiness failed. Restarting once." else warn "OpenClaw gateway version mismatch detected (CLI: ${cli_version:-unknown}, gateway: ${gateway_version:-unknown}). Restarting gateway." fi if ! spin "Restarting OpenClaw gateway for version parity" "systemctl restart openclaw-gateway"; then warn "OpenClaw gateway restart failed." return 1 fi local waited=0 while (( waited < 60 )); do if openclaw_gateway_http_ready; then cli_version="$(openclaw_cli_version)" gateway_version="$(openclaw_gateway_version)" if [[ -n "${cli_version}" && -n "${gateway_version}" && "${cli_version}" == "${gateway_version}" ]] \ && OPENCLAW_ALLOW_ROOT=1 openclaw gateway status --require-rpc --timeout 10000 >> "$LOG_FILE" 2>&1; then ok "OpenClaw gateway ${gateway_version} matches CLI" return 0 fi fi sleep 3 waited=$((waited + 3)) done warn "OpenClaw gateway still mismatched after restart (CLI: ${cli_version:-unknown}, gateway: ${gateway_version:-unknown})." return 1 } valid_local_image_tag() { local image="${1:-}" [[ "${image}" =~ ^[a-z0-9][a-z0-9._/-]*(:[A-Za-z0-9._-]+)?$ ]] } valid_docker_image_id() { local image_id="${1:-}" [[ "${image_id}" =~ ^sha256:[a-f0-9]{64}$ ]] } docker_image_label() { local image="$1" local label="$2" docker image inspect --format "{{ index .Config.Labels \"${label}\" }}" "${image}" 2>/dev/null } docker_image_id() { local image="$1" docker image inspect --format '{{.Id}}' "${image}" 2>/dev/null } capture_managed_project_egress_inventory() ( # Capture both discovery authorities. Reserved deterministic names find # partially labelled planes; policy labels find managed claimants whose # names drifted. Neither inventory is trusted unless both resolve to the # same immutable resources and every full identity has the exact topology # the backend will attest during Project-provider qualification. local all_containers managed_containers all_networks managed_networks local row resource_id resource_name extra found snapshot_root local container_format network_format local -a reserved_proxy_ids=() local -a labelled_proxy_ids=() local -a reserved_network_ids=() local -a labelled_network_ids=() local -A seen_resource_ids=() report_project_egress_inventory_failure() { local reason="$1" local source row diagnostic_id diagnostic_name extra separator="" local resources="" container_command="" network_command="" local -A reported=() for source in "${all_containers:-}" "${managed_containers:-}"; do while IFS= read -r row; do [[ -n "${row}" ]] || continue read -r diagnostic_id diagnostic_name extra <<<"${row}" [[ -z "${extra:-}" \ && "${diagnostic_id}" =~ ^[a-f0-9]{64}$ \ && "${diagnostic_name}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ \ && ( "${diagnostic_name}" =~ ^p4e-proxy-[a-f0-9]{20}$ \ || "${source}" == "${managed_containers:-}" ) ]] || continue [[ -z "${reported["container:${diagnostic_id}"]:-}" ]] || continue reported["container:${diagnostic_id}"]=1 resources+="${separator}proxy ${diagnostic_name} (${diagnostic_id})" separator=", " container_command+=" ${diagnostic_id}" done <<<"${source}" done for source in "${all_networks:-}" "${managed_networks:-}"; do while IFS= read -r row; do [[ -n "${row}" ]] || continue read -r diagnostic_id diagnostic_name extra <<<"${row}" [[ -z "${extra:-}" \ && "${diagnostic_id}" =~ ^[a-f0-9]{64}$ \ && "${diagnostic_name}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ \ && ( "${diagnostic_name}" =~ ^p4e-(in|out)-[a-f0-9]{20}$ \ || "${source}" == "${managed_networks:-}" ) ]] || continue [[ -z "${reported["network:${diagnostic_id}"]:-}" ]] || continue reported["network:${diagnostic_id}"]=1 resources+="${separator}network ${diagnostic_name} (${diagnostic_id})" separator=", " network_command+=" ${diagnostic_id}" done <<<"${source}" done printf '%s\n' \ "Managed Project egress inventory validation failed (${reason}) for ${resources:-unresolved reserved resources}." >&2 if [[ -n "${container_command}" || -n "${network_command}" ]]; then printf 'Remediation:' >&2 [[ -z "${container_command}" ]] \ || printf ' docker container inspect%s;' "${container_command}" >&2 [[ -z "${network_command}" ]] \ || printf ' docker network inspect%s;' "${network_command}" >&2 printf '%s\n' \ ' stop attached Project runtimes, then retry the Portal update. Remove only a fully empty parentless network pair.' >&2 else printf '%s\n' \ 'Remediation: docker container ls --all --no-trunc; docker network ls --no-trunc; stop attached Project runtimes, then retry the Portal update.' >&2 fi } all_containers="$(docker container ls --all --no-trunc \ --format '{{.ID}} {{.Names}}' 2>/dev/null | LC_ALL=C sort)" \ || return 2 managed_containers="$(docker container ls --all --no-trunc \ --filter "label=${PROJECT_EGRESS_POLICY_LABEL}=${PROJECT_EGRESS_POLICY_VERSION}" \ --format '{{.ID}} {{.Names}}' 2>/dev/null | LC_ALL=C sort)" \ || return 2 all_networks="$(docker network ls --no-trunc \ --format '{{.ID}} {{.Name}}' 2>/dev/null | LC_ALL=C sort)" \ || return 2 managed_networks="$(docker network ls --no-trunc \ --filter "label=${PROJECT_EGRESS_POLICY_LABEL}=${PROJECT_EGRESS_POLICY_VERSION}" \ --format '{{.ID}} {{.Name}}' 2>/dev/null | LC_ALL=C sort)" \ || return 2 while IFS= read -r row; do [[ -n "${row}" ]] || continue read -r resource_id resource_name extra <<<"${row}" [[ -z "${extra:-}" && "${resource_id}" =~ ^[a-f0-9]{64}$ ]] || { report_project_egress_inventory_failure "invalid container inventory" return 3 } [[ -z "${seen_resource_ids["container:${resource_id}"]:-}" ]] \ || { report_project_egress_inventory_failure "duplicate container identity" return 3 } seen_resource_ids["container:${resource_id}"]=1 if [[ "${resource_name}" =~ ^p4e-proxy-[a-f0-9]{20}$ ]]; then reserved_proxy_ids+=("${resource_id}") fi done <<<"${all_containers}" while IFS= read -r row; do [[ -n "${row}" ]] || continue read -r resource_id resource_name extra <<<"${row}" [[ -z "${extra:-}" \ && "${resource_id}" =~ ^[a-f0-9]{64}$ \ && "${resource_name}" =~ ^p4e-proxy-[a-f0-9]{20}$ ]] \ || { report_project_egress_inventory_failure "managed proxy name or identity drift" return 3 } labelled_proxy_ids+=("${resource_id}") done <<<"${managed_containers}" while IFS= read -r row; do [[ -n "${row}" ]] || continue read -r resource_id resource_name extra <<<"${row}" [[ -z "${extra:-}" && "${resource_id}" =~ ^[a-f0-9]{64}$ ]] || { report_project_egress_inventory_failure "invalid network inventory" return 3 } [[ -z "${seen_resource_ids["network:${resource_id}"]:-}" ]] \ || { report_project_egress_inventory_failure "duplicate network identity" return 3 } seen_resource_ids["network:${resource_id}"]=1 if [[ "${resource_name}" =~ ^p4e-(in|out)-[a-f0-9]{20}$ ]]; then reserved_network_ids+=("${resource_id}") fi done <<<"${all_networks}" while IFS= read -r row; do [[ -n "${row}" ]] || continue read -r resource_id resource_name extra <<<"${row}" [[ -z "${extra:-}" \ && "${resource_id}" =~ ^[a-f0-9]{64}$ \ && "${resource_name}" =~ ^p4e-(in|out)-[a-f0-9]{20}$ ]] \ || { report_project_egress_inventory_failure "managed network name or identity drift" return 3 } labelled_network_ids+=("${resource_id}") done <<<"${managed_networks}" [[ "${#reserved_proxy_ids[@]}" -eq "${#labelled_proxy_ids[@]}" \ && "${#reserved_network_ids[@]}" -eq "${#labelled_network_ids[@]}" ]] \ || { report_project_egress_inventory_failure "reserved names and managed labels disagree" return 3 } for resource_id in "${reserved_proxy_ids[@]}"; do found=false for actual_id in "${labelled_proxy_ids[@]}"; do if [[ "${actual_id}" == "${resource_id}" ]]; then found=true break fi done ${found} || { report_project_egress_inventory_failure "reserved proxy lacks exact managed ownership" return 3 } done for resource_id in "${reserved_network_ids[@]}"; do found=false for actual_id in "${labelled_network_ids[@]}"; do if [[ "${actual_id}" == "${resource_id}" ]]; then found=true break fi done ${found} || { report_project_egress_inventory_failure "reserved network lacks exact managed ownership" return 3 } done if ((${#reserved_proxy_ids[@]} == 0 \ && ${#reserved_network_ids[@]} == 0)); then return 1 fi snapshot_root="$(mktemp -d)" || return 2 [[ -n "${snapshot_root}" && -d "${snapshot_root}" ]] || return 2 chmod 0700 "${snapshot_root}" || return 2 trap 'rm -rf -- "${snapshot_root}"' EXIT mkdir -m 0700 "${snapshot_root}/containers" "${snapshot_root}/networks" \ || return 2 printf '%s\n' "${all_containers}" > "${snapshot_root}/container-list" printf '%s\n' "${all_networks}" > "${snapshot_root}/network-list" # Selected JSON lines deliberately omit environment and mount contents. They # capture every identity/topology field needed for a stable comparison # without copying Project credentials into the update transaction. container_format='{{json .Id}} {{json .Name}} {{json .Image}} {{json .Config.Image}} {{json .Config.Labels}} {{json .HostConfig.NetworkMode}} {{json .NetworkSettings.Networks}} {{json .State.Running}}' network_format='{{json .Id}} {{json .Name}} {{json .Driver}} {{json .Internal}} {{json .Attachable}} {{json .Ingress}} {{json .EnableIPv6}} {{json .Labels}} {{json .Options}} {{json .IPAM}} {{json .Containers}}' while IFS= read -r row; do [[ -n "${row}" ]] || continue read -r resource_id resource_name extra <<<"${row}" [[ -z "${extra:-}" && "${resource_id}" =~ ^[a-f0-9]{64}$ ]] \ || return 3 docker container inspect --format "${container_format}" \ "${resource_id}" > "${snapshot_root}/containers/${resource_id}" 2>/dev/null \ || return 2 done <<<"${all_containers}" for resource_id in "${reserved_network_ids[@]}"; do docker network inspect --format "${network_format}" \ "${resource_id}" > "${snapshot_root}/networks/${resource_id}" 2>/dev/null \ || return 2 done node - "${snapshot_root}" \ "${PROJECT_EGRESS_POLICY_VERSION}" \ "${PROJECT_EGRESS_POLICY_LABEL}" \ "${PROJECT_EGRESS_ROLE_LABEL}" \ "${PROJECT_EGRESS_IDENTITY_LABEL}" \ "${PROJECT_EGRESS_ACTOR_LABEL}" \ "${PROJECT_EGRESS_PROJECT_LABEL}" \ "${PROJECT_EGRESS_PROVIDER_LABEL}" \ "${PROJECT_EGRESS_CONSUMER_KIND_LABEL}" \ "${PROJECT_EGRESS_WORKLOAD_LABEL}" \ "${PROJECT_EGRESS_FINGERPRINT_LABEL}" \ "${PROJECT_EGRESS_TOKEN_HASH_LABEL}" \ "${PROJECT_EGRESS_RUNTIME_FINGERPRINT_LABEL}" <<'NODE' const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); const inputArgs = process.argv.slice(2); const diagnosticSnapshotRoot = inputArgs[0]; try { const [ snapshotRoot, policyVersion, policyLabel, roleLabel, identityLabel, actorLabel, projectLabel, providerLabel, consumerKindLabel, workloadLabel, fingerprintLabel, tokenHashLabel, runtimeFingerprintLabel, ] = inputArgs; const immutableId = /^[a-f0-9]{64}$/; const digest = /^[a-f0-9]{64}$/; const imageId = /^sha256:[a-f0-9]{64}$/; const providers = new Set([ 'OPENCLAW', 'CLAUDE_CODE', 'CODEX', 'AGENT_ZERO', 'GEMINI', 'OLLAMA', 'GROK_BUILD', 'PORTAL_WORKLOAD', ]); const workloadKinds = new Set([ 'PORTAL_GIT', 'PORTAL_LIFECYCLE', 'PORTAL_APP', ]); const fail = () => { throw new Error('invalid Project egress topology'); }; const object = (value) => { if (!value || typeof value !== 'object' || Array.isArray(value)) fail(); return value; }; const exactBoolean = (value) => { if (typeof value !== 'boolean') fail(); return value; }; const opaque = (value) => { if (typeof value !== 'string' || Buffer.byteLength(value, 'utf8') < 1 || Buffer.byteLength(value, 'utf8') > 512 || /[\u0000-\u001f\u007f]/.test(value)) fail(); return value; }; const sha256 = (value) => crypto.createHash('sha256').update(value).digest('hex'); const jsonStringHash = (value) => sha256(JSON.stringify(value)); const readJsonLines = (file, count) => { const raw = fs.readFileSync(file, 'utf8'); const lines = raw.endsWith('\n') ? raw.slice(0, -1).split('\n') : raw.split('\n'); if (lines.length !== count) fail(); return lines.map((line) => JSON.parse(line)); }; const readList = (file) => { const raw = fs.readFileSync(file, 'utf8').trim(); if (!raw) return []; return raw.split('\n').map((line) => { const fields = line.split(/\s+/); if (fields.length !== 2 || !immutableId.test(fields[0]) || !fields[1]) fail(); return { id: fields[0], name: fields[1] }; }); }; const sameKeys = (value, expected) => { const actual = Object.keys(value).sort(); const wanted = [...expected].sort(); return JSON.stringify(actual) === JSON.stringify(wanted); }; const canonical = (value) => { if (Array.isArray(value)) return value.map(canonical); if (value && typeof value === 'object') { return Object.fromEntries( Object.keys(value).sort().map((key) => [key, canonical(value[key])]), ); } return value; }; const containerList = readList(path.join(snapshotRoot, 'container-list')); const networkList = readList(path.join(snapshotRoot, 'network-list')); if (new Set(containerList.map(({ id }) => id)).size !== containerList.length || new Set(networkList.map(({ id }) => id)).size !== networkList.length) fail(); const containers = new Map(); for (const listed of containerList) { const [ id, inspectedName, image, configuredImage, labelsInput, networkMode, networksInput, running, ] = readJsonLines(path.join(snapshotRoot, 'containers', listed.id), 8); const name = String(inspectedName || '').replace(/^\//, ''); if (id !== listed.id || name !== listed.name || typeof networkMode !== 'string') fail(); containers.set(id, { id, name, image, configuredImage, labels: object(labelsInput), networkMode, networks: object(networksInput), running: exactBoolean(running), }); } const networks = new Map(); for (const listed of networkList) { if (!/^p4e-(?:in|out)-[a-f0-9]{20}$/.test(listed.name)) continue; const [ id, name, driver, internal, attachable, ingress, enableIPv6, labelsInput, optionsInput, ipamInput, membersInput, ] = readJsonLines(path.join(snapshotRoot, 'networks', listed.id), 11); if (id !== listed.id || name !== listed.name || driver !== 'bridge' || attachable !== false || ingress !== false || typeof enableIPv6 !== 'boolean') fail(); const options = optionsInput === null ? null : object(optionsInput); const ipam = object(ipamInput); if (typeof ipam.Driver !== 'string' || (ipam.Options !== null && ipam.Options !== undefined && (typeof ipam.Options !== 'object' || Array.isArray(ipam.Options))) || !Array.isArray(ipam.Config) || ipam.Config.length < 1) fail(); const ipamConfig = ipam.Config.find( (entry) => entry && typeof entry === 'object' && !Array.isArray(entry) && String(entry.Subnet || '').includes('.'), ); if (!ipamConfig) fail(); const subnetMatch = String(ipamConfig.Subnet || '').match( /^((?:\d{1,3}\.){3}\d{1,3})\/(\d{1,2})$/, ); const gatewayValue = String(ipamConfig.Gateway || ''); const baseValue = gatewayValue || (subnetMatch ? subnetMatch[1] : ''); const gatewayMatch = gatewayValue ? gatewayValue.match(/^((?:\d{1,3}\.){3}\d{1,3})$/) : null; if (!subnetMatch || Number.parseInt(subnetMatch[2], 10) < 1 || Number.parseInt(subnetMatch[2], 10) > 32) fail(); const octets = (value) => value.split('.').map((part) => Number.parseInt(part, 10)); const subnetOctets = octets(subnetMatch[1]); const baseOctets = octets(baseValue); if ((gatewayValue && !gatewayMatch) || [...subnetOctets, ...baseOctets].some( (part) => !Number.isInteger(part) || part < 0 || part > 255, ) || baseOctets.length !== 4 || baseOctets[3] + (gatewayValue ? 1 : 2) > 255) fail(); const staticProxyOctets = [...baseOctets]; staticProxyOctets[3] += gatewayValue ? 1 : 2; networks.set(id, { id, name, driver, internal: exactBoolean(internal), attachable, ingress, enableIPv6, labels: object(labelsInput), options, ipam, subnet: subnetMatch[0], staticProxyIpv4: staticProxyOctets.join('.'), members: membersInput === null ? {} : object(membersInput), }); } const identityFromLabels = (labels, expectedRole) => { if (labels[policyLabel] !== policyVersion || labels[roleLabel] !== expectedRole) fail(); const actorId = opaque(labels[actorLabel]); const projectId = opaque(labels[projectLabel]); const provider = opaque(labels[providerLabel]); if (!providers.has(provider)) fail(); const identity = { actorId, projectId, provider }; if (provider === 'PORTAL_WORKLOAD') { const consumerKind = opaque(labels[consumerKindLabel]); const workloadId = opaque(labels[workloadLabel]); if (!workloadKinds.has(consumerKind)) fail(); identity.consumerKind = consumerKind; identity.workloadId = workloadId; } else if (labels[consumerKindLabel] !== undefined || labels[workloadLabel] !== undefined) fail(); const identityFingerprint = jsonStringHash(identity); if (labels[identityLabel] !== identityFingerprint || !digest.test(String(labels[fingerprintLabel] || ''))) fail(); return { ...identity, identityFingerprint, policyFingerprint: labels[fingerprintLabel], }; }; const planes = new Map(); const addPlaneResource = (identity, key, resource) => { let plane = planes.get(identity.identityFingerprint); if (!plane) { plane = { identity, proxy: null, internal: null, publicNetwork: null }; planes.set(identity.identityFingerprint, plane); } else if (JSON.stringify(plane.identity) !== JSON.stringify(identity)) fail(); if (plane[key]) fail(); plane[key] = resource; }; for (const container of containers.values()) { if (!/^p4e-proxy-[a-f0-9]{20}$/.test(container.name)) continue; const identity = identityFromLabels(container.labels, 'proxy'); const expectedName = `p4e-proxy-${identity.identityFingerprint.slice(0, 20)}`; if (container.name !== expectedName || !imageId.test(String(container.image || '')) || container.configuredImage !== container.image || !digest.test(String(container.labels[tokenHashLabel] || ''))) fail(); addPlaneResource(identity, 'proxy', container); } for (const network of networks.values()) { const role = network.name.startsWith('p4e-in-') ? 'internal' : 'proxy-public'; const identity = identityFromLabels(network.labels, role); const expectedName = `p4e-${role === 'internal' ? 'in' : 'out'}-${identity.identityFingerprint.slice(0, 20)}`; if (network.name !== expectedName || network.internal !== (role === 'internal')) fail(); addPlaneResource( identity, role === 'internal' ? 'internal' : 'publicNetwork', network, ); } const runtimeIdentityMatches = (runtime, plane) => { const labels = runtime.labels; const fingerprint = String(labels[runtimeFingerprintLabel] || ''); if (!digest.test(fingerprint)) return false; const actor = plane.identity.actorId; const project = plane.identity.projectId; switch (plane.identity.provider) { case 'OPENCLAW': return runtime.name.startsWith('p4oc-') && labels['openclaw.sandbox'] === '1' && labels['com.bridgesllm.openclaw-project.actor'] === sha256(actor) && labels['com.bridgesllm.openclaw-project.identity'] === sha256(project); case 'CODEX': return runtime.name === `p4cx-${fingerprint.slice(0, 24)}` && ['portal-project-sandbox-v3', 'portal-project-sandbox-v2'].includes( labels['com.bridgesllm.codex-project.policy'], ) && labels['com.bridgesllm.codex-project.actor'] === jsonStringHash(actor) && labels['com.bridgesllm.codex-project.project'] === jsonStringHash(project) && labels['com.bridgesllm.codex-project.egress'] === plane.identity.policyFingerprint; case 'CLAUDE_CODE': case 'GEMINI': { const prefix = plane.identity.provider === 'CLAUDE_CODE' ? 'p4cc' : 'p4ag'; return runtime.name === `${prefix}-${fingerprint.slice(0, 24)}` && labels['com.bridgesllm.native-cli-project.policy'] === (plane.identity.provider === 'CLAUDE_CODE' ? 'portal-claude-code-project-sandbox-v1' : 'portal-antigravity-project-sandbox-v1') && labels['com.bridgesllm.native-cli-project.provider'] === plane.identity.provider && labels['com.bridgesllm.native-cli-project.actor'] === jsonStringHash(actor) && labels['com.bridgesllm.native-cli-project.project'] === jsonStringHash(project) && labels['com.bridgesllm.native-cli-project.egress'] === plane.identity.policyFingerprint; } case 'AGENT_ZERO': { const key = String(labels['io.bridgesllm.project-key'] || ''); return digest.test(key) && runtime.name === `bridgesllm-a0p-${key.slice(0, 24)}` && labels['io.bridgesllm.managed'] === 'agent-zero-project' && labels['io.bridgesllm.runtime'] === 'agent-zero-project-sandbox-v4' && labels['io.bridgesllm.policy'] === 'agent-zero-project-isolation-v4' && labels['io.bridgesllm.project-id'] === project && labels['io.bridgesllm.actor-id'] === actor; } case 'PORTAL_WORKLOAD': { const kind = plane.identity.consumerKind; const workload = plane.identity.workloadId; const prefix = kind === 'PORTAL_GIT' ? 'bridgesllm-project-git' : kind === 'PORTAL_APP' ? 'bridgesllm-project-app' : 'bridgesllm-project-job'; const discriminator = kind === 'PORTAL_GIT' ? `${actor}\0${project}\0${workload}` : `${actor}\0${project}\0${kind === 'PORTAL_APP' ? 'app' : 'job'}\0${workload}`; return runtime.name === `${prefix}-${sha256(discriminator).slice(0, 20)}` && labels['com.bridgesllm.project-workload.policy'] === 'portal-project-workload-v1' && labels['com.bridgesllm.project-workload.actor-id'] === actor && labels['com.bridgesllm.project-workload.project-id'] === project && labels['com.bridgesllm.project-workload.kind'] === kind && labels['com.bridgesllm.project-workload.workload-id'] === workload; } // Ollama's Project tool container is intentionally --network none, and // GROK_BUILD has no current managed Project container runtime. Either // provider may own a proxy plane, but neither may contribute an internal // network member. case 'OLLAMA': case 'GROK_BUILD': return false; default: return false; } }; const topologyRecords = []; for (const plane of planes.values()) { const { identity, proxy, internal, publicNetwork } = plane; if (!proxy) { if (!internal || !publicNetwork || identity.policyFingerprint !== internal.labels[fingerprintLabel] || identity.policyFingerprint !== publicNetwork.labels[fingerprintLabel]) fail(); const referenced = (network) => [...containers.values()].some((container) => ( Object.prototype.hasOwnProperty.call(container.networks, network.name) || [network.id, network.name].includes(container.networkMode.toLowerCase()) )); if (Object.keys(internal.members).length !== 0 || Object.keys(publicNetwork.members).length !== 0 || referenced(internal) || referenced(publicNetwork)) fail(); const orphanTopology = canonical({ identity, internal, publicNetwork }); topologyRecords.push( `orphan-network|${internal.id}|${internal.name}|internal`, `orphan-network|${publicNetwork.id}|${publicNetwork.name}|proxy-public`, `orphan-plane|${identity.identityFingerprint}|${sha256(JSON.stringify(orphanTopology))}|`, ); continue; } if (!internal || !publicNetwork || identity.policyFingerprint !== internal.labels[fingerprintLabel] || identity.policyFingerprint !== publicNetwork.labels[fingerprintLabel]) fail(); const expectedNetworkNames = [internal.name, publicNetwork.name].sort(); if (!sameKeys(proxy.networks, expectedNetworkNames) || ![publicNetwork.id, publicNetwork.name].includes( proxy.networkMode.toLowerCase(), )) fail(); const internalAttachment = object(proxy.networks[internal.name]); const publicAttachment = object(proxy.networks[publicNetwork.name]); const reportedInternalId = String(internalAttachment.NetworkID || '').toLowerCase(); const reportedPublicId = String(publicAttachment.NetworkID || '').toLowerCase(); const stoppedUnmaterialized = proxy.running === false && proxy.networkMode.toLowerCase() === publicNetwork.id; if ((reportedInternalId !== internal.id && !(stoppedUnmaterialized && reportedInternalId === '')) || (reportedPublicId !== publicNetwork.id && !(stoppedUnmaterialized && reportedPublicId === '')) || !Array.isArray(internalAttachment.Aliases) || !internalAttachment.Aliases.includes('portal-project-egress')) fail(); const staticPublicIpv4 = String(publicAttachment.IPAMConfig?.IPv4Address || ''); if (staticPublicIpv4 !== publicNetwork.staticProxyIpv4) fail(); const reversePublic = []; const reverseInternal = []; for (const container of containers.values()) { if (Object.prototype.hasOwnProperty.call(container.networks, publicNetwork.name)) { reversePublic.push(container); } if (Object.prototype.hasOwnProperty.call(container.networks, internal.name)) { reverseInternal.push(container); } } if (reversePublic.length !== 1 || reversePublic[0].id !== proxy.id) fail(); const runtimeCandidates = reverseInternal.filter(({ id }) => id !== proxy.id); if (!reverseInternal.some(({ id }) => id === proxy.id) || runtimeCandidates.length > 1) fail(); const runtime = runtimeCandidates[0] || null; if (runtime) { if (!runtimeIdentityMatches(runtime, plane) || !sameKeys(runtime.networks, [internal.name]) || runtime.networkMode === 'host') fail(); const attachmentId = String(runtime.networks[internal.name]?.NetworkID || '').toLowerCase(); if (attachmentId !== internal.id && !(attachmentId === '' && runtime.running === false && ['none', internal.id].includes(runtime.networkMode.toLowerCase()))) fail(); } const attestMembership = (network, expectedRunning) => { const entries = Object.entries(network.members); if (entries.some(([id, member]) => !immutableId.test(id) || !containers.has(id) || containers.get(id).name !== member?.Name)) fail(); const actual = entries.map(([id]) => id).sort(); const expected = expectedRunning.map(({ id }) => id).sort(); if (JSON.stringify(actual) !== JSON.stringify(expected)) fail(); }; attestMembership( publicNetwork, proxy.running ? [proxy] : [], ); attestMembership( internal, [proxy, runtime].filter((candidate) => candidate?.running), ); // A proxy that never started (or exited before Docker materialized its // endpoints) can leave the exact managed container and its empty network // pair behind. Treat only that fully detached, stopped shape as reclaimable // debris. A running proxy, any runtime peer, or any network-side member // remains an authoritative topology failure. const exitedDetachedProxy = proxy.running === false && runtime === null && Object.keys(internal.members).length === 0 && Object.keys(publicNetwork.members).length === 0; const exitedDetachedProxyDebris = exitedDetachedProxy && reportedInternalId === '' && reportedPublicId === '' && String(internalAttachment.EndpointID || '') === '' && String(publicAttachment.EndpointID || '') === '' && String(internalAttachment.IPAddress || '') === '' && String(publicAttachment.IPAddress || '') === '' && String(internalAttachment.GlobalIPv6Address || '') === '' && String(publicAttachment.GlobalIPv6Address || '') === '' && String(internalAttachment.IPAMConfig?.IPv6Address || '') === '' && String(publicAttachment.IPAMConfig?.IPv6Address || '') === ''; if (exitedDetachedProxyDebris) { const orphanTopology = canonical({ identity, proxy: { id: proxy.id, name: proxy.name, image: proxy.image, configuredImage: proxy.configuredImage, labels: proxy.labels, networkMode: proxy.networkMode, networks: proxy.networks, running: proxy.running, }, internal, publicNetwork, runtime: null, }); topologyRecords.push( `orphan-container|${proxy.id}|/${proxy.name}|${proxy.image}`, `orphan-network|${internal.id}|${internal.name}|internal`, `orphan-network|${publicNetwork.id}|${publicNetwork.name}|proxy-public`, `orphan-plane|${identity.identityFingerprint}|${sha256(JSON.stringify(orphanTopology))}|`, ); continue; } if (exitedDetachedProxy) fail(); const topology = canonical({ identity, proxy: { id: proxy.id, name: proxy.name, image: proxy.image, configuredImage: proxy.configuredImage, labels: proxy.labels, networkMode: proxy.networkMode, networks: proxy.networks, running: proxy.running, }, internal, publicNetwork, runtime: runtime ? { id: runtime.id, name: runtime.name, labels: runtime.labels, networkMode: runtime.networkMode, networks: runtime.networks, running: runtime.running, } : null, }); topologyRecords.push( `container|${proxy.id}|/${proxy.name}|${proxy.image}`, `network|${internal.id}|${internal.name}|internal`, `network|${publicNetwork.id}|${publicNetwork.name}|proxy-public`, `topology|${identity.identityFingerprint}|${sha256(JSON.stringify(topology))}`, ); } if (planes.size === 0) fail(); process.stdout.write(`${topologyRecords.sort().join('\n')}\n`); } catch { const resources = []; const containerIds = []; const networkIds = []; try { for (const [kind, file, namePattern] of [ ['proxy', 'container-list', /^p4e-proxy-[a-f0-9]{20}$/], ['network', 'network-list', /^p4e-(?:in|out)-[a-f0-9]{20}$/], ]) { const raw = fs.readFileSync(path.join(diagnosticSnapshotRoot, file), 'utf8').trim(); if (!raw) continue; for (const line of raw.split('\n')) { const fields = line.split(/\s+/); if (fields.length === 2 && /^[a-f0-9]{64}$/.test(fields[0]) && namePattern.test(fields[1])) { resources.push(`${kind} ${fields[1]} (${fields[0]})`); (kind === 'proxy' ? containerIds : networkIds).push(fields[0]); } } } } catch { // Keep the primary validation failure authoritative when diagnostics race. } console.error( `Managed Project egress inventory validation failed for ${resources.join(', ') || 'unresolved reserved resources'}. ` + `Remediation: ${containerIds.length ? `docker container inspect ${containerIds.join(' ')}; ` : ''}` + `${networkIds.length ? `docker network inspect ${networkIds.join(' ')}; ` : ''}` + 'stop attached Project runtimes, then retry the Portal update. Remove only a fully empty parentless network pair.', ); process.exit(3); } NODE ) discover_unique_managed_project_egress_proxy_image_id() { # Two complete immutable snapshots make image selection a CAS boundary. # Any create/remove/relabel between them is a race, not a generation the # updater may guess through. local first_inventory="" second_inventory="" barrier_inventory="" removal_inventory="" local first_status=0 second_status=0 barrier_status=0 removal_status=0 local row kind resource_id name detail selected_image_id="" index local cleanup_attempted=false resource_summary="" separator="" local -a orphan_container_ids=() local -a orphan_container_names=() local -a orphan_network_ids=() local -a orphan_network_names=() while true; do first_inventory="" second_inventory="" selected_image_id="" orphan_container_ids=() orphan_container_names=() orphan_network_ids=() orphan_network_names=() if first_inventory="$(capture_managed_project_egress_inventory)"; then first_status=0 else first_status=$? fi if second_inventory="$(capture_managed_project_egress_inventory)"; then second_status=0 else second_status=$? fi if [[ "${first_status}" != "${second_status}" \ || "${first_inventory}" != "${second_inventory}" ]]; then return 5 fi case "${first_status}" in 0) ;; 1|2|3) return "${first_status}" ;; *) return 5 ;; esac while IFS='|' read -r kind resource_id name detail; do case "${kind}" in container) [[ "${resource_id}" =~ ^[a-f0-9]{64}$ \ && "${name}" =~ ^/p4e-proxy-[a-f0-9]{20}$ ]] \ || return 3 valid_docker_image_id "${detail}" || return 3 if [[ -n "${selected_image_id}" \ && "${selected_image_id}" != "${detail}" ]]; then return 4 fi selected_image_id="${detail}" ;; network) [[ "${resource_id}" =~ ^[a-f0-9]{64}$ \ && "${name}" =~ ^p4e-(in|out)-[a-f0-9]{20}$ \ && ( "${detail}" == "internal" || "${detail}" == "proxy-public" ) ]] \ || return 3 ;; topology|orphan-plane) [[ "${resource_id}" =~ ^[a-f0-9]{64}$ \ && "${name}" =~ ^[a-f0-9]{64}$ \ && -z "${detail}" ]] \ || return 3 ;; orphan-network) [[ "${resource_id}" =~ ^[a-f0-9]{64}$ \ && "${name}" =~ ^p4e-(in|out)-[a-f0-9]{20}$ \ && ( "${detail}" == "internal" || "${detail}" == "proxy-public" ) ]] \ || return 3 orphan_network_ids+=("${resource_id}") orphan_network_names+=("${name}") ;; orphan-container) [[ "${resource_id}" =~ ^[a-f0-9]{64}$ \ && "${name}" =~ ^/p4e-proxy-[a-f0-9]{20}$ ]] \ || return 3 valid_docker_image_id "${detail}" || return 3 orphan_container_ids+=("${resource_id}") orphan_container_names+=("${name#/}") ;; *) return 3 ;; esac done <<<"${first_inventory}" if ((${#orphan_network_ids[@]} > 0)); then if ${cleanup_attempted} || ((${#orphan_network_ids[@]} % 2 != 0)); then return 5 fi barrier_inventory="" if barrier_inventory="$(capture_managed_project_egress_inventory)"; then barrier_status=0 else barrier_status=$? fi if [[ "${barrier_status}" != "0" \ || "${barrier_inventory}" != "${first_inventory}" ]]; then return 5 fi # One final complete discovery immediately before the first exact-ID # removal closes the gap between the cleanup barrier and mutation. A # stopped proxy that is reattached or otherwise changes remains intact. removal_inventory="" if removal_inventory="$(capture_managed_project_egress_inventory)"; then removal_status=0 else removal_status=$? fi if [[ "${removal_status}" != "0" \ || "${removal_inventory}" != "${first_inventory}" ]]; then return 5 fi resource_summary="" separator="" for ((index = 0; index < ${#orphan_container_ids[@]}; index++)); do resource_summary+="${separator}${orphan_container_names[index]} (${orphan_container_ids[index]})" separator=", " done for ((index = 0; index < ${#orphan_network_ids[@]}; index++)); do resource_summary+="${separator}${orphan_network_names[index]} (${orphan_network_ids[index]})" separator=", " done printf '%s\n' \ "Reclaiming exact stopped, detached Project egress debris: ${resource_summary}." >&2 for ((index = 0; index < ${#orphan_container_ids[@]}; index++)); do if ! docker container rm "${orphan_container_ids[index]}" \ >> "${LOG_FILE}" 2>&1; then printf '%s\n' \ "Could not reclaim Project egress debris ${resource_summary}. Remediation: docker container inspect ${orphan_container_ids[*]}; docker network inspect ${orphan_network_ids[*]}; verify the proxy is stopped and every network endpoint is gone, then retry the Portal update." >&2 return 3 fi if docker container inspect "${orphan_container_ids[index]}" \ >/dev/null 2>&1; then printf '%s\n' \ "Project egress proxy ${orphan_container_names[index]} (${orphan_container_ids[index]}) remained after exact removal. Remediation: docker container inspect ${orphan_container_ids[index]}; retry only after the proxy is stopped and detached." >&2 return 3 fi done for ((index = 0; index < ${#orphan_network_ids[@]}; index++)); do if ! docker network rm "${orphan_network_ids[index]}" \ >> "${LOG_FILE}" 2>&1; then printf '%s\n' \ "Could not reclaim Project egress debris ${resource_summary}. Remediation: docker network inspect ${orphan_network_ids[*]}; stop attached Project runtimes, then retry the Portal update. Do not remove a network that still has endpoints." >&2 return 3 fi if docker network inspect "${orphan_network_ids[index]}" \ >/dev/null 2>&1; then printf '%s\n' \ "Project egress network ${orphan_network_names[index]} (${orphan_network_ids[index]}) remained after exact removal. Remediation: docker network inspect ${orphan_network_ids[index]}; retry the Portal update after all endpoints are gone." >&2 return 3 fi done cleanup_attempted=true continue fi valid_docker_image_id "${selected_image_id}" || return 3 printf '%s\n' "${selected_image_id}" return 0 done } install_project_runtime_confinement_file() { local source_file="$1" local target_file="$2" local expected_sha256="$3" local install_mode="${4:-replace}" local target_dir temporary [[ "${install_mode}" == "replace" || "${install_mode}" == "additive" ]] \ || fail "Project runtime confinement install mode is invalid." [[ -f "${source_file}" && ! -L "${source_file}" ]] \ || fail "Project runtime confinement source ${source_file} is missing or unsafe." [[ "$(stat -c '%u:%g' "${source_file}")" == "0:0" ]] \ || fail "Project runtime confinement source ${source_file} is not root-owned." (( (8#$(stat -c '%a' "${source_file}") & 0022) == 0 )) \ || fail "Project runtime confinement source ${source_file} is writable by an unsafe principal." [[ "$(sha256sum "${source_file}" | awk '{print $1}')" == "${expected_sha256}" ]] \ || fail "Project runtime confinement source ${source_file} failed its release digest." target_dir="$(dirname "${target_file}")" if [[ ! -e "${target_dir}" ]]; then install -d -o root -g root -m 0755 "${target_dir}" \ || fail "Could not create the Project runtime confinement directory ${target_dir}." fi [[ -d "${target_dir}" && ! -L "${target_dir}" \ && "$(stat -c '%u:%g' "${target_dir}")" == "0:0" \ && $((8#$(stat -c '%a' "${target_dir}") & 0022)) -eq 0 ]] \ || fail "Project runtime confinement directory ${target_dir} has unsafe metadata." if [[ "${install_mode}" == "additive" \ && ( -e "${target_file}" || -L "${target_file}" ) ]]; then [[ -f "${target_file}" && ! -L "${target_file}" \ && "$(stat -c '%u:%g:%a:%h' "${target_file}")" == "0:0:644:1" \ && "$(sha256sum "${target_file}" | awk '{print $1}')" == "${expected_sha256}" ]] \ || fail "An update cannot replace confinement policy ${target_file} before cutover. Publish changed policy under a new versioned path and profile name." return 0 fi temporary="$(mktemp "${target_dir}/.bridgesllm-project-runtime.XXXXXX")" install -o root -g root -m 0644 "${source_file}" "${temporary}" || { rm -f -- "${temporary}" fail "Could not stage Project runtime confinement profile ${target_file}." } [[ "$(sha256sum "${temporary}" | awk '{print $1}')" == "${expected_sha256}" ]] || { rm -f -- "${temporary}" fail "Staged Project runtime confinement profile ${target_file} changed unexpectedly." } mv -f -- "${temporary}" "${target_file}" \ || fail "Could not atomically install Project runtime confinement profile ${target_file}." chown root:root "${target_file}" chmod 0644 "${target_file}" [[ -f "${target_file}" && ! -L "${target_file}" \ && "$(stat -c '%u:%g:%a:%h' "${target_file}")" == "0:0:644:1" \ && "$(sha256sum "${target_file}" | awk '{print $1}')" == "${expected_sha256}" ]] \ || fail "Installed Project runtime confinement profile ${target_file} failed metadata or digest attestation." } project_runtime_confinement_docker_args() { local -n output_args="$1" local policy="${PROJECT_RUNTIME_CONFINEMENT_POLICY:-${PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY}}" output_args=( --security-opt no-new-privileges --security-opt "seccomp=${PROJECT_RUNTIME_SECCOMP_PROFILE_PATH}" ) case "${policy}" in "${PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY}") output_args+=(--security-opt "apparmor=${PROJECT_RUNTIME_APPARMOR_PROFILE_NAME}") ;; "${PROJECT_RUNTIME_SECCOMP_ONLY_POLICY}") ;; *) fail "Project runtime confinement policy is invalid." ;; esac } codex_project_runtime_confinement_docker_args() { local -n output_args="$1" ${CODEX_PROJECT_RUNTIME_CONFINEMENT_AVAILABLE} || return 1 output_args=( --security-opt no-new-privileges --security-opt "seccomp=${CODEX_PROJECT_RUNTIME_SECCOMP_PROFILE_PATH}" ) case "${PROJECT_RUNTIME_CONFINEMENT_POLICY}" in "${PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY}") output_args+=(--security-opt "apparmor=${CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_NAME}") ;; "${PROJECT_RUNTIME_SECCOMP_ONLY_POLICY}") ;; *) return 1 ;; esac } # One provisioning boundary for every Project runtime dependency. With # --skip-project-runtimes it persists the explicit disabled policy (which the # backend fails closed on with a truthful reason) and skips confinement and # image provisioning entirely; the rest of the install proceeds. provision_project_runtimes() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" if ${SKIP_PROJECT_RUNTIMES}; then CODEX_PROJECT_RUNTIME_CONFINEMENT_AVAILABLE=false PROJECT_RUNTIME_CONFINEMENT_POLICY="${PROJECT_RUNTIME_DISABLED_POLICY}" set_env_value_atomic \ "${env_file}" \ "PROJECT_RUNTIME_CONFINEMENT_POLICY" \ "${PROJECT_RUNTIME_DISABLED_POLICY}" \ || fail "Could not persist the disabled Project runtime policy." [[ "$(read_env_value "${env_file}" PROJECT_RUNTIME_CONFINEMENT_POLICY 2>/dev/null || true)" \ == "${PROJECT_RUNTIME_DISABLED_POLICY}" ]] \ || fail "Could not verify the disabled Project runtime policy." warn "Project runtimes are disabled (--skip-project-runtimes): Project Chat providers will be unavailable on this host until the installer runs again without the flag on a supported host." return 0 fi ensure_project_runtime_confinement_profiles "${env_file}" ensure_project_egress_proxy_image "${env_file}" ensure_codex_project_sandbox_image "${env_file}" ensure_claude_code_project_sandbox_image "${env_file}" ensure_antigravity_project_sandbox_image "${env_file}" ensure_ollama_project_sandbox_image "${env_file}" ensure_agent_zero_project_sandbox_image "${env_file}" ensure_openclaw_sandbox_image "${env_file}" ensure_portal_project_runtime_image \ "${PORTAL_PROJECT_RUNTIME_IMAGE_TAG}" "${env_file}" prune_project_runtime_build_debris } prune_project_runtime_build_debris() { # The sandbox image builds above leave tens of GB behind per convergence # cycle (observed: 18.2 GB of build cache plus 14 GB of dangling layers a # few hours after a clean box, and 17 GB still held as "in-use" BuildKit # cache after a default prune). Rebuilds only happen inside updates at pin # bumps, so standing cache buys minutes there at tens of GB of permanent # cost: prune it all. Tagged images, containers, and volumes are never # touched; an untagged digest-pulled base may be reclaimed and simply # re-pulls on the next convergence. A prune failure must never fail an # install. spin "Reclaiming Docker build cache from sandbox image builds" \ "docker builder prune --all --force" \ || warn "Docker build cache prune did not complete; disk reclaim will retry on the next update." spin "Removing dangling Docker image layers" \ "docker image prune --force" \ || warn "Dangling image prune did not complete; disk reclaim will retry on the next update." } ensure_project_runtime_confinement_profiles() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local source_root="${2:-${PORTAL_DIR}}" local install_mode="${3:-replace}" local docker_security_options apparmor_enabled=false docker_has_apparmor=false local shared_seccomp_source="${source_root}/installer/bridgesllm-project-runtime-v1.seccomp.json" local codex_seccomp_source="${source_root}/installer/bridgesllm-codex-project-runtime-v1.seccomp.json" local shared_apparmor_source="${source_root}/installer/bridgesllm-project-runtime-v1.apparmor" local codex_apparmor_source="${source_root}/installer/bridgesllm-codex-project-runtime-v1.apparmor" CODEX_PROJECT_RUNTIME_CONFINEMENT_AVAILABLE=false [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for Project runtime confinement." command -v docker >/dev/null 2>&1 \ || fail "Docker is required for Project runtime confinement attestation." install_project_runtime_confinement_file \ "${shared_seccomp_source}" \ "${PROJECT_RUNTIME_SECCOMP_PROFILE_PATH}" \ "${PROJECT_RUNTIME_SECCOMP_PROFILE_SHA256}" "${install_mode}" install_project_runtime_confinement_file \ "${codex_seccomp_source}" \ "${CODEX_PROJECT_RUNTIME_SECCOMP_PROFILE_PATH}" \ "${CODEX_PROJECT_RUNTIME_SECCOMP_PROFILE_SHA256}" "${install_mode}" python3 -m json.tool "${PROJECT_RUNTIME_SECCOMP_PROFILE_PATH}" >/dev/null \ || fail "Installed Project runtime seccomp profile is not valid JSON." python3 -m json.tool "${CODEX_PROJECT_RUNTIME_SECCOMP_PROFILE_PATH}" >/dev/null \ || fail "Installed Codex Project runtime seccomp profile is not valid JSON." install_project_runtime_confinement_file \ "${shared_apparmor_source}" \ "${PROJECT_RUNTIME_APPARMOR_PROFILE_PATH}" \ "${PROJECT_RUNTIME_APPARMOR_PROFILE_SHA256}" "${install_mode}" install_project_runtime_confinement_file \ "${codex_apparmor_source}" \ "${CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_PATH}" \ "${CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_SHA256}" "${install_mode}" docker_security_options="$(docker info --format '{{json .SecurityOptions}}' 2>/dev/null || true)" [[ "${docker_security_options}" == *'"name=seccomp'* ]] \ || fail "Docker does not report seccomp support; Project runtimes remain unavailable." [[ "$(cat /sys/module/apparmor/parameters/enabled 2>/dev/null || true)" == "Y" ]] \ && apparmor_enabled=true [[ "${docker_security_options}" == *'"name=apparmor"'* ]] \ && docker_has_apparmor=true if ${apparmor_enabled} && ${docker_has_apparmor}; then command -v apparmor_parser >/dev/null 2>&1 \ || fail "AppArmor is supported but apparmor_parser is unavailable; Project runtimes remain unavailable." apparmor_parser -r "${PROJECT_RUNTIME_APPARMOR_PROFILE_PATH}" >> "${LOG_FILE}" 2>&1 \ || fail "Project runtime AppArmor profile could not be loaded in enforce mode." grep -Fx "${PROJECT_RUNTIME_APPARMOR_PROFILE_NAME} (enforce)" \ /sys/kernel/security/apparmor/profiles >/dev/null 2>&1 \ || fail "Project runtime AppArmor profile is not loaded in enforce mode." if apparmor_parser -Q -T "${CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_PATH}" \ >> "${LOG_FILE}" 2>&1 \ && apparmor_parser -r "${CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_PATH}" \ >> "${LOG_FILE}" 2>&1 \ && grep -Fx "${CODEX_PROJECT_RUNTIME_APPARMOR_PROFILE_NAME} (enforce)" \ /sys/kernel/security/apparmor/profiles >/dev/null 2>&1; then CODEX_PROJECT_RUNTIME_CONFINEMENT_AVAILABLE=true else warn "This AppArmor parser cannot enforce the Codex Project user-namespace profile; Codex Project is unavailable, while other confined providers remain enabled." fi PROJECT_RUNTIME_CONFINEMENT_POLICY="${PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY}" elif ! ${apparmor_enabled} && ! ${docker_has_apparmor}; then PROJECT_RUNTIME_CONFINEMENT_POLICY="${PROJECT_RUNTIME_SECCOMP_ONLY_POLICY}" CODEX_PROJECT_RUNTIME_CONFINEMENT_AVAILABLE=true warn "AppArmor is genuinely unsupported on this Docker host; Project runtimes use the explicit seccomp-only policy." else fail "AppArmor kernel support (enabled: ${apparmor_enabled}) and Docker AppArmor support (available: ${docker_has_apparmor}) disagree on this host, so confined Project runtimes cannot be attested. This is common on LXC/OpenVZ/nested-container hosts where the kernel exposes AppArmor but the container runtime cannot hand it to Docker. Either install on a host/VM where Docker reports AppArmor support, or re-run the installer with --skip-project-runtimes to install the Portal with Project Chat disabled." fi set_env_value_atomic \ "${env_file}" \ "PROJECT_RUNTIME_CONFINEMENT_POLICY" \ "${PROJECT_RUNTIME_CONFINEMENT_POLICY}" \ || fail "Could not persist the Project runtime confinement policy." [[ "$(read_env_value "${env_file}" PROJECT_RUNTIME_CONFINEMENT_POLICY 2>/dev/null || true)" \ == "${PROJECT_RUNTIME_CONFINEMENT_POLICY}" ]] \ || fail "Project runtime confinement policy did not persist exactly." ok "Project runtime AppArmor/seccomp confinement (${PROJECT_RUNTIME_CONFINEMENT_POLICY})" } record_installer_image_id() { local env_file="$1" local key="$2" local image_id="$3" valid_docker_image_id "${image_id}" || return 1 case "${key}" in OPENCLAW_PROJECT_SANDBOX_IMAGE_ID) OPENCLAW_PROJECT_SANDBOX_IMAGE_ID="${image_id}" ;; CODEX_PROJECT_SANDBOX_IMAGE_ID) CODEX_PROJECT_SANDBOX_IMAGE_ID="${image_id}" ;; CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID) CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID="${image_id}" ;; ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID) ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID="${image_id}" ;; OLLAMA_PROJECT_SANDBOX_IMAGE_ID) OLLAMA_PROJECT_SANDBOX_IMAGE_ID="${image_id}" ;; AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID) AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID="${image_id}" ;; PROJECT_EGRESS_PROXY_IMAGE_ID) PROJECT_EGRESS_PROXY_IMAGE_ID="${image_id}" ;; *) return 1 ;; esac set_env_value_atomic "${env_file}" "${key}" "${image_id}" || return 1 [[ "$(read_env_value "${env_file}" "${key}" 2>/dev/null || true)" == "${image_id}" ]] } clear_installer_image_id() { local env_file="$1" local key="$2" case "${key}" in OPENCLAW_PROJECT_SANDBOX_IMAGE_ID) OPENCLAW_PROJECT_SANDBOX_IMAGE_ID="" ;; CODEX_PROJECT_SANDBOX_IMAGE_ID) CODEX_PROJECT_SANDBOX_IMAGE_ID="" ;; CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID) CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID="" ;; ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID) ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID="" ;; OLLAMA_PROJECT_SANDBOX_IMAGE_ID) OLLAMA_PROJECT_SANDBOX_IMAGE_ID="" ;; AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID) AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID="" ;; PROJECT_EGRESS_PROXY_IMAGE_ID) PROJECT_EGRESS_PROXY_IMAGE_ID="" ;; *) return 1 ;; esac set_env_value_atomic "${env_file}" "${key}" "" } start_docker_for_image_build() { systemctl start docker >> "${LOG_FILE}" 2>&1 \ || fail "Docker could not be started for Project runtime image convergence." docker info >> "${LOG_FILE}" 2>&1 \ || fail "Docker is installed but its daemon is not ready for Project runtime image convergence." } project_egress_artifact_fingerprint() { local artifact_dir="$1" node - \ "${artifact_dir}/projectEgressPolicy.js" \ "${artifact_dir}/projectEgressProxy.js" <<'NODE' const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); const files = process.argv.slice(2); const hash = crypto.createHash('sha256'); hash.update('bridgesllm-project-egress-artifacts-v1\0'); for (const file of files) { hash.update(path.basename(file)); hash.update('\0'); hash.update(fs.readFileSync(file)); hash.update('\0'); } process.stdout.write(hash.digest('hex')); NODE } image_recipe_fingerprint() { local dockerfile="$1" local artifact_fingerprint="${2:-}" node - "${dockerfile}" "${artifact_fingerprint}" <<'NODE' const crypto = require('crypto'); const fs = require('fs'); const dockerfile = process.argv[2]; const artifactFingerprint = process.argv[3] || ''; const hash = crypto.createHash('sha256'); hash.update('bridgesllm-installer-image-recipe-v1\0'); hash.update(fs.readFileSync(dockerfile)); hash.update('\0'); hash.update(artifactFingerprint); process.stdout.write(hash.digest('hex')); NODE } root_protected_regular_file_path() { local target="$1" [[ "${target}" == /* && -f "${target}" && ! -L "${target}" ]] || return 1 [[ "$(stat -c '%u' "${target}" 2>/dev/null || true)" == "0" ]] || return 1 (( (8#$(stat -c '%a' "${target}" 2>/dev/null || printf '777') & 0022) == 0 )) || return 1 local directory directory="$(dirname -- "${target}")" while true; do [[ -d "${directory}" && ! -L "${directory}" ]] || return 1 [[ "$(stat -c '%u' "${directory}" 2>/dev/null || true)" == "0" ]] || return 1 (( (8#$(stat -c '%a' "${directory}" 2>/dev/null || printf '777') & 0022) == 0 )) || return 1 [[ "${directory}" == "/" ]] && break directory="$(dirname -- "${directory}")" done } copy_root_protected_regular_file() { local source="$1" destination="$2" python3 - "${source}" "${destination}" <<'PY' import os import stat import sys source, destination = sys.argv[1:3] source_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) destination_flags = ( os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) ) source_fd = os.open(source, source_flags) destination_fd = -1 try: source_stat = os.fstat(source_fd) if ( not stat.S_ISREG(source_stat.st_mode) or source_stat.st_uid != 0 or source_stat.st_mode & 0o022 ): raise OSError("source is not a root-protected regular file") destination_fd = os.open(destination, destination_flags, 0o644) while True: chunk = os.read(source_fd, 1024 * 1024) if not chunk: break view = memoryview(chunk) while view: written = os.write(destination_fd, view) if written <= 0: raise OSError("short write while copying protected recipe") view = view[written:] os.fchmod(destination_fd, 0o644) os.fsync(destination_fd) finally: if destination_fd >= 0: os.close(destination_fd) os.close(source_fd) PY } normalize_agent_zero_project_architecture() { case "${1:-}" in x86_64|amd64|x64) printf 'amd64\n' ;; aarch64|arm64) printf 'arm64\n' ;; *) return 1 ;; esac } agent_zero_project_upstream_digest() { case "${1:-}" in amd64) printf '%s\n' "${AGENT_ZERO_PROJECT_AMD64_UPSTREAM_DIGEST}" ;; arm64) printf '%s\n' "${AGENT_ZERO_PROJECT_ARM64_UPSTREAM_DIGEST}" ;; *) return 1 ;; esac } write_openclaw_sandbox_dockerfile() { local destination="$1" cat > "${destination}" <<'SANDBOX_DOCKERFILE' FROM node:22.23.1-bookworm-slim ARG PORTAL_RECIPE_SHA256 LABEL com.bridgesllm.openclaw-sandbox.recipe-sha256="${PORTAL_RECIPE_SHA256}" ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ ca-certificates \ curl \ git \ jq \ python3 \ python3-pip \ python3-venv \ ripgrep \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /home/openclaw \ && chown 1000:1000 /home/openclaw USER 1000:1000 WORKDIR /home/openclaw ENTRYPOINT [] CMD ["sleep", "infinity"] SANDBOX_DOCKERFILE } write_codex_project_sandbox_dockerfile() { local destination="$1" cat > "${destination}" <<'CODEX_PROJECT_DOCKERFILE' FROM node:22.23.1-bookworm-slim ARG PORTAL_RECIPE_SHA256 ARG CODEX_CLI_VERSION LABEL com.bridgesllm.codex-project.recipe-sha256="${PORTAL_RECIPE_SHA256}" LABEL com.bridgesllm.codex-project.cli-version="${CODEX_CLI_VERSION}" ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ build-essential \ ca-certificates \ curl \ git \ jq \ openssh-client \ python3 \ python3-pip \ python3-venv \ ripgrep \ unzip \ && rm -rf /var/lib/apt/lists/* RUN npm install --global --omit=dev "@openai/codex@${CODEX_CLI_VERSION}" \ && ln -sf /usr/local/bin/codex /usr/bin/codex \ && codex --version | grep -F "${CODEX_CLI_VERSION}" RUN mkdir -p /home/codex /workspace/project \ && chown -R 1000:1000 /home/codex /workspace ENV HOME=/home/codex CODEX_HOME=/home/codex/.codex LANG=C.UTF-8 LC_ALL=C.UTF-8 USER 1000:1000 WORKDIR /workspace/project ENTRYPOINT [] CMD ["node", "-e", "setInterval(()=>{},2147483647)"] CODEX_PROJECT_DOCKERFILE } write_claude_code_project_sandbox_dockerfile() { local destination="$1" cat > "${destination}" <<'CLAUDE_CODE_PROJECT_DOCKERFILE' FROM node:22.23.1-bookworm-slim ARG PORTAL_RECIPE_SHA256 ARG CLAUDE_CODE_VERSION LABEL com.bridgesllm.claude-code-project.recipe-sha256="${PORTAL_RECIPE_SHA256}" LABEL com.bridgesllm.claude-code-project.cli-version="${CLAUDE_CODE_VERSION}" ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ bubblewrap \ build-essential \ ca-certificates \ curl \ git \ jq \ openssh-client \ procps \ python3 \ python3-pip \ python3-venv \ ripgrep \ socat \ unzip \ && rm -rf /var/lib/apt/lists/* RUN npm install --global --omit=dev "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ && claude --version | grep -F "${CLAUDE_CODE_VERSION}" RUN mkdir -p /home/project-agent /workspace/project \ && chown -R 1000:1000 /home/project-agent /workspace ENV HOME=/home/project-agent LANG=C.UTF-8 LC_ALL=C.UTF-8 CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 DISABLE_AUTOUPDATER=1 DISABLE_TELEMETRY=1 USER 1000:1000 WORKDIR /workspace/project ENTRYPOINT [] CMD ["node", "-e", "setInterval(()=>{},2147483647)"] CLAUDE_CODE_PROJECT_DOCKERFILE } write_antigravity_project_sandbox_dockerfile() { local destination="$1" cat > "${destination}" <<'ANTIGRAVITY_PROJECT_DOCKERFILE' FROM node:22.23.1-bookworm-slim ARG PORTAL_RECIPE_SHA256 ARG ANTIGRAVITY_VERSION ARG ANTIGRAVITY_BINARY_SHA256 LABEL com.bridgesllm.antigravity-project.recipe-sha256="${PORTAL_RECIPE_SHA256}" LABEL com.bridgesllm.antigravity-project.cli-version="${ANTIGRAVITY_VERSION}" LABEL com.bridgesllm.antigravity-project.binary-sha256="${ANTIGRAVITY_BINARY_SHA256}" ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ bubblewrap \ build-essential \ ca-certificates \ curl \ git \ jq \ openssh-client \ procps \ python3 \ python3-pip \ python3-venv \ ripgrep \ socat \ unzip \ && rm -rf /var/lib/apt/lists/* COPY agy /usr/local/bin/agy RUN chmod 0755 /usr/local/bin/agy \ && AGY_CLI_DISABLE_AUTO_UPDATE=1 agy --version | grep -F "${ANTIGRAVITY_VERSION}" \ && echo "${ANTIGRAVITY_BINARY_SHA256} /usr/local/bin/agy" | sha256sum --check --status RUN mkdir -p /home/project-agent /workspace/project \ && chown -R 1000:1000 /home/project-agent /workspace ENV HOME=/home/project-agent LANG=C.UTF-8 LC_ALL=C.UTF-8 AGY_CLI_DISABLE_AUTO_UPDATE=1 GOOGLE_CLOUD_TELEMETRY_DISABLED=1 USER 1000:1000 WORKDIR /workspace/project ENTRYPOINT [] CMD ["node", "-e", "setInterval(()=>{},2147483647)"] ANTIGRAVITY_PROJECT_DOCKERFILE } write_ollama_project_sandbox_dockerfile() { local destination="$1" cat > "${destination}" <<'OLLAMA_PROJECT_DOCKERFILE' FROM node:22.23.1-bookworm-slim ARG PORTAL_RECIPE_SHA256 LABEL com.bridgesllm.ollama-project.recipe-sha256="${PORTAL_RECIPE_SHA256}" ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ build-essential \ ca-certificates \ git \ python3 \ ripgrep \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /home/project-agent /workspace/project \ && chown -R 1000:1000 /home/project-agent /workspace ENV HOME=/home/project-agent LANG=C.UTF-8 LC_ALL=C.UTF-8 USER 1000:1000 WORKDIR /workspace/project ENTRYPOINT [] CMD ["node", "-e", "setInterval(()=>{},2147483647)"] OLLAMA_PROJECT_DOCKERFILE } write_project_egress_proxy_dockerfile() { local destination="$1" cat > "${destination}" <<'PROJECT_EGRESS_DOCKERFILE' FROM node:22.16.0-bookworm-slim ARG PORTAL_RECIPE_SHA256 ARG PORTAL_ARTIFACTS_SHA256 LABEL com.bridgesllm.project-egress.recipe-sha256="${PORTAL_RECIPE_SHA256}" LABEL com.bridgesllm.project-egress.artifacts-sha256="${PORTAL_ARTIFACTS_SHA256}" COPY --chown=1000:1000 projectEgressPolicy.js projectEgressProxy.js /opt/bridgesllm/backend/dist/services/ USER 1000:1000 WORKDIR /opt/bridgesllm/backend ENTRYPOINT [] CMD ["node", "/opt/bridgesllm/backend/dist/services/projectEgressProxy.js"] PROJECT_EGRESS_DOCKERFILE } verify_openclaw_sandbox_image() { local image="$1" local recipe_fingerprint="$2" local -a confinement_args=() project_runtime_confinement_docker_args confinement_args [[ "$(docker_image_label "${image}" "${OPENCLAW_SANDBOX_RECIPE_LABEL}" || true)" == "${recipe_fingerprint}" ]] \ || return 1 [[ "$(docker image inspect --format '{{.Config.User}}' "${image}" 2>/dev/null || true)" == "1000:1000" ]] \ || return 1 docker run --rm \ --network none \ --read-only \ --cap-drop ALL \ "${confinement_args[@]}" \ --pids-limit 64 \ --memory 256m \ --entrypoint sh \ "${image}" -lc \ 'test "$(id -u)" = 1000 && test "$(id -g)" = 1000 && test "$(node --version)" = v22.23.1 && npm --version >/dev/null 2>&1 && python3 --version >/dev/null 2>&1 && python3 -m venv --help >/dev/null 2>&1 && command -v curl >/dev/null 2>&1 && command -v rg >/dev/null 2>&1 && command -v git >/dev/null 2>&1 && command -v jq >/dev/null 2>&1' \ >/dev/null 2>&1 } verify_codex_project_sandbox_image() { local image="$1" local recipe_fingerprint="$2" local -a confinement_args=() codex_project_runtime_confinement_docker_args confinement_args || return 1 [[ "$(docker_image_label "${image}" "${CODEX_PROJECT_RECIPE_LABEL}" || true)" == "${recipe_fingerprint}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${CODEX_PROJECT_CLI_LABEL}" || true)" == "${PIN_CODEX_CLI_VERSION}" ]] \ || return 1 [[ "$(docker image inspect --format '{{.Config.User}}' "${image}" 2>/dev/null || true)" == "1000:1000" ]] \ || return 1 docker run --rm \ --network none \ --read-only \ --cap-drop ALL \ "${confinement_args[@]}" \ --pids-limit 64 \ --memory 384m \ --tmpfs /tmp:rw,noexec,nosuid,nodev,size=33554432 \ --tmpfs /run:rw,noexec,nosuid,nodev,size=16777216 \ --tmpfs /home/codex/.codex:rw,noexec,nosuid,nodev,size=33554432,uid=1000,gid=1000,mode=0700 \ --tmpfs /workspace/project:rw,nosuid,nodev,size=33554432,uid=1000,gid=1000,mode=0755 \ --workdir /workspace/project \ --entrypoint sh \ "${image}" -lc \ 'set -eu test "$(id -u)" = 1000 test "$(id -g)" = 1000 test "$(node --version)" = v22.23.1 codex --version | grep -F "'"${PIN_CODEX_CLI_VERSION}"'" >/dev/null command -v npm >/dev/null command -v python3 >/dev/null command -v rg >/dev/null command -v git >/dev/null command -v curl >/dev/null cat > "${CODEX_HOME}/portal-project.config.toml" < "${CODEX_HOME}/auth.json" cat > /workspace/project/.portal-installer-sandbox-smoke.sh <<\CODEX_SMOKE #!/bin/sh set -eu probe=.portal-installer-sandbox-probe printf workspace-ok > "$probe" test "$(cat "$probe")" = workspace-ok rm -f "$probe" if cat /home/codex/.codex/auth.json >/dev/null 2>&1; then exit 61 fi printf sandbox-ok CODEX_SMOKE chmod 0700 /workspace/project/.portal-installer-sandbox-smoke.sh test "$(/usr/bin/codex sandbox --profile portal-project --permission-profile portal_project --cd /workspace/project /bin/sh /workspace/project/.portal-installer-sandbox-smoke.sh)" = "sandbox-ok"' \ >/dev/null 2>&1 } degrade_codex_project_runtime() { local env_file="$1" local reason="$2" clear_installer_image_id "${env_file}" "CODEX_PROJECT_SANDBOX_IMAGE_ID" \ || warn "Could not clear the unavailable Codex Project runtime image identity." warn "${reason} Other confined Project providers remain available." } verify_claude_code_project_sandbox_image() { local image="$1" local recipe_fingerprint="$2" local -a confinement_args=() project_runtime_confinement_docker_args confinement_args [[ "$(docker_image_label "${image}" "${CLAUDE_CODE_PROJECT_RECIPE_LABEL}" || true)" == "${recipe_fingerprint}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${CLAUDE_CODE_PROJECT_CLI_LABEL}" || true)" == "${PIN_CLAUDE_CODE_VERSION}" ]] \ || return 1 [[ "$(docker image inspect --format '{{.Config.User}}' "${image}" 2>/dev/null || true)" == "1000:1000" ]] \ || return 1 docker run --rm \ --network none \ --read-only \ --cap-drop ALL \ "${confinement_args[@]}" \ --pids-limit 64 \ --memory 384m \ --tmpfs /tmp:rw,noexec,nosuid,nodev,size=33554432 \ --tmpfs /home/project-agent:rw,noexec,nosuid,nodev,size=33554432,uid=1000,gid=1000,mode=0700 \ --entrypoint sh \ "${image}" -lc \ 'test "$(id -u)" = 1000 && test "$(id -g)" = 1000 && test "$(node --version)" = v22.23.1 && claude --version | grep -F "'"${PIN_CLAUDE_CODE_VERSION}"'" >/dev/null && command -v bwrap >/dev/null && command -v socat >/dev/null && command -v python3 >/dev/null && command -v rg >/dev/null && command -v git >/dev/null && command -v curl >/dev/null' \ >/dev/null 2>&1 } verify_antigravity_project_sandbox_image() { local image="$1" local recipe_fingerprint="$2" local binary_fingerprint="$3" local -a confinement_args=() project_runtime_confinement_docker_args confinement_args [[ "$(docker_image_label "${image}" "${ANTIGRAVITY_PROJECT_RECIPE_LABEL}" || true)" == "${recipe_fingerprint}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${ANTIGRAVITY_PROJECT_CLI_LABEL}" || true)" == "${PIN_ANTIGRAVITY_VERSION}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${ANTIGRAVITY_PROJECT_BINARY_LABEL}" || true)" == "${binary_fingerprint}" ]] \ || return 1 [[ "$(docker image inspect --format '{{.Config.User}}' "${image}" 2>/dev/null || true)" == "1000:1000" ]] \ || return 1 docker run --rm \ --network none \ --read-only \ --cap-drop ALL \ "${confinement_args[@]}" \ --pids-limit 64 \ --memory 384m \ --tmpfs /tmp:rw,noexec,nosuid,nodev,size=33554432 \ --tmpfs /home/project-agent:rw,noexec,nosuid,nodev,size=33554432,uid=1000,gid=1000,mode=0700 \ --entrypoint sh \ "${image}" -lc \ 'test "$(id -u)" = 1000 && test "$(id -g)" = 1000 && test "$(node --version)" = v22.23.1 && AGY_CLI_DISABLE_AUTO_UPDATE=1 agy --version | grep -F "'"${PIN_ANTIGRAVITY_VERSION}"'" >/dev/null && test "$(sha256sum /usr/local/bin/agy | awk '\''{print $1}'\'')" = "'"${binary_fingerprint}"'" && command -v bwrap >/dev/null && command -v socat >/dev/null && command -v python3 >/dev/null && command -v rg >/dev/null && command -v git >/dev/null && command -v curl >/dev/null' \ >/dev/null 2>&1 } verify_ollama_project_sandbox_image() { local image="$1" local recipe_fingerprint="$2" local -a confinement_args=() project_runtime_confinement_docker_args confinement_args [[ "$(docker_image_label "${image}" "${OLLAMA_PROJECT_RECIPE_LABEL}" || true)" == "${recipe_fingerprint}" ]] \ || return 1 [[ "$(docker image inspect --format '{{.Config.User}}' "${image}" 2>/dev/null || true)" == "1000:1000" ]] \ || return 1 docker run --rm \ --network none \ --read-only \ --cap-drop ALL \ "${confinement_args[@]}" \ --pids-limit 64 \ --memory 384m \ --tmpfs /tmp:rw,noexec,nosuid,nodev,size=33554432 \ --tmpfs /home/project-agent:rw,noexec,nosuid,nodev,size=16777216,uid=1000,gid=1000,mode=0700 \ --entrypoint sh \ "${image}" -lc \ 'test "$(id -u)" = 1000 && test "$(id -g)" = 1000 && test "$(node --version)" = v22.23.1 && command -v npm >/dev/null && command -v python3 >/dev/null && command -v rg >/dev/null && command -v git >/dev/null && command -v cc >/dev/null && command -v make >/dev/null' \ >/dev/null 2>&1 } verify_agent_zero_project_sandbox_image() { local image="$1" local recipe_fingerprint="$2" local upstream_digest="$3" local source_commit="$4" local image_command image_entrypoint image_workdir local -a confinement_args=() project_runtime_confinement_docker_args confinement_args [[ "$(docker_image_label "${image}" "${AGENT_ZERO_PROJECT_RECIPE_LABEL}" || true)" == "${recipe_fingerprint}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${AGENT_ZERO_PROJECT_SOURCE_COMMIT_LABEL}" || true)" == "${source_commit}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${AGENT_ZERO_PROJECT_UPSTREAM_DIGEST_LABEL}" || true)" == "${upstream_digest}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${AGENT_ZERO_PROJECT_RUNTIME_USER_LABEL}" || true)" == "${AGENT_ZERO_PROJECT_RUNTIME_USER}" ]] \ || return 1 [[ "$(docker image inspect --format '{{.Config.User}}' "${image}" 2>/dev/null || true)" == "${AGENT_ZERO_PROJECT_RUNTIME_USER}" ]] \ || return 1 image_command="$(docker image inspect --format '{{json .Config.Cmd}}' "${image}" 2>/dev/null || true)" [[ "${image_command}" == '["/opt/venv-a0/bin/python","/a0/run_ui.py","--dockerized=true","--port=80","--host=0.0.0.0"]' ]] \ || return 1 image_entrypoint="$(docker image inspect --format '{{json .Config.Entrypoint}}' "${image}" 2>/dev/null || true)" [[ -z "${image_entrypoint}" || "${image_entrypoint}" == "null" || "${image_entrypoint}" == "[]" ]] \ || return 1 image_workdir="$(docker image inspect --format '{{.Config.WorkingDir}}' "${image}" 2>/dev/null || true)" [[ "${image_workdir}" == "/a0" ]] || return 1 docker run --rm \ --network none \ --read-only \ --cap-drop ALL \ "${confinement_args[@]}" \ --pids-limit 64 \ --memory 512m \ --tmpfs /tmp:rw,noexec,nosuid,nodev,size=33554432,mode=1777 \ --tmpfs /a0/tmp:rw,noexec,nosuid,nodev,size=33554432,mode=1777 \ --mount type=volume,target=/a0/usr \ --entrypoint sh \ "${image}" -lc \ 'test "$(id -u):$(id -g)" = "1000:1000" \ && test -x /opt/venv-a0/bin/python \ && test -f /a0/run_ui.py \ && test -f /a0/plugins/_a0_connector/api/v1/capabilities.py \ && test -f /a0/plugins/_a0_connector/api/ws_connector.py \ && test ! -e /a0/.git \ && for runtime_dir in /a0/usr/agents /a0/usr/home /a0/usr/knowledge /a0/usr/plugins /a0/usr/projects /a0/usr/skills /a0/usr/workdir; do \ test -d "${runtime_dir}" \ && test ! -L "${runtime_dir}" \ && test "$(stat -c "%u:%g" "${runtime_dir}")" = "1000:1000"; \ done \ && /opt/venv-a0/bin/python -c "import pathlib; assert pathlib.Path(\"/a0/run_ui.py\").is_file()"' \ >/dev/null 2>&1 } verify_project_egress_proxy_image() { local image="$1" local recipe_fingerprint="$2" local artifact_fingerprint="$3" local verify_script local -a confinement_args=() project_runtime_confinement_docker_args confinement_args verify_script='const crypto=require("crypto");const fs=require("fs");const path=require("path");const files=["/opt/bridgesllm/backend/dist/services/projectEgressPolicy.js","/opt/bridgesllm/backend/dist/services/projectEgressProxy.js"];const hash=crypto.createHash("sha256");hash.update("bridgesllm-project-egress-artifacts-v1\0");for(const file of files){hash.update(path.basename(file));hash.update("\0");hash.update(fs.readFileSync(file));hash.update("\0");}if(hash.digest("hex")!==process.argv[1])process.exit(11);const policy=require(files[0]);const proxy=require(files[1]);if(policy.PROJECT_EGRESS_POLICY_VERSION!=="portal-project-egress-v1"||typeof proxy.createProjectEgressProxyServer!=="function")process.exit(12);if(typeof process.getuid!=="function"||process.getuid()!==1000||process.getgid()!==1000)process.exit(13);' [[ "$(docker_image_label "${image}" "${PROJECT_EGRESS_RECIPE_LABEL}" || true)" == "${recipe_fingerprint}" ]] \ || return 1 [[ "$(docker_image_label "${image}" "${PROJECT_EGRESS_ARTIFACTS_LABEL}" || true)" == "${artifact_fingerprint}" ]] \ || return 1 [[ "$(docker image inspect --format '{{.Config.User}}' "${image}" 2>/dev/null || true)" == "1000:1000" ]] \ || return 1 docker run --rm \ --network none \ --read-only \ --cap-drop ALL \ "${confinement_args[@]}" \ --pids-limit 64 \ --memory 256m \ --entrypoint node \ "${image}" -e "${verify_script}" "${artifact_fingerprint}" \ >/dev/null 2>&1 } ensure_openclaw_sandbox_image() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local image="${2:-${OPENCLAW_PROJECT_SANDBOX_IMAGE_TAG}}" [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for OpenClaw sandbox image attestation." if $SKIP_OPENCLAW || ! command -v openclaw &>/dev/null; then clear_installer_image_id "${env_file}" "OPENCLAW_PROJECT_SANDBOX_IMAGE_ID" \ || fail "Could not clear the unavailable OpenClaw sandbox image ID." return 0 fi command -v docker &>/dev/null \ || fail "Docker is required for OpenClaw project sandboxes, but docker is not installed." valid_local_image_tag "${image}" || fail "OpenClaw sandbox image tag is invalid." start_docker_for_image_build local build_dir dockerfile recipe_fingerprint image_id build_dir="$(mktemp -d)" dockerfile="${build_dir}/Dockerfile" write_openclaw_sandbox_dockerfile "${dockerfile}" recipe_fingerprint="$(image_recipe_fingerprint "${dockerfile}")" [[ "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]] || { rm -rf -- "${build_dir}" fail "Could not fingerprint the OpenClaw sandbox recipe." } if verify_openclaw_sandbox_image "${image}" "${recipe_fingerprint}"; then ok "OpenClaw sandbox image recipe (verified)" else if docker image inspect "${image}" >/dev/null 2>&1; then warn "Existing OpenClaw sandbox image recipe drifted; rebuilding deterministically." fi if ! spin "Building OpenClaw project sandbox image" \ "docker build --pull --tag '${image}' --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' '${build_dir}'"; then rm -rf -- "${build_dir}" fail "Failed to build ${image}. Project assistants require this exact sandbox recipe." fi if ! verify_openclaw_sandbox_image "${image}" "${recipe_fingerprint}"; then rm -rf -- "${build_dir}" fail "Built ${image}, but its recipe label, non-root identity, or required tools failed attestation." fi fi rm -rf -- "${build_dir}" image_id="$(docker_image_id "${image}" || true)" valid_docker_image_id "${image_id}" \ || fail "OpenClaw sandbox image did not resolve to an immutable Docker image ID." verify_openclaw_sandbox_image "${image_id}" "${recipe_fingerprint}" \ || fail "The resolved OpenClaw sandbox image ID failed immutable recipe/runtime attestation." record_installer_image_id "${env_file}" "OPENCLAW_PROJECT_SANDBOX_IMAGE_ID" "${image_id}" \ || fail "OpenClaw sandbox image did not resolve and persist an immutable Docker image ID." ok "OpenClaw sandbox image ${image_id:0:19}… (pinned)" } ensure_codex_project_sandbox_image() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local image="${2:-${CODEX_PROJECT_SANDBOX_IMAGE_TAG}}" [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for Codex Project runtime image attestation." if ! ${CODEX_PROJECT_RUNTIME_CONFINEMENT_AVAILABLE}; then degrade_codex_project_runtime \ "${env_file}" \ "Codex Project runtime image convergence was skipped because its dedicated confinement profile is unavailable." return 0 fi command -v docker &>/dev/null \ || fail "Docker is required for Codex Project sandboxes, but docker is not installed." valid_local_image_tag "${image}" || fail "Codex Project runtime image tag is invalid." start_docker_for_image_build local build_dir dockerfile recipe_fingerprint image_id build_dir="$(mktemp -d)" dockerfile="${build_dir}/Dockerfile" write_codex_project_sandbox_dockerfile "${dockerfile}" recipe_fingerprint="$(image_recipe_fingerprint "${dockerfile}" "codex-cli=${PIN_CODEX_CLI_VERSION}")" [[ "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]] || { rm -rf -- "${build_dir}" fail "Could not fingerprint the Codex Project runtime recipe." } if verify_codex_project_sandbox_image "${image}" "${recipe_fingerprint}"; then ok "Codex Project runtime image recipe/CLI (verified)" else if docker image inspect "${image}" >/dev/null 2>&1; then warn "Existing Codex Project runtime drifted; rebuilding the tested CLI image." fi if ! spin "Building Codex Project sandbox image" \ "docker build --pull --tag '${image}' --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' --build-arg 'CODEX_CLI_VERSION=${PIN_CODEX_CLI_VERSION}' '${build_dir}'"; then rm -rf -- "${build_dir}" degrade_codex_project_runtime \ "${env_file}" \ "Codex Project runtime ${image} could not be built with Codex ${PIN_CODEX_CLI_VERSION}." return 0 fi if ! verify_codex_project_sandbox_image "${image}" "${recipe_fingerprint}"; then rm -rf -- "${build_dir}" degrade_codex_project_runtime \ "${env_file}" \ "Codex Project runtime ${image} failed its dedicated recipe, CLI, confinement, or sandbox smoke attestation." return 0 fi fi rm -rf -- "${build_dir}" image_id="$(docker_image_id "${image}" || true)" if ! valid_docker_image_id "${image_id}"; then degrade_codex_project_runtime \ "${env_file}" \ "Codex Project runtime image did not resolve to an immutable Docker image ID." return 0 fi if ! verify_codex_project_sandbox_image "${image_id}" "${recipe_fingerprint}"; then degrade_codex_project_runtime \ "${env_file}" \ "The resolved Codex Project runtime image ID failed immutable recipe/runtime attestation." return 0 fi if ! record_installer_image_id "${env_file}" "CODEX_PROJECT_SANDBOX_IMAGE_ID" "${image_id}"; then degrade_codex_project_runtime \ "${env_file}" \ "Codex Project runtime could not persist its immutable Docker image ID." return 0 fi ok "Codex Project runtime image ${image_id:0:19}… (pinned)" } ensure_claude_code_project_sandbox_image() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local image="${2:-${CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_TAG}}" [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for Claude Code Project runtime image attestation." command -v docker &>/dev/null \ || fail "Docker is required for Claude Code Project sandboxes, but docker is not installed." valid_local_image_tag "${image}" || fail "Claude Code Project runtime image tag is invalid." start_docker_for_image_build local build_dir dockerfile recipe_fingerprint image_id build_dir="$(mktemp -d)" dockerfile="${build_dir}/Dockerfile" write_claude_code_project_sandbox_dockerfile "${dockerfile}" recipe_fingerprint="$(image_recipe_fingerprint "${dockerfile}" "claude-code=${PIN_CLAUDE_CODE_VERSION};bubblewrap+socat=required")" [[ "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]] || { rm -rf -- "${build_dir}" fail "Could not fingerprint the Claude Code Project runtime recipe." } if verify_claude_code_project_sandbox_image "${image}" "${recipe_fingerprint}"; then ok "Claude Code Project runtime image recipe/CLI (verified)" else if docker image inspect "${image}" >/dev/null 2>&1; then warn "Existing Claude Code Project runtime drifted; rebuilding the tested CLI image." fi if ! spin "Building Claude Code Project sandbox image" \ "docker build --pull --tag '${image}' --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' --build-arg 'CLAUDE_CODE_VERSION=${PIN_CLAUDE_CODE_VERSION}' '${build_dir}'"; then rm -rf -- "${build_dir}" fail "Failed to build ${image} with Claude Code ${PIN_CLAUDE_CODE_VERSION}." fi if ! verify_claude_code_project_sandbox_image "${image}" "${recipe_fingerprint}"; then rm -rf -- "${build_dir}" fail "Built ${image}, but its recipe, CLI pin, inner-sandbox tools, or non-root identity failed attestation." fi fi rm -rf -- "${build_dir}" image_id="$(docker_image_id "${image}" || true)" valid_docker_image_id "${image_id}" \ || fail "Claude Code Project runtime image did not resolve to an immutable Docker image ID." verify_claude_code_project_sandbox_image "${image_id}" "${recipe_fingerprint}" \ || fail "The resolved Claude Code Project runtime image ID failed immutable recipe/runtime attestation." record_installer_image_id "${env_file}" "CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID" "${image_id}" \ || fail "Claude Code Project runtime did not persist an immutable Docker image ID." ok "Claude Code Project runtime image ${image_id:0:19}… (pinned)" } ensure_antigravity_project_sandbox_image() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local image="${2:-${ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_TAG}}" local host_binary="${3:-/usr/local/bin/agy}" [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for Antigravity Project runtime image attestation." if [[ ! -f "${host_binary}" || -L "${host_binary}" || ! -x "${host_binary}" ]] \ || [[ "$(AGY_CLI_DISABLE_AUTO_UPDATE=1 "${host_binary}" --version 2>/dev/null | sed -nE 's/.*(^|[^0-9])([0-9]+\.[0-9]+\.[0-9]+)([^0-9].*|$)/\2/p' | head -1)" != "${PIN_ANTIGRAVITY_VERSION}" ]]; then clear_installer_image_id "${env_file}" "ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID" \ || fail "Could not clear the unavailable Antigravity Project runtime image ID." warn "Antigravity ${PIN_ANTIGRAVITY_VERSION} is not verified; its Project provider remains unavailable." return 0 fi command -v docker &>/dev/null \ || fail "Docker is required for Antigravity Project sandboxes, but docker is not installed." valid_local_image_tag "${image}" || fail "Antigravity Project runtime image tag is invalid." start_docker_for_image_build local build_dir dockerfile recipe_fingerprint binary_fingerprint image_id build_dir="$(mktemp -d)" dockerfile="${build_dir}/Dockerfile" install -m 0755 "${host_binary}" "${build_dir}/agy" binary_fingerprint="$(sha256sum "${build_dir}/agy" | awk '{print $1}')" [[ "${binary_fingerprint}" =~ ^[a-f0-9]{64}$ ]] || { rm -rf -- "${build_dir}" fail "Could not fingerprint the verified Antigravity binary." } write_antigravity_project_sandbox_dockerfile "${dockerfile}" recipe_fingerprint="$(image_recipe_fingerprint "${dockerfile}" "antigravity=${PIN_ANTIGRAVITY_VERSION};binary=${binary_fingerprint};bubblewrap+socat=required")" [[ "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]] || { rm -rf -- "${build_dir}" fail "Could not fingerprint the Antigravity Project runtime recipe." } if verify_antigravity_project_sandbox_image "${image}" "${recipe_fingerprint}" "${binary_fingerprint}"; then ok "Antigravity Project runtime image recipe/CLI (verified)" else if docker image inspect "${image}" >/dev/null 2>&1; then warn "Existing Antigravity Project runtime drifted; rebuilding the tested CLI image." fi if ! spin "Building Antigravity Project sandbox image" \ "docker build --pull --tag '${image}' --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' --build-arg 'ANTIGRAVITY_VERSION=${PIN_ANTIGRAVITY_VERSION}' --build-arg 'ANTIGRAVITY_BINARY_SHA256=${binary_fingerprint}' '${build_dir}'"; then rm -rf -- "${build_dir}" fail "Failed to build ${image} with Antigravity ${PIN_ANTIGRAVITY_VERSION}." fi if ! verify_antigravity_project_sandbox_image "${image}" "${recipe_fingerprint}" "${binary_fingerprint}"; then rm -rf -- "${build_dir}" fail "Built ${image}, but its recipe, CLI pin, binary hash, inner-sandbox tools, or non-root identity failed attestation." fi fi rm -rf -- "${build_dir}" image_id="$(docker_image_id "${image}" || true)" valid_docker_image_id "${image_id}" \ || fail "Antigravity Project runtime image did not resolve to an immutable Docker image ID." verify_antigravity_project_sandbox_image "${image_id}" "${recipe_fingerprint}" "${binary_fingerprint}" \ || fail "The resolved Antigravity Project runtime image ID failed immutable recipe/runtime attestation." record_installer_image_id "${env_file}" "ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID" "${image_id}" \ || fail "Antigravity Project runtime did not persist an immutable Docker image ID." ok "Antigravity Project runtime image ${image_id:0:19}… (pinned)" } ensure_ollama_project_sandbox_image() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local image="${2:-${OLLAMA_PROJECT_SANDBOX_IMAGE_TAG}}" [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for Ollama Project runtime image attestation." # This image is the networkless coding-tool plane; it contains no Ollama # client and never performs model inference. Build/attest it independently # from the optional local Ollama install so a verified Tailnet authority can # qualify against the same immutable Project sandbox. command -v docker &>/dev/null \ || fail "Docker is required for Ollama Project sandboxes, but docker is not installed." valid_local_image_tag "${image}" || fail "Ollama Project runtime image tag is invalid." start_docker_for_image_build local build_dir dockerfile recipe_fingerprint image_id build_dir="$(mktemp -d)" dockerfile="${build_dir}/Dockerfile" write_ollama_project_sandbox_dockerfile "${dockerfile}" recipe_fingerprint="$(image_recipe_fingerprint "${dockerfile}" "node=22.23.1;uid=1000;network=none;coding-toolchain=v1")" [[ "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]] || { rm -rf -- "${build_dir}" fail "Could not fingerprint the Ollama Project runtime recipe." } if verify_ollama_project_sandbox_image "${image}" "${recipe_fingerprint}"; then ok "Ollama Project runtime image recipe/toolchain (verified)" else if docker image inspect "${image}" >/dev/null 2>&1; then warn "Existing Ollama Project runtime drifted; rebuilding the networkless coding image." fi if ! spin "Building Ollama Project sandbox image" \ "docker build --pull --tag '${image}' --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' '${build_dir}'"; then rm -rf -- "${build_dir}" fail "Failed to build ${image}." fi if ! verify_ollama_project_sandbox_image "${image}" "${recipe_fingerprint}"; then rm -rf -- "${build_dir}" fail "Built ${image}, but its recipe, networkless toolchain, or non-root identity failed attestation." fi fi rm -rf -- "${build_dir}" image_id="$(docker_image_id "${image}" || true)" valid_docker_image_id "${image_id}" \ || fail "Ollama Project runtime image did not resolve to an immutable Docker image ID." verify_ollama_project_sandbox_image "${image_id}" "${recipe_fingerprint}" \ || fail "The resolved Ollama Project runtime image ID failed immutable recipe/runtime attestation." record_installer_image_id "${env_file}" "OLLAMA_PROJECT_SANDBOX_IMAGE_ID" "${image_id}" \ || fail "Ollama Project runtime did not persist an immutable Docker image ID." ok "Ollama Project runtime image ${image_id:0:19}… (pinned)" } ensure_agent_zero_project_sandbox_image() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local image="${2:-${AGENT_ZERO_PROJECT_SANDBOX_IMAGE_TAG}}" local recipe_source="${3:-${PORTAL_DIR}/installer/agent-zero-project-sandbox.Dockerfile}" local requested_architecture="${4:-$(uname -m)}" [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for Agent Zero Project runtime image attestation." root_protected_regular_file_path "${recipe_source}" \ || fail "The Agent Zero Project runtime image recipe or one of its ancestors is not root-protected." command -v docker &>/dev/null \ || fail "Docker is required for Agent Zero Project sandboxes, but docker is not installed." valid_local_image_tag "${image}" || fail "Agent Zero Project runtime image tag is invalid." start_docker_for_image_build local architecture upstream_digest upstream_ref architecture="$(normalize_agent_zero_project_architecture "${requested_architecture}" || true)" if [[ -z "${architecture}" ]]; then clear_installer_image_id "${env_file}" "AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID" \ || fail "Could not clear the unsupported Agent Zero Project runtime image ID." fail "Agent Zero Project sandboxes do not support host architecture ${requested_architecture}." fi upstream_digest="$(agent_zero_project_upstream_digest "${architecture}" || true)" valid_docker_image_id "${upstream_digest}" \ || fail "Agent Zero Project upstream image digest is invalid for ${architecture}." [[ "${AGENT_ZERO_PROJECT_SOURCE_COMMIT}" =~ ^[a-f0-9]{40}$ ]] \ || fail "Agent Zero Project source commit pin is malformed." upstream_ref="agent0ai/agent-zero@${upstream_digest}" local build_dir dockerfile recipe_fingerprint image_id build_dir="$(mktemp -d)" dockerfile="${build_dir}/Dockerfile" copy_root_protected_regular_file "${recipe_source}" "${dockerfile}" || { rm -rf -- "${build_dir}" fail "The Agent Zero Project runtime image recipe changed or became unsafe while it was being opened." } recipe_fingerprint="$(image_recipe_fingerprint \ "${dockerfile}" \ "agent-zero-project-v1;architecture=${architecture};upstream=${upstream_digest};source=${AGENT_ZERO_PROJECT_SOURCE_COMMIT};runtime-user=${AGENT_ZERO_PROJECT_RUNTIME_USER}")" [[ "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]] || { rm -rf -- "${build_dir}" fail "Could not fingerprint the Agent Zero Project runtime recipe." } if verify_agent_zero_project_sandbox_image \ "${image}" "${recipe_fingerprint}" "${upstream_digest}" "${AGENT_ZERO_PROJECT_SOURCE_COMMIT}"; then ok "Agent Zero Project runtime image recipe/source (verified)" else if docker image inspect "${image}" >/dev/null 2>&1; then warn "Existing Agent Zero Project runtime drifted; rebuilding the audited non-root image." fi if ! spin "Building Agent Zero Project sandbox image" \ "docker build --pull --tag '${image}' --network none --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' --build-arg 'A0_BASE_IMAGE=${upstream_ref}' --build-arg 'A0_SOURCE_COMMIT=${AGENT_ZERO_PROJECT_SOURCE_COMMIT}' --build-arg 'A0_UPSTREAM_DIGEST=${upstream_digest}' '${build_dir}'"; then rm -rf -- "${build_dir}" fail "Failed to build ${image} from the audited Agent Zero v2.5 source." fi if ! verify_agent_zero_project_sandbox_image \ "${image}" "${recipe_fingerprint}" "${upstream_digest}" "${AGENT_ZERO_PROJECT_SOURCE_COMMIT}"; then rm -rf -- "${build_dir}" fail "Built ${image}, but its immutable source, direct-launch command, volume seed, or non-root runtime failed attestation." fi fi rm -rf -- "${build_dir}" image_id="$(docker_image_id "${image}" || true)" valid_docker_image_id "${image_id}" \ || fail "Agent Zero Project runtime image did not resolve to an immutable Docker image ID." [[ "${image_id}" != "${upstream_digest}" ]] \ || fail "Agent Zero Project runtime resolved to the raw upstream manifest instead of the derived Portal image." verify_agent_zero_project_sandbox_image \ "${image_id}" "${recipe_fingerprint}" "${upstream_digest}" "${AGENT_ZERO_PROJECT_SOURCE_COMMIT}" \ || fail "The resolved Agent Zero Project image ID failed immutable source/runtime attestation." record_installer_image_id "${env_file}" "AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID" "${image_id}" \ || fail "Agent Zero Project runtime did not persist an immutable derived image ID." ok "Agent Zero Project runtime image ${image_id:0:19}… (pinned)" } ensure_docker_address_pools() { # Each Project egress plane consumes two bridge networks. Docker's default # address pools top out around 31 networks, which a modest number of # projects exhausts ("all predefined address pools have been fully # subnetted"). Provision a dedicated /16 carved into /28s (4096 networks). command -v docker &>/dev/null || return 0 command -v python3 &>/dev/null || return 0 local daemon_config="/etc/docker/daemon.json" local changed changed="$(python3 - "$daemon_config" <<'POOLS_PY' import json, os, sys path = sys.argv[1] config = {} if os.path.exists(path): try: with open(path) as handle: config = json.load(handle) or {} except Exception: raise SystemExit("unparseable") pools = config.get("default-address-pools") if isinstance(pools, list) and pools: print("unchanged") raise SystemExit(0) config["default-address-pools"] = [{"base": "10.201.0.0/16", "size": 28}] os.makedirs(os.path.dirname(path), exist_ok=True) tmp = f"{path}.portal-tmp" with open(tmp, "w") as handle: json.dump(config, handle, indent=2) handle.write("\n") os.replace(tmp, path) print("updated") POOLS_PY )" || { warn "Could not evaluate Docker daemon config for address pools; leaving it untouched."; return 0; } if [[ "$changed" == "updated" ]]; then info "Expanding Docker bridge address pools for Project sandboxes..." systemctl restart docker >> "$LOG_FILE" 2>&1 || warn "Docker restart after address-pool expansion failed; existing pools remain in use until the next restart." fi } ensure_project_egress_proxy_image() { local env_file="${1:-${PORTAL_DIR}/backend/.env.production}" local artifact_dir="${2:-${PORTAL_DIR}/backend/dist/services}" local image="${3:-${PROJECT_EGRESS_PROXY_IMAGE_TAG}}" local policy_artifact="${artifact_dir}/projectEgressPolicy.js" local proxy_artifact="${artifact_dir}/projectEgressProxy.js" [[ -f "${env_file}" && ! -L "${env_file}" ]] \ || fail "Portal production environment is unavailable for Project egress image attestation." [[ -f "${policy_artifact}" && ! -L "${policy_artifact}" \ && -f "${proxy_artifact}" && ! -L "${proxy_artifact}" ]] \ || fail "Built Project egress proxy/policy artifacts are missing; refusing to start Portal without its egress sidecar image." command -v docker &>/dev/null \ || fail "Docker is required for the Project egress proxy image, but docker is not installed." valid_local_image_tag "${image}" || fail "Project egress proxy image tag is invalid." start_docker_for_image_build local build_dir artifact_fingerprint recipe_fingerprint image_id local in_use_image_id="" discovery_status=0 image_ready=false build_dir="$(mktemp -d)" install -m 0644 "${policy_artifact}" "${build_dir}/projectEgressPolicy.js" install -m 0644 "${proxy_artifact}" "${build_dir}/projectEgressProxy.js" write_project_egress_proxy_dockerfile "${build_dir}/Dockerfile" artifact_fingerprint="$(project_egress_artifact_fingerprint "${build_dir}")" recipe_fingerprint="$(image_recipe_fingerprint "${build_dir}/Dockerfile" "${artifact_fingerprint}")" if [[ ! "${artifact_fingerprint}" =~ ^[a-f0-9]{64}$ \ || ! "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]]; then rm -rf -- "${build_dir}" fail "Could not fingerprint the built Project egress image inputs." fi # Preserve a single exact in-use generation before consulting mutable tags. # This keeps prepared Project planes valid across an update whose equivalent # transaction-scoped build happened to receive another Docker image ID. if in_use_image_id="$( discover_unique_managed_project_egress_proxy_image_id )"; then if [[ "$(docker_image_id "${in_use_image_id}" || true)" \ != "${in_use_image_id}" ]] \ || ! verify_project_egress_proxy_image \ "${in_use_image_id}" "${recipe_fingerprint}" "${artifact_fingerprint}"; then rm -rf -- "${build_dir}" fail "The in-use managed Project egress proxy image does not exactly match this Portal's compiled policy; refusing to strand active Project runtimes." fi if [[ "$(docker_image_id "${image}" || true)" != "${in_use_image_id}" ]] \ || ! verify_project_egress_proxy_image \ "${image}" "${recipe_fingerprint}" "${artifact_fingerprint}"; then if ! docker image tag "${in_use_image_id}" "${image}" \ >> "${LOG_FILE}" 2>&1; then rm -rf -- "${build_dir}" fail "The verified in-use Project egress proxy image could not be assigned to ${image}." fi fi if [[ "$(docker_image_id "${image}" || true)" != "${in_use_image_id}" ]] \ || ! verify_project_egress_proxy_image \ "${image}" "${recipe_fingerprint}" "${artifact_fingerprint}"; then rm -rf -- "${build_dir}" fail "The reused Project egress proxy image failed immutable candidate attestation." fi image_ready=true ok "Project egress proxy image recipe/artifacts (verified in-use generation)" else discovery_status=$? case "${discovery_status}" in 1) ;; 2) rm -rf -- "${build_dir}" fail "Managed Project egress proxy inventory could not be inspected safely." ;; 3) rm -rf -- "${build_dir}" fail "Managed Project egress resource ownership, identity, or proxy/network topology is ambiguous; refusing image convergence." ;; 4) rm -rf -- "${build_dir}" fail "Managed Project egress proxies reference multiple immutable image generations; refusing image convergence." ;; 5) rm -rf -- "${build_dir}" fail "Managed Project egress resources changed during immutable inventory; retry after Project runtime activity is quiet." ;; *) rm -rf -- "${build_dir}" fail "Managed Project egress proxy image discovery returned an invalid state." ;; esac fi if ! ${image_ready}; then if verify_project_egress_proxy_image \ "${image}" "${recipe_fingerprint}" "${artifact_fingerprint}"; then ok "Project egress proxy image recipe/artifacts (verified)" elif [[ "${image}" != "${PROJECT_EGRESS_PROXY_IMAGE_TAG}" ]] \ && verify_project_egress_proxy_image \ "${PROJECT_EGRESS_PROXY_IMAGE_TAG}" \ "${recipe_fingerprint}" "${artifact_fingerprint}"; then image_id="$(docker_image_id "${PROJECT_EGRESS_PROXY_IMAGE_TAG}" || true)" if ! valid_docker_image_id "${image_id}" \ || [[ "$(docker_image_id "${image_id}" || true)" != "${image_id}" ]] \ || ! verify_project_egress_proxy_image \ "${image_id}" "${recipe_fingerprint}" "${artifact_fingerprint}"; then rm -rf -- "${build_dir}" fail "The canonical Project egress proxy tag did not resolve to an exact immutable image." fi if ! docker image tag "${image_id}" "${image}" \ >> "${LOG_FILE}" 2>&1; then rm -rf -- "${build_dir}" fail "The verified canonical Project egress proxy image could not be assigned to ${image}." fi if [[ "$(docker_image_id "${image}" || true)" != "${image_id}" ]] \ || ! verify_project_egress_proxy_image \ "${image}" "${recipe_fingerprint}" "${artifact_fingerprint}"; then rm -rf -- "${build_dir}" fail "The reused canonical Project egress proxy image failed immutable candidate attestation." fi ok "Project egress proxy image recipe/artifacts (reused canonical generation)" else if docker image inspect "${image}" >/dev/null 2>&1; then warn "Existing Project egress proxy image drifted from its built artifacts; rebuilding." fi if ! spin "Building Project egress proxy image" \ "docker build --pull --tag '${image}' --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' --build-arg 'PORTAL_ARTIFACTS_SHA256=${artifact_fingerprint}' '${build_dir}'"; then rm -rf -- "${build_dir}" fail "Failed to build ${image} from the Portal's compiled egress proxy/policy artifacts." fi if ! verify_project_egress_proxy_image \ "${image}" "${recipe_fingerprint}" "${artifact_fingerprint}"; then rm -rf -- "${build_dir}" fail "Built ${image}, but its labels, compiled artifacts, or non-root runtime failed attestation." fi fi fi rm -rf -- "${build_dir}" image_id="$(docker_image_id "${image}" || true)" valid_docker_image_id "${image_id}" \ || fail "Project egress proxy image did not resolve to an immutable Docker image ID." verify_project_egress_proxy_image \ "${image_id}" "${recipe_fingerprint}" "${artifact_fingerprint}" \ || fail "The resolved Project egress proxy image ID failed immutable artifact/runtime attestation." record_installer_image_id "${env_file}" "PROJECT_EGRESS_PROXY_IMAGE_ID" "${image_id}" \ || fail "Project egress proxy image did not resolve and persist an immutable Docker image ID." ok "Project egress proxy image ${image_id:0:19}… (pinned)" } attest_project_runtime_image_repair_authority() { local installed_script="$1" installed_env="$2" executing_script="$3" local source_only="${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" python3 - "${installed_script}" "${installed_env}" \ "${executing_script}" "${source_only}" <<'PY' import os import re import stat import sys script_path, env_path, executing_path, source_only = sys.argv[1:] expected_script = "/opt/bridgesllm/portal/installer/install.sh" expected_env = "/opt/bridgesllm/portal/backend/.env.production" if source_only != "1" and (script_path != expected_script or env_path != expected_env): raise SystemExit("Project runtime repair authority is not the fixed installed Portal") if source_only != "1" and not re.fullmatch(r"/proc/(?:self|[0-9]+)/fd/[0-9]+", executing_path): raise SystemExit("Project runtime repair was not launched through its attested script descriptor") def secure_open(path, *, environment=False): if not os.path.isabs(path) or os.path.normpath(path) != path: raise SystemExit("Project runtime repair authority path is not canonical") parts = [part for part in path.split("/") if part] directory_fd = os.open("/", os.O_RDONLY | os.O_DIRECTORY) opened = [directory_fd] try: for part in parts[:-1]: next_fd = os.open( part, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), dir_fd=directory_fd, ) opened.append(next_fd) info = os.fstat(next_fd) if ( not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 ): raise SystemExit("Project runtime repair authority directory is unsafe") directory_fd = next_fd descriptor = os.open( parts[-1], os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NOATIME", 0), dir_fd=directory_fd, ) info = os.fstat(descriptor) if ( not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 or (environment and stat.S_IMODE(info.st_mode) != 0o600) ): os.close(descriptor) raise SystemExit("Project runtime repair authority file is unsafe") return descriptor, info finally: for opened_fd in reversed(opened): os.close(opened_fd) script_fd, script_info = secure_open(script_path) env_fd, _ = secure_open(env_path, environment=True) try: current_fd = os.open( executing_path, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOATIME", 0), ) try: current_info = os.fstat(current_fd) if (current_info.st_dev, current_info.st_ino) != ( script_info.st_dev, script_info.st_ino, ): raise SystemExit("Executing repair script is not the installed Portal script") finally: os.close(current_fd) finally: os.close(script_fd) os.close(env_fd) PY } verify_project_runtime_repair_health() { local service_name="${1:-bridgesllm-product.service}" local health_url="${2:-http://127.0.0.1:4001/health}" local timeout_secs="${3:-60}" waited=0 payload="" while (( waited < timeout_secs )); do if systemctl is-active --quiet "${service_name}"; then payload="$(curl -fsS --max-time 2 "${health_url}" 2>/dev/null || true)" if printf '%s' "${payload}" | python3 /dev/fd/3 "${VERSION}" 3<<'PY' import json import sys expected = sys.argv[1] try: payload = json.load(sys.stdin) except Exception: raise SystemExit(1) if payload.get("status") not in {"ok", "degraded"}: raise SystemExit(1) if payload.get("version") != expected: raise SystemExit(1) PY then return 0 fi fi sleep 2 waited=$((waited + 2)) done return 1 } project_runtime_repair_environment_transaction() { local action="$1" env_file="$2" transaction_dir="$3" local image_id="${4:-}" python3 - "${action}" "${env_file}" "${transaction_dir}" \ "${image_id}" <<'PY' import base64 import hashlib import json import os import re import secrets import stat import sys import time action, env_path, transaction_path, image_id = sys.argv[1:] key = b"PORTAL_PROJECT_RUNTIME_IMAGE_ID" maximum_size = 1024 * 1024 def fail(message): raise SystemExit(message) def stat_record(info): return { "dev": info.st_dev, "ino": info.st_ino, "mode": stat.S_IMODE(info.st_mode), "uid": info.st_uid, "gid": info.st_gid, "nlink": info.st_nlink, "size": info.st_size, "atime_ns": info.st_atime_ns, "mtime_ns": info.st_mtime_ns, "ctime_ns": info.st_ctime_ns, } def identity_matches(info, record): current = stat_record(info) return all(current[name] == record[name] for name in ( "dev", "ino", "mode", "uid", "gid", "nlink", "size", "mtime_ns", "ctime_ns", )) def open_regular(path): descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NOATIME", 0), ) info = os.fstat(descriptor) if ( not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 or info.st_size > maximum_size ): os.close(descriptor) fail("Project runtime repair environment is unsafe") return descriptor, info def read_descriptor(descriptor): chunks = [] total = 0 while True: chunk = os.read(descriptor, min(65536, maximum_size + 1 - total)) if not chunk: break chunks.append(chunk) total += len(chunk) if total > maximum_size: fail("Project runtime repair environment exceeds 1 MiB") return b"".join(chunks) def descriptor_xattrs(descriptor): values = {} for name in os.listxattr(descriptor): values[name] = base64.b64encode(os.getxattr(descriptor, name)).decode("ascii") return values def write_private(path, payload): descriptor = os.open( path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600, ) try: os.fchmod(descriptor, 0o600) view = memoryview(payload) while view: written = os.write(descriptor, view) if written <= 0: fail("Project runtime repair transaction write stalled") view = view[written:] os.fsync(descriptor) except BaseException: try: os.unlink(path) except FileNotFoundError: pass raise finally: os.close(descriptor) def read_private(path, *, limit=maximum_size): descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NOATIME", 0), ) try: info = os.fstat(descriptor) if ( not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or stat.S_IMODE(info.st_mode) != 0o600 or info.st_size > limit ): fail("Project runtime repair transaction artifact is unsafe") return read_descriptor(descriptor) finally: os.close(descriptor) transaction_info = os.lstat(transaction_path) if ( not stat.S_ISDIR(transaction_info.st_mode) or stat.S_ISLNK(transaction_info.st_mode) or transaction_info.st_uid != 0 or transaction_info.st_gid != 0 or stat.S_IMODE(transaction_info.st_mode) != 0o700 ): fail("Project runtime repair transaction directory is unsafe") original_path = os.path.join(transaction_path, "original.env") original_meta_path = os.path.join(transaction_path, "original.json") committed_path = os.path.join(transaction_path, "committed.env") committed_meta_path = os.path.join(transaction_path, "committed.json") transaction_artifact_sets = { "empty": frozenset(), "snapshot": frozenset({"original.env", "original.json"}), "staged": frozenset({"original.env", "original.json", "committed.env"}), "durable": frozenset({ "original.env", "original.json", "committed.env", "committed.json" }), } def fsync_transaction_directory(): directory_fd = os.open( transaction_path, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) def require_transaction_artifact_set(*allowed_states): directory_fd = os.open( transaction_path, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0), ) try: names = os.listdir(directory_fd) if len(names) != len(set(names)): fail("Project runtime repair transaction artifact inventory is ambiguous") observed = frozenset(names) state = next( ( candidate for candidate in allowed_states if observed == transaction_artifact_sets[candidate] ), None, ) if state is None: fail("Project runtime repair transaction has an unexpected artifact set") for name in names: info = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) if ( not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or stat.S_IMODE(info.st_mode) != 0o600 or info.st_size > maximum_size ): fail("Project runtime repair transaction artifact is unsafe") return state finally: os.close(directory_fd) if action not in { "snapshot", "commit", "inspect", "reconcile", "recover-commit", "restore" }: fail("Unsupported Project runtime repair transaction action") if action == "snapshot": require_transaction_artifact_set("empty") descriptor, before = open_regular(env_path) try: payload = read_descriptor(descriptor) after = os.fstat(descriptor) if not identity_matches(after, stat_record(before)): fail("Project runtime repair environment changed during snapshot") metadata = stat_record(before) metadata["sha256"] = hashlib.sha256(payload).hexdigest() metadata["xattrs"] = descriptor_xattrs(descriptor) metadata["environmentPath"] = env_path finally: os.close(descriptor) write_private(original_path, payload) write_private( original_meta_path, (json.dumps(metadata, sort_keys=True, separators=(",", ":")) + "\n").encode(), ) require_transaction_artifact_set("snapshot") fsync_transaction_directory() parent_fd = os.open(os.path.dirname(transaction_path), os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(parent_fd) finally: os.close(parent_fd) raise SystemExit(0) transaction_state = require_transaction_artifact_set( "snapshot", "staged", "durable" ) if action == "commit" and transaction_state != "snapshot": fail("Project runtime repair transaction is not ready for commit") if action == "recover-commit" and transaction_state != "staged": fail("Project runtime repair transaction is not a staged commit") if action == "restore" and transaction_state != "durable": fail("Project runtime repair transaction has no durable commit receipt") try: original = read_private(original_path) metadata = json.loads(read_private(original_meta_path)) except Exception: fail("Project runtime repair snapshot is incomplete") if hashlib.sha256(original).hexdigest() != metadata.get("sha256"): fail("Project runtime repair snapshot checksum is invalid") if metadata.get("environmentPath") != env_path: fail("Project runtime repair snapshot belongs to another environment") def install_payload(payload, restore_times): parent = os.path.dirname(env_path) leaf = os.path.basename(env_path) parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)) temporary = f".{leaf}.runtime-repair-{secrets.token_hex(12)}" descriptor = -1 try: descriptor = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600, dir_fd=parent_fd, ) os.fchown(descriptor, metadata["uid"], metadata["gid"]) os.fchmod(descriptor, metadata["mode"]) for name, encoded in metadata.get("xattrs", {}).items(): os.setxattr(descriptor, name, base64.b64decode(encoded)) view = memoryview(payload) while view: written = os.write(descriptor, view) if written <= 0: fail("Project runtime repair environment write stalled") view = view[written:] if restore_times: os.utime( descriptor, ns=(metadata["atime_ns"], metadata["mtime_ns"]), ) os.fsync(descriptor) os.close(descriptor) descriptor = -1 os.replace(temporary, leaf, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) os.fsync(parent_fd) finally: if descriptor >= 0: os.close(descriptor) try: os.unlink(temporary, dir_fd=parent_fd) except FileNotFoundError: pass os.close(parent_fd) if action == "commit": if not re.fullmatch(r"sha256:[a-f0-9]{64}", image_id): fail("Project runtime repair image identity is invalid") descriptor, current_info = open_regular(env_path) try: current = read_descriptor(descriptor) finally: os.close(descriptor) if not identity_matches(current_info, metadata) or current != original: fail("Project runtime repair environment changed before commit") lines = original.splitlines(keepends=True) matches = [] for index, line in enumerate(lines): body = line.rstrip(b"\r\n") if body.startswith(key + b"="): matches.append(index) if len(matches) > 1: fail("Project runtime repair environment contains duplicate image keys") replacement = key + b"=" + image_id.encode("ascii") if matches: index = matches[0] ending = lines[index][len(lines[index].rstrip(b"\r\n")):] if lines[index].rstrip(b"\r\n") == replacement: print("unchanged") raise SystemExit(0) lines[index] = replacement + ending committed = b"".join(lines) else: separator = b"" if not original or original.endswith((b"\n", b"\r")) else b"\n" committed = original + separator + replacement + b"\n" write_private(committed_path, committed) # The staged payload must be durably discoverable before the installed # environment can expose it. A crash after replacement can then always be # classified as a recoverable pre-receipt transaction. fsync_transaction_directory() require_transaction_artifact_set("staged") install_payload(committed, False) test_hook = os.environ.get("BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_HOOK") if ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and test_hook == "pause-after-replace-before-receipt" ): marker = os.environ.get("BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_MARKER", "") if not marker or not os.path.isabs(marker): fail("Project runtime repair signal-test marker is invalid") write_private(marker, b"replaced\n") time.sleep(300) if ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and test_hook == "commit-after-replace" ): fail("Injected Project runtime repair failure after environment replacement") descriptor, committed_info = open_regular(env_path) try: installed = read_descriptor(descriptor) finally: os.close(descriptor) if installed != committed: # Best-effort immediate restoration before reporting a failed commit. install_payload(original, True) fail("Project runtime repair environment commit did not verify") committed_metadata = stat_record(committed_info) committed_metadata["sha256"] = hashlib.sha256(committed).hexdigest() write_private( committed_meta_path, (json.dumps(committed_metadata, sort_keys=True, separators=(",", ":")) + "\n").encode(), ) require_transaction_artifact_set("durable") fsync_transaction_directory() print("changed") raise SystemExit(0) def original_visible(info, payload, xattrs): return ( payload == original and stat.S_IMODE(info.st_mode) == metadata["mode"] and info.st_uid == metadata["uid"] and info.st_gid == metadata["gid"] and info.st_nlink == metadata["nlink"] and info.st_atime_ns == metadata["atime_ns"] and info.st_mtime_ns == metadata["mtime_ns"] and xattrs == metadata.get("xattrs", {}) ) def read_current(): descriptor, info = open_regular(env_path) try: payload = read_descriptor(descriptor) xattrs = descriptor_xattrs(descriptor) finally: os.close(descriptor) return info, payload, xattrs def verify_restored_original(message): info, payload, xattrs = read_current() if not original_visible(info, payload, xattrs): fail(message) if action in {"inspect", "reconcile"}: committed_exists = transaction_state in {"staged", "durable"} receipt_exists = transaction_state == "durable" current_info, current, current_xattrs = read_current() if not committed_exists: if not original_visible(current_info, current, current_xattrs): fail("Project runtime repair snapshot no longer matches the installed environment") print("snapshot") raise SystemExit(0) try: committed = read_private(committed_path) except Exception: fail("Project runtime repair staged commit is incomplete") if not receipt_exists: if current == committed: if action == "reconcile": install_payload(original, True) elif not original_visible(current_info, current, current_xattrs): fail("Project runtime repair environment is neither the original nor staged commit") if action == "reconcile": verify_restored_original( "Project runtime repair pre-receipt recovery did not verify" ) print("staged") raise SystemExit(0) try: committed_metadata = json.loads(read_private(committed_meta_path)) except Exception: fail("Project runtime repair commit receipt is incomplete") if hashlib.sha256(committed).hexdigest() != committed_metadata.get("sha256"): fail("Project runtime repair committed payload checksum is invalid") if current == committed: if not identity_matches(current_info, committed_metadata): fail("Project runtime repair committed environment changed after receipt") if action == "reconcile": install_payload(original, True) elif not original_visible(current_info, current, current_xattrs): fail("Project runtime repair durable commit no longer matches the installed environment") if action == "reconcile": verify_restored_original( "Project runtime repair durable-commit recovery did not verify" ) print("durable") raise SystemExit(0) if action == "recover-commit": try: committed = read_private(committed_path) except Exception: fail("Project runtime repair staged commit is incomplete") descriptor, current_info = open_regular(env_path) try: current = read_descriptor(descriptor) current_xattrs = descriptor_xattrs(descriptor) finally: os.close(descriptor) if current == committed: install_payload(original, True) elif current == original: if ( stat.S_IMODE(current_info.st_mode) != metadata["mode"] or current_info.st_uid != metadata["uid"] or current_info.st_gid != metadata["gid"] or current_info.st_atime_ns != metadata["atime_ns"] or current_info.st_mtime_ns != metadata["mtime_ns"] or current_xattrs != metadata.get("xattrs", {}) ): fail("Project runtime repair original environment metadata changed during failed commit") else: fail("Project runtime repair environment is neither the original nor staged commit") descriptor, restored_info = open_regular(env_path) try: restored = read_descriptor(descriptor) restored_xattrs = descriptor_xattrs(descriptor) finally: os.close(descriptor) if ( restored != original or stat.S_IMODE(restored_info.st_mode) != metadata["mode"] or restored_info.st_uid != metadata["uid"] or restored_info.st_gid != metadata["gid"] or restored_info.st_atime_ns != metadata["atime_ns"] or restored_info.st_mtime_ns != metadata["mtime_ns"] or restored_xattrs != metadata.get("xattrs", {}) ): fail("Project runtime repair failed-commit recovery did not verify") raise SystemExit(0) if action == "restore": try: committed = read_private(committed_path) committed_metadata = json.loads(read_private(committed_meta_path)) except Exception: fail("Project runtime repair commit receipt is incomplete") descriptor, current_info = open_regular(env_path) try: current = read_descriptor(descriptor) finally: os.close(descriptor) if ( not identity_matches(current_info, committed_metadata) or current != committed or hashlib.sha256(current).hexdigest() != committed_metadata.get("sha256") ): fail("Project runtime repair environment changed after commit; refusing to overwrite it") install_payload(original, True) descriptor, restored_info = open_regular(env_path) try: restored = read_descriptor(descriptor) restored_xattrs = descriptor_xattrs(descriptor) finally: os.close(descriptor) if ( restored != original or stat.S_IMODE(restored_info.st_mode) != metadata["mode"] or restored_info.st_uid != metadata["uid"] or restored_info.st_gid != metadata["gid"] or restored_info.st_atime_ns != metadata["atime_ns"] or restored_info.st_mtime_ns != metadata["mtime_ns"] or restored_xattrs != metadata.get("xattrs", {}) ): fail("Project runtime repair environment rollback did not verify") raise SystemExit(0) fail("Unsupported Project runtime repair transaction action") PY } write_portal_project_runtime_dockerfile() { local target="$1" cat > "${target}" <<'PROJECT_RUNTIME_DOCKERFILE' FROM node:22.16.0-bookworm-slim ARG PORTAL_RECIPE_SHA256 LABEL com.bridgesllm.portal-project-runtime.recipe-sha256="${PORTAL_RECIPE_SHA256}" ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ build-essential \ ca-certificates \ git \ python3 \ python3-pip \ python3-venv \ && rm -rf /var/lib/apt/lists/* USER 1000:1000 WORKDIR /workspace/project # The sandbox attestation requires a bare image: the runtime container's # command is pinned at create time and the base image's docker-entrypoint.sh # must not run. ENTRYPOINT [] CMD ["node", "--version"] PROJECT_RUNTIME_DOCKERFILE } verify_portal_project_runtime_image() { local image="$1" expected_recipe="$2" existing_entrypoint="" [[ "${expected_recipe}" =~ ^[a-f0-9]{64}$ ]] || return 1 valid_docker_image_id "$(docker_image_id "${image}" || true)" || return 1 [[ "$(docker_image_label "${image}" "${PORTAL_PROJECT_RUNTIME_RECIPE_LABEL}" || true)" \ == "${expected_recipe}" ]] || return 1 existing_entrypoint="$( docker image inspect --format '{{len .Config.Entrypoint}}' \ "${image}" 2>/dev/null || echo 1 )" [[ "${existing_entrypoint}" == "0" ]] || return 1 docker run --rm --entrypoint sh "${image}" -lc \ 'test "$(id -u)" = 1000 && test "$(id -g)" = 1000 && node --version >/dev/null 2>&1 && npm --version >/dev/null 2>&1 && python3 --version >/dev/null 2>&1 && python3 -m venv --help >/dev/null 2>&1 && command -v g++ >/dev/null 2>&1 && command -v make >/dev/null 2>&1' \ >> "${LOG_FILE}" 2>&1 } ensure_portal_project_runtime_image() { local image="${1:-${PORTAL_PROJECT_RUNTIME_IMAGE_TAG}}" local prepared_env_file="${2:-}" local may_start_docker="${3:-true}" if ! command -v docker &>/dev/null; then fail "Docker is required for isolated project installs, builds, and full-stack apps." fi valid_local_image_tag "${image}" \ || fail "Portal project runtime image tag is invalid." if [[ -n "${prepared_env_file}" ]]; then [[ -f "${prepared_env_file}" && ! -L "${prepared_env_file}" ]] \ || fail "Prepared Project runtime environment is unavailable." fi local build_dir recipe_fingerprint build_dir="$(mktemp -d)" write_portal_project_runtime_dockerfile "${build_dir}/Dockerfile" \ || { rm -rf -- "${build_dir}"; fail "Canonical Project runtime recipe could not be staged."; } recipe_fingerprint="$(sha256sum "${build_dir}/Dockerfile" | awk '{print $1}')" [[ "${recipe_fingerprint}" =~ ^[a-f0-9]{64}$ ]] \ || { rm -rf -- "${build_dir}"; fail "Canonical Project runtime recipe fingerprint is invalid."; } PORTAL_PROJECT_RUNTIME_RECIPE_SHA256="${recipe_fingerprint}" if [[ "${may_start_docker}" == "true" ]]; then systemctl start docker >> "$LOG_FILE" 2>&1 || true elif [[ "${may_start_docker}" == "false" ]]; then systemctl is-active --quiet docker \ || fail "Docker must already be active before Project runtime image repair." else fail "Project runtime Docker-management mode is invalid." fi local image_ready=false if docker image inspect "$image" >/dev/null 2>&1; then if verify_portal_project_runtime_image "${image}" "${recipe_fingerprint}"; then ok "Portal project runtime image" image_ready=true else warn "Existing Portal project runtime image is incomplete; rebuilding." fi fi if ! ${image_ready}; then if ! spin "Building isolated Portal project runtime" \ "docker build --tag '${image}' --build-arg 'PORTAL_RECIPE_SHA256=${recipe_fingerprint}' '${build_dir}'"; then rm -rf "$build_dir" fail "Failed to build ${image}. Project code execution remains disabled." fi fi if ! verify_portal_project_runtime_image "${image}" "${recipe_fingerprint}"; then rm -rf -- "${build_dir}" fail "Built ${image}, but its required project tools are unavailable." fi rm -rf -- "${build_dir}" local image_id image_id="$(docker_image_id "${image}" || true)" valid_docker_image_id "${image_id}" \ || fail "Portal project runtime image did not resolve to an immutable Docker image ID." if [[ -n "${prepared_env_file}" ]]; then set_env_value_atomic \ "${prepared_env_file}" "PORTAL_PROJECT_RUNTIME_IMAGE_ID" "${image_id}" \ || fail "Portal project runtime image ID could not be recorded for update cutover." fi ok "Portal project runtime image" } project_runtime_repair_transaction_root() { local root="/run" if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && -n "${BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_ROOT:-}" ]]; then root="${BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_ROOT}" fi python3 - "${root}" <<'PY' import os import stat import sys root = sys.argv[1] if not os.path.isabs(root) or os.path.normpath(root) != root: raise SystemExit("Project runtime repair transaction root is not canonical") descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)) try: info = os.fstat(descriptor) if ( not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 ): raise SystemExit("Project runtime repair transaction root is unsafe") finally: os.close(descriptor) print(root) PY } cleanup_project_runtime_image_repair_transaction() { local directory="${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR:-}" local root="${PROJECT_RUNTIME_REPAIR_TRANSACTION_ROOT:-}" [[ -n "${directory}" ]] || return 0 [[ -n "${root}" \ && "$(dirname -- "${directory}")" == "${root}" \ && "$(basename -- "${directory}")" \ =~ ^bridgesllm-project-runtime-image-repair\.[A-Za-z0-9]{8}$ \ && -d "${directory}" && ! -L "${directory}" ]] || return 1 rm -rf -- "${directory}" || return 1 [[ ! -e "${directory}" && ! -L "${directory}" ]] || return 1 python3 - "${root}" <<'PY' || return 1 import os import sys descriptor = os.open(sys.argv[1], os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)) try: os.fsync(descriptor) finally: os.close(descriptor) PY PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR="" } settle_project_runtime_image_repair_child() { local pid="${PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID:-}" local current child index local -a pending=() descendants=() [[ -n "${pid}" ]] || return 0 [[ "${pid}" =~ ^[1-9][0-9]*$ ]] || return 1 pending=("${pid}") index=0 while (( index < ${#pending[@]} )); do current="${pending[index]}" index=$((index + 1)) if [[ -r "/proc/${current}/task/${current}/children" ]]; then for child in $(<"/proc/${current}/task/${current}/children"); do [[ "${child}" =~ ^[1-9][0-9]*$ ]] || continue descendants+=("${child}") pending+=("${child}") done fi done if (( ${#descendants[@]} > 0 )); then kill -TERM "${descendants[@]}" >/dev/null 2>&1 || true fi kill -TERM "${pid}" >/dev/null 2>&1 || true for _ in {1..50}; do kill -0 "${pid}" >/dev/null 2>&1 || break sleep 0.02 done if kill -0 "${pid}" >/dev/null 2>&1; then if (( ${#descendants[@]} > 0 )); then kill -KILL "${descendants[@]}" >/dev/null 2>&1 || true fi kill -KILL "${pid}" >/dev/null 2>&1 || true fi wait "${pid}" >/dev/null 2>&1 || true PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID="" PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_KIND="" } run_project_runtime_repair_environment_commit() { local env_file="$1" transaction_dir="$2" image_id="$3" status=0 ( trap - EXIT ERR SIGINT SIGTERM SIGHUP project_runtime_repair_environment_transaction \ commit "${env_file}" "${transaction_dir}" "${image_id}" ) >> "${PROJECT_RUNTIME_REPAIR_ACTIVE_LOG_FILE:-${LOG_FILE}}" 2>&1 & PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID=$! PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_KIND="environment-commit" if wait "${PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID}"; then status=0 else status=$? fi PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID="" PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_KIND="" return "${status}" } run_project_runtime_repair_portal_restart() { local status=0 ( trap - EXIT ERR SIGINT SIGTERM SIGHUP systemctl restart bridgesllm-product.service ) >> "${PROJECT_RUNTIME_REPAIR_ACTIVE_LOG_FILE:-${LOG_FILE}}" 2>&1 & PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID=$! PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_KIND="portal-restart" if wait "${PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID}"; then status=0 else status=$? fi PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID="" PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_KIND="" return "${status}" } rollback_project_runtime_image_repair() { local outcome="" [[ -n "${PROJECT_RUNTIME_REPAIR_ENV_FILE:-}" \ && -n "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR:-}" ]] || return 1 outcome="$( project_runtime_repair_environment_transaction \ reconcile \ "${PROJECT_RUNTIME_REPAIR_ENV_FILE}" \ "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR}" )" || return 1 case "${outcome}" in snapshot|staged) return 0 ;; durable) systemctl restart bridgesllm-product.service \ >> "${PROJECT_RUNTIME_REPAIR_ACTIVE_LOG_FILE:-${LOG_FILE}}" 2>&1 \ || return 1 verify_project_runtime_repair_health \ bridgesllm-product.service http://127.0.0.1:4001/health 60 \ || return 1 return 0 ;; *) return 1 ;; esac } recover_pending_project_runtime_image_repair() { local installed_env="$1" root="$2" pending="" outcome="" local -a candidates=() shopt -s nullglob candidates=("${root}"/bridgesllm-project-runtime-image-repair.*) shopt -u nullglob (( ${#candidates[@]} <= 1 )) \ || fail "Multiple interrupted Project runtime image repairs exist; refusing to guess recovery order." (( ${#candidates[@]} == 1 )) || return 0 pending="${candidates[0]}" [[ -d "${pending}" && ! -L "${pending}" \ && "$(basename -- "${pending}")" \ =~ ^bridgesllm-project-runtime-image-repair\.[A-Za-z0-9]{8}$ ]] \ || fail "An interrupted Project runtime image repair transaction is unsafe." PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR="${pending}" PROJECT_RUNTIME_REPAIR_TRANSACTION_ROOT="${root}" PROJECT_RUNTIME_REPAIR_ENV_FILE="${installed_env}" outcome="$( project_runtime_repair_environment_transaction \ inspect "${installed_env}" "${pending}" )" || fail "An interrupted Project runtime image repair could not be inspected safely. Its transaction was preserved." rollback_project_runtime_image_repair \ || fail "An interrupted Project runtime image repair could not be reconciled safely. Its transaction was preserved." cleanup_project_runtime_image_repair_transaction \ || fail "A reconciled Project runtime image repair transaction could not be removed." info "Recovered interrupted Project runtime image repair (${outcome})." } abort_project_runtime_image_repair_after_commit() { local message="$1" trap '' SIGINT TERM HUP trap - ERR set +e if rollback_project_runtime_image_repair; then if cleanup_project_runtime_image_repair_transaction; then fail "${message} The exact previous environment was restored and Portal is healthy on it." fi trap - EXIT echo -e " ${RED}${BOLD}Project runtime image repair rollback verified, but its transaction cleanup did not.${NC}" >&2 echo -e " ${RED}The root-only transaction was preserved for the next locked repair.${NC}" >&2 exit 1 fi trap - EXIT echo "" >&2 echo -e " ${RED}${BOLD}Project runtime image repair failed after environment commit.${NC}" >&2 echo -e " ${RED}${message}${NC}" >&2 echo -e " ${RED}Automatic environment/service recovery did not verify. The root-only repair log and transaction state require manual inspection.${NC}" >&2 exit 1 } handle_project_runtime_image_repair_exit() { local exit_code="${1:-1}" trap - EXIT ERR SIGINT SIGTERM SIGHUP set +e settle_project_runtime_image_repair_child >/dev/null 2>&1 || true if [[ -n "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR:-}" ]]; then if rollback_project_runtime_image_repair; then if ! cleanup_project_runtime_image_repair_transaction >/dev/null 2>&1; then echo -e " ${RED}${BOLD}Project runtime image repair transaction cleanup did not verify; its root-only state was preserved.${NC}" >&2 exit_code=1 fi else echo -e " ${RED}${BOLD}Project runtime image repair exit recovery did not verify; its root-only transaction was preserved.${NC}" >&2 exit_code=1 fi fi [[ "${exit_code}" =~ ^[0-9]+$ ]] || exit_code=1 (( exit_code != 0 )) || exit_code=1 exit "${exit_code}" } handle_project_runtime_image_repair_signal() { local exit_code="$1" label="$2" trap '' SIGINT TERM HUP trap - ERR set +e settle_project_runtime_image_repair_child >/dev/null 2>&1 || true if [[ -n "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR:-}" ]] \ && ! rollback_project_runtime_image_repair; then trap - EXIT echo -e " ${RED}${BOLD}${label}; automatic repair rollback did not verify and the root-only transaction was preserved.${NC}" >&2 exit "${exit_code}" fi if ! cleanup_project_runtime_image_repair_transaction >/dev/null 2>&1; then trap - EXIT echo -e " ${RED}${BOLD}${label}; rollback verified but transaction cleanup did not. Root-only recovery state was preserved.${NC}" >&2 exit "${exit_code}" fi echo -e " ${RED}${label}.${NC}" >&2 exit "${exit_code}" } handle_project_runtime_image_repair_err() { local exit_code="${1:-1}" line="${2:-unknown}" [[ "${exit_code}" =~ ^[1-9][0-9]*$ ]] || exit_code=1 handle_project_runtime_image_repair_signal \ "${exit_code}" "Unexpected Project runtime image repair error at line ${line}" } repair_project_runtime_image() { local installed_script="${PORTAL_DIR}/installer/install.sh" local installed_env="${PORTAL_DIR}/backend/.env.production" local source_only="${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" local repair_log_file="${LOG_FILE}" repair_log_dir="${LOG_DIR}" local image_id="" commit_state="" transaction_root="" if (( $# > 0 )); then [[ "${source_only}" == "1" && $# -eq 3 ]] \ || fail "Project runtime repair authority paths cannot be overridden." installed_script="$1" installed_env="$2" repair_log_file="$3" repair_log_dir="$(dirname -- "${repair_log_file}")" fi CURRENT_STEP="Project runtime image repair" attest_project_runtime_image_repair_authority \ "${installed_script}" "${installed_env}" "${BASH_SOURCE[0]}" \ || fail "The installed repair script or production environment failed root authority attestation." mkdir -p "${repair_log_dir}" touch "${repair_log_file}" chmod 0600 "${repair_log_file}" PROJECT_RUNTIME_REPAIR_ACTIVE_LOG_FILE="${repair_log_file}" transaction_root="$(project_runtime_repair_transaction_root)" \ || fail "The Project runtime repair transaction root failed authority attestation." PROJECT_RUNTIME_REPAIR_TRANSACTION_ROOT="${transaction_root}" PROJECT_RUNTIME_REPAIR_ENV_FILE="${installed_env}" PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR="" PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_PID="" PROJECT_RUNTIME_REPAIR_ACTIVE_CHILD_KIND="" trap 'handle_project_runtime_image_repair_exit $?' EXIT trap 'handle_project_runtime_image_repair_err $? $LINENO' ERR trap 'handle_project_runtime_image_repair_signal 130 "Project runtime image repair was interrupted"' SIGINT trap 'handle_project_runtime_image_repair_signal 143 "Project runtime image repair was terminated"' SIGTERM trap 'handle_project_runtime_image_repair_signal 129 "Project runtime image repair session disconnected"' SIGHUP # Holding the shared installer lock makes this the sole repair authority. # Reconcile a hard-killed predecessor before image inspection or any # idempotent no-restart decision can observe its staged environment. recover_pending_project_runtime_image_repair \ "${installed_env}" "${transaction_root}" converge_unsafe_docker_prune_automation \ || fail "Unsafe scheduled Docker cleanup remains active. Repair it before rebuilding a pinned Project runtime image." verify_project_runtime_repair_health \ bridgesllm-product.service http://127.0.0.1:4001/health 10 \ || fail "Portal must be healthy before Project runtime image repair begins." PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR="$( mktemp -d "${transaction_root}/bridgesllm-project-runtime-image-repair.XXXXXXXX" )" || fail "A private Project runtime repair transaction directory could not be created." chmod 0700 "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR}" \ || fail "The Project runtime repair transaction could not be made private." project_runtime_repair_environment_transaction \ snapshot "${installed_env}" "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR}" \ || fail "The exact installed Portal environment could not be snapshotted before repair." parse_strict_systemd_environment_file \ "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR}/original.env" \ || fail "The installed Portal environment is not a strict systemd environment file." # Docker must already be active. This mode never starts or restarts Docker, # optional provider tools, Caddy, OpenClaw, or any other managed service. ensure_portal_project_runtime_image \ "${PORTAL_PROJECT_RUNTIME_IMAGE_TAG}" "" false image_id="$(docker_image_id "${PORTAL_PROJECT_RUNTIME_IMAGE_TAG}" || true)" valid_docker_image_id "${image_id}" \ || fail "The repaired Project runtime tag did not resolve to an immutable image ID." [[ "$(docker_image_id "${image_id}" || true)" == "${image_id}" ]] \ || fail "The repaired Project runtime image ID did not resolve exactly." [[ "${PORTAL_PROJECT_RUNTIME_RECIPE_SHA256}" =~ ^[a-f0-9]{64}$ ]] \ || fail "The canonical Project runtime recipe fingerprint was not retained." verify_portal_project_runtime_image \ "${image_id}" "${PORTAL_PROJECT_RUNTIME_RECIPE_SHA256}" \ || fail "The repaired immutable Project runtime image failed runtime attestation." if ! run_project_runtime_repair_environment_commit \ "${installed_env}" "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR}" \ "${image_id}"; then if rollback_project_runtime_image_repair; then if cleanup_project_runtime_image_repair_transaction; then fail "The Project runtime image ID commit failed; the exact previous environment was restored before any service restart." fi trap - EXIT echo -e " ${RED}${BOLD}The failed Project runtime image commit was rolled back, but transaction cleanup did not verify.${NC}" >&2 echo -e " ${RED}The root-only transaction was preserved for the next locked repair; Portal was not restarted.${NC}" >&2 exit 1 fi trap - EXIT echo -e " ${RED}${BOLD}Project runtime image environment commit failed and recovery could not be verified.${NC}" >&2 echo -e " ${RED}The root-only repair transaction was preserved for automatic recovery by the next locked repair; Portal was not restarted.${NC}" >&2 exit 1 fi commit_state="$( project_runtime_repair_environment_transaction \ inspect "${installed_env}" "${PROJECT_RUNTIME_REPAIR_TRANSACTION_DIR}" )" || abort_project_runtime_image_repair_after_commit \ "The durable Project runtime image environment state could not be verified." case "${commit_state}" in snapshot) verify_project_runtime_repair_health \ bridgesllm-product.service http://127.0.0.1:4001/health 10 \ || fail "Portal became unhealthy during an idempotent Project runtime image repair." cleanup_project_runtime_image_repair_transaction \ || fail "The completed Project runtime repair transaction could not be removed." trap - EXIT ok "Project runtime image was already repaired ${DIM}(${image_id:0:19}…)${NC}" return 0 ;; durable) ;; *) abort_project_runtime_image_repair_after_commit \ "Project runtime image repair returned an invalid durable environment state." ;; esac if [[ "${source_only}" == "1" \ && "${BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_HOOK:-}" \ == "pause-after-receipt-before-restart" ]]; then [[ -n "${BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_MARKER:-}" ]] \ || abort_project_runtime_image_repair_after_commit \ "The Project runtime repair signal-test marker is invalid." printf '%s\n' 'receipt' \ > "${BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_MARKER}" chmod 0600 "${BRIDGESLLM_PROJECT_RUNTIME_REPAIR_TEST_MARKER}" while :; do sleep 0.05; done fi if ! run_project_runtime_repair_portal_restart; then abort_project_runtime_image_repair_after_commit \ "Portal could not restart with the repaired Project runtime image." fi if ! verify_project_runtime_repair_health \ bridgesllm-product.service http://127.0.0.1:4001/health 60; then abort_project_runtime_image_repair_after_commit \ "Portal did not become healthy with the repaired Project runtime image." fi cleanup_project_runtime_image_repair_transaction \ || fail "The completed Project runtime repair transaction could not be removed." trap - EXIT ok "Project runtime image repaired ${DIM}(${image_id:0:19}…)${NC}" } update_project_runtime_prepared_env_path() { local stage_dir="$1" [[ -d "${stage_dir}" && ! -L "${stage_dir}" ]] || return 1 printf '%s/project-runtimes.prepared.env\n' "${stage_dir}" } update_project_runtime_preparation_mode_path() { local stage_dir="$1" [[ -d "${stage_dir}" && ! -L "${stage_dir}" ]] || return 1 printf '%s/project-runtimes.preparation-mode-v1\n' "${stage_dir}" } write_update_project_runtime_preparation_mode() { local stage_dir="$1" transaction_id="$2" mode="$3" marker [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ \ && ( "${mode}" == "disabled" || "${mode}" == "docker" ) ]] \ || return 1 marker="$(update_project_runtime_preparation_mode_path "${stage_dir}")" \ || return 1 python3 - "${stage_dir}" "${marker}" "${transaction_id}" "${mode}" <<'PY' import os import secrets import stat import sys directory, path, transaction_id, mode = sys.argv[1:] details = os.lstat(directory) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or stat.S_IMODE(details.st_mode) != 0o700 or os.path.dirname(path) != directory ): raise SystemExit(1) payload = f"v1:{transaction_id}:{mode}\n".encode("ascii") if os.path.lexists(path): raise SystemExit(1) temporary = os.path.join( directory, f".project-runtime-preparation-mode.{os.getpid()}.{secrets.token_hex(8)}", ) fd = -1 try: fd = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) os.write(fd, payload) os.fchmod(fd, 0o600) os.fsync(fd) info = os.fstat(fd) if ( not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_gid != os.getegid() or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_size != len(payload) ): raise SystemExit(1) os.close(fd) fd = -1 if os.path.lexists(path): raise SystemExit(1) os.rename(temporary, path) finally: if fd >= 0: os.close(fd) try: os.unlink(temporary) except FileNotFoundError: pass directory_fd = os.open( directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY } read_update_project_runtime_preparation_mode() { local stage_dir="$1" transaction_id="$2" marker [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 marker="$(update_project_runtime_preparation_mode_path "${stage_dir}")" \ || return 1 python3 - "${marker}" "${transaction_id}" <<'PY' import os import stat import sys path, transaction_id = sys.argv[1:] flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) fd = os.open(path, flags) try: details = os.fstat(fd) if ( not stat.S_ISREG(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or stat.S_IMODE(details.st_mode) != 0o600 or details.st_nlink != 1 or not 1 <= details.st_size <= 96 ): raise SystemExit(1) chunks = [] remaining = 97 while remaining: chunk = os.read(fd, remaining) if not chunk: break chunks.append(chunk) remaining -= len(chunk) raw = b"".join(chunks) after = os.fstat(fd) if ( (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) != (details.st_dev, details.st_ino, details.st_size, details.st_mtime_ns) or len(raw) != details.st_size ): raise SystemExit(1) finally: os.close(fd) for mode in ("disabled", "docker"): if raw == f"v1:{transaction_id}:{mode}\n".encode("ascii"): print(mode) raise SystemExit(0) raise SystemExit(1) PY } update_project_runtime_staging_tag() { local transaction_id="$1" component="$2" [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 case "${component}" in egress|codex|claude|antigravity|ollama|agent-zero|openclaw|portal) ;; *) return 1 ;; esac printf 'bridgesllm-update-%s-%s:candidate\n' \ "${transaction_id}" "${component}" } validate_prepared_update_project_runtime_environment() { local prepared_env="$1" expected_transaction_id="$2" local expected_owner="$(id -u):$(id -g):600:1" [[ "${expected_transaction_id}" =~ ^[a-f0-9]{32}$ \ && -f "${prepared_env}" && ! -L "${prepared_env}" \ && "$(stat -c '%u:%g:%a:%h' "${prepared_env}")" == "${expected_owner}" ]] \ || return 1 assert_env_file_no_duplicate_keys "${prepared_env}" || return 1 assert_prisma_runtime_environment_safe "${prepared_env}" || return 1 [[ "$(read_env_value \ "${prepared_env}" BRIDGESLLM_UPDATE_PROJECT_RUNTIMES_PREPARED || true)" \ == "1" \ && "$(read_env_value \ "${prepared_env}" BRIDGESLLM_UPDATE_PROJECT_RUNTIME_TRANSACTION_ID || true)" \ == "${expected_transaction_id}" ]] || return 1 local policy key value policy="$(read_env_value \ "${prepared_env}" PROJECT_RUNTIME_CONFINEMENT_POLICY || true)" case "${policy}" in "${PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY}"|\ "${PROJECT_RUNTIME_SECCOMP_ONLY_POLICY}") for key in \ PROJECT_EGRESS_PROXY_IMAGE_ID \ CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID \ OLLAMA_PROJECT_SANDBOX_IMAGE_ID \ AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID \ PORTAL_PROJECT_RUNTIME_IMAGE_ID; do value="$(read_env_value "${prepared_env}" "${key}" || true)" valid_docker_image_id "${value}" || return 1 done for key in \ CODEX_PROJECT_SANDBOX_IMAGE_ID \ ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID \ OPENCLAW_PROJECT_SANDBOX_IMAGE_ID; do value="$(read_env_value "${prepared_env}" "${key}" || true)" [[ -z "${value}" ]] || valid_docker_image_id "${value}" || return 1 done ;; "${PROJECT_RUNTIME_DISABLED_POLICY}") ;; *) return 1 ;; esac } prepare_update_project_runtimes() { local staged_portal="$1" source_env="$2" transaction_id="$3" local stage_dir prepared_env preparation_mode marker [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ \ && -d "${staged_portal}" && ! -L "${staged_portal}" \ && -f "${source_env}" && ! -L "${source_env}" ]] \ || fail "Update Project runtime preparation received an unsafe input." stage_dir="$(dirname "${staged_portal}")" prepared_env="$(update_project_runtime_prepared_env_path "${stage_dir}")" \ || fail "Could not resolve the prepared Project runtime environment." marker="$(update_project_runtime_preparation_mode_path "${stage_dir}")" \ || fail "Could not resolve the Project runtime preparation marker." [[ ! -e "${prepared_env}" && ! -L "${prepared_env}" \ && ! -e "${marker}" && ! -L "${marker}" ]] \ || fail "Prepared Project runtime environment already exists." preparation_mode="docker" ${SKIP_PROJECT_RUNTIMES} && preparation_mode="disabled" write_update_project_runtime_preparation_mode \ "${stage_dir}" "${transaction_id}" "${preparation_mode}" \ || fail "Could not durably record the Project runtime preparation mode." install -m 0600 "${source_env}" "${prepared_env}" \ || fail "Could not create the private prepared Project runtime environment." [[ "$(stat -c '%u:%g:%a:%h' "${prepared_env}")" \ == "$(id -u):$(id -g):600:1" ]] \ || fail "Prepared Project runtime environment has unsafe metadata." # This is the durable "preparation started" boundary. No Docker/tag command # may move above it: after a crash, absence of this inode proves that no # transaction-scoped image tag could have been created. sync -f "${prepared_env}" \ && sync -f "${stage_dir}" \ || fail "Could not durably record the Project runtime preparation boundary." local key if ${SKIP_PROJECT_RUNTIMES}; then set_env_value_atomic \ "${prepared_env}" PROJECT_RUNTIME_CONFINEMENT_POLICY \ "${PROJECT_RUNTIME_DISABLED_POLICY}" \ || fail "Could not stage the disabled Project runtime policy." for key in \ PROJECT_EGRESS_PROXY_IMAGE_ID \ CODEX_PROJECT_SANDBOX_IMAGE_ID \ CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID \ ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID \ OLLAMA_PROJECT_SANDBOX_IMAGE_ID \ AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID \ OPENCLAW_PROJECT_SANDBOX_IMAGE_ID \ PORTAL_PROJECT_RUNTIME_IMAGE_ID; do set_env_value_atomic "${prepared_env}" "${key}" "" \ || fail "Could not clear disabled Project runtime identity ${key}." done else # This boundary is deliberately additive. Candidate images receive # transaction-specific tags and no prune runs before cutover, so a failed # update can restore its old environment while every prior immutable image # ID and canonical tag remain runnable. Changed confinement policy must use # a new versioned path/name; an in-place replacement fails before downtime. ensure_project_runtime_confinement_profiles \ "${prepared_env}" "${staged_portal}" additive ensure_project_egress_proxy_image \ "${prepared_env}" "${staged_portal}/backend/dist/services" \ "$(update_project_runtime_staging_tag "${transaction_id}" egress)" ensure_codex_project_sandbox_image \ "${prepared_env}" \ "$(update_project_runtime_staging_tag "${transaction_id}" codex)" ensure_claude_code_project_sandbox_image \ "${prepared_env}" \ "$(update_project_runtime_staging_tag "${transaction_id}" claude)" ensure_antigravity_project_sandbox_image \ "${prepared_env}" \ "$(update_project_runtime_staging_tag "${transaction_id}" antigravity)" ensure_ollama_project_sandbox_image \ "${prepared_env}" \ "$(update_project_runtime_staging_tag "${transaction_id}" ollama)" ensure_agent_zero_project_sandbox_image \ "${prepared_env}" \ "$(update_project_runtime_staging_tag "${transaction_id}" agent-zero)" \ "${staged_portal}/installer/agent-zero-project-sandbox.Dockerfile" ensure_openclaw_sandbox_image \ "${prepared_env}" \ "$(update_project_runtime_staging_tag "${transaction_id}" openclaw)" ensure_portal_project_runtime_image \ "$(update_project_runtime_staging_tag "${transaction_id}" portal)" \ "${prepared_env}" fi set_env_value_atomic \ "${prepared_env}" BRIDGESLLM_UPDATE_PROJECT_RUNTIME_TRANSACTION_ID \ "${transaction_id}" \ && set_env_value_atomic \ "${prepared_env}" BRIDGESLLM_UPDATE_PROJECT_RUNTIMES_PREPARED 1 \ || fail "Could not seal the prepared Project runtime environment." sync -f "${prepared_env}" \ && sync -f "${stage_dir}" \ || fail "Could not durably seal the prepared Project runtime environment." validate_prepared_update_project_runtime_environment \ "${prepared_env}" "${transaction_id}" \ || fail "Prepared Project runtime identities failed validation." } run_staged_portal_rebootability_preflight() { local staged_portal="$1" source_env="$2" plan_file="${3:-}" local backend_dir="${staged_portal}/backend" local preflight="${backend_dir}/dist/cli/portalRebootabilityPreflight.js" local -a arguments=(--env-file "${source_env}") if [[ -n "${plan_file}" ]]; then [[ "${plan_file}" == /* && ! -e "${plan_file}" && ! -L "${plan_file}" ]] \ || return 1 arguments+=(--plan-file "${plan_file}") fi [[ -d "${backend_dir}" && ! -L "${backend_dir}" \ && -f "${preflight}" && ! -L "${preflight}" \ && -f "${source_env}" && ! -L "${source_env}" ]] \ || return 1 [[ "$(stat -c '%u:%g:%h' "${preflight}")" == "0:0:1" \ && "$(stat -c '%u:%g:%h' "${source_env}")" == "0:0:1" \ && $((8#$(stat -c '%a' "${preflight}") & 0022)) -eq 0 \ && $((8#$(stat -c '%a' "${source_env}") & 0022)) -eq 0 ]] \ || return 1 ( cd "${backend_dir}" /usr/bin/node "${preflight}" "${arguments[@]}" ) >> "${LOG_FILE}" 2>&1 } run_staged_portal_continuity_repair() { local staged_portal="$1" source_env="$2" plan_file="$3" local backend_dir="${staged_portal}/backend" local repair="${backend_dir}/dist/cli/portalContinuityRepair.js" [[ -d "${backend_dir}" && ! -L "${backend_dir}" \ && -f "${repair}" && ! -L "${repair}" \ && -f "${source_env}" && ! -L "${source_env}" \ && -f "${plan_file}" && ! -L "${plan_file}" ]] \ || return 1 [[ "$(stat -c '%u:%g:%h' "${repair}")" == "0:0:1" \ && "$(stat -c '%u:%g:%h' "${source_env}")" == "0:0:1" \ && "$(stat -c '%u:%g:%h' "${plan_file}")" == "0:0:1" \ && $((8#$(stat -c '%a' "${repair}") & 0022)) -eq 0 \ && $((8#$(stat -c '%a' "${source_env}") & 0022)) -eq 0 \ && $((8#$(stat -c '%a' "${plan_file}") & 0077)) -eq 0 ]] \ || return 1 ( cd "${backend_dir}" /usr/bin/node "${repair}" \ --env-file "${source_env}" \ --plan-file "${plan_file}" ) >> "${LOG_FILE}" 2>&1 } reattest_quiesced_update_project_egress_generation() { # Preparation happens while the old Portal is serving requests. After its # service is proven stopped, close that admission window by comparing the # sealed candidate ID with a fresh double inventory. A plane created during # staging may use the old environment; cutover must not strand it. local stage_dir="$1" transaction_id="$2" local prepared_env policy prepared_image_id live_image_id="" status=0 prepared_env="$(update_project_runtime_prepared_env_path "${stage_dir}")" \ || return 1 validate_prepared_update_project_runtime_environment \ "${prepared_env}" "${transaction_id}" || return 1 policy="$(read_env_value \ "${prepared_env}" PROJECT_RUNTIME_CONFINEMENT_POLICY || true)" prepared_image_id="$(read_env_value \ "${prepared_env}" PROJECT_EGRESS_PROXY_IMAGE_ID || true)" if live_image_id="$(discover_unique_managed_project_egress_proxy_image_id)"; then status=0 else status=$? fi case "${status}" in 0) [[ "${policy}" != "${PROJECT_RUNTIME_DISABLED_POLICY}" \ && "${live_image_id}" == "${prepared_image_id}" ]] || return 1 ;; 1) # No managed egress plane exists, so there is no live generation for # this candidate to preserve. ;; 2|3|4|5) return 1 ;; *) return 1 ;; esac } apply_prepared_update_project_runtime_environment() { local destination_env="$1" stage_dir="$2" transaction_id="$3" local prepared_env key value prepared_env="$(update_project_runtime_prepared_env_path "${stage_dir}")" \ || return 1 validate_prepared_update_project_runtime_environment \ "${prepared_env}" "${transaction_id}" || return 1 for key in \ PROJECT_RUNTIME_CONFINEMENT_POLICY \ PROJECT_EGRESS_PROXY_IMAGE_ID \ CODEX_PROJECT_SANDBOX_IMAGE_ID \ CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID \ ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID \ OLLAMA_PROJECT_SANDBOX_IMAGE_ID \ AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID \ OPENCLAW_PROJECT_SANDBOX_IMAGE_ID \ PORTAL_PROJECT_RUNTIME_IMAGE_ID; do value="$(read_env_value "${prepared_env}" "${key}" || true)" set_env_value_atomic "${destination_env}" "${key}" "${value}" || return 1 done } update_project_runtime_promotion_manifest_path() { local target="$1" transaction_dir transaction_dir="$(read_update_transaction_field \ "${target}" transaction_dir)" || return 1 [[ -d "${transaction_dir}" && ! -L "${transaction_dir}" ]] || return 1 printf '%s/project-runtime-images-v1.json\n' "${transaction_dir}" } update_project_runtime_promotion_marker_path() { local target="$1" transaction_dir transaction_dir="$(read_update_transaction_field \ "${target}" transaction_dir)" || return 1 [[ -d "${transaction_dir}" && ! -L "${transaction_dir}" ]] || return 1 printf '%s/project-runtime-images-v1.promoted\n' "${transaction_dir}" } read_update_project_runtime_promotion_manifest() { local target="$1" transaction_id manifest transaction_id="$(read_update_transaction_field \ "${target}" transaction_id)" || return 1 manifest="$(update_project_runtime_promotion_manifest_path "${target}")" \ || return 1 python3 - "${manifest}" "${transaction_id}" <<'PY' import json import os import re import stat import sys path, expected_transaction_id = sys.argv[1:] keys = ( "PROJECT_EGRESS_PROXY_IMAGE_ID", "CODEX_PROJECT_SANDBOX_IMAGE_ID", "CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID", "ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID", "OLLAMA_PROJECT_SANDBOX_IMAGE_ID", "AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID", "OPENCLAW_PROJECT_SANDBOX_IMAGE_ID", "PORTAL_PROJECT_RUNTIME_IMAGE_ID", ) flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) fd = os.open(path, flags) try: details = os.fstat(fd) if ( not stat.S_ISREG(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or stat.S_IMODE(details.st_mode) != 0o600 or details.st_nlink != 1 or not 1 <= details.st_size <= 8192 ): raise SystemExit(1) chunks = [] remaining = 8193 while remaining: chunk = os.read(fd, remaining) if not chunk: break chunks.append(chunk) remaining -= len(chunk) raw = b"".join(chunks) after = os.fstat(fd) if ( (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) != (details.st_dev, details.st_ino, details.st_size, details.st_mtime_ns) or len(raw) != details.st_size or len(raw) > 8192 ): raise SystemExit(1) finally: os.close(fd) record = json.loads(raw.decode("ascii")) if ( set(record) != {"schema", "transactionId", "policy", "images"} or record["schema"] != "bridgesllm-update-project-runtime-images-v1" or record["transactionId"] != expected_transaction_id or not isinstance(record["images"], dict) or set(record["images"]) != set(keys) ): raise SystemExit(1) policy = record["policy"] if policy not in { "apparmor-seccomp-v1", "seccomp-only-apparmor-unsupported-v1", "project-runtimes-disabled-v1", }: raise SystemExit(1) image_pattern = re.compile(r"sha256:[a-f0-9]{64}") for key in keys: value = record["images"][key] if not isinstance(value, str) or (value and image_pattern.fullmatch(value) is None): raise SystemExit(1) if policy != "project-runtimes-disabled-v1": for key in ( "PROJECT_EGRESS_PROXY_IMAGE_ID", "CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID", "OLLAMA_PROJECT_SANDBOX_IMAGE_ID", "AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID", "PORTAL_PROJECT_RUNTIME_IMAGE_ID", ): if not record["images"][key]: raise SystemExit(1) canonical = ( json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" ).encode("ascii") if raw != canonical: raise SystemExit(1) print(policy) for key in keys: print(record["images"][key]) PY } seal_update_project_runtime_promotion_manifest() { local target="$1" stage_dir transaction_id transaction_dir prepared_env local backup_dir candidate_env manifest policy key value local -a values=() stage_dir="$(read_update_transaction_field "${target}" stage_dir)" || return 1 transaction_id="$(read_update_transaction_field \ "${target}" transaction_id)" || return 1 transaction_dir="$(read_update_transaction_field \ "${target}" transaction_dir)" || return 1 backup_dir="$(read_update_transaction_field "${target}" backup_dir)" || return 1 prepared_env="$(update_project_runtime_prepared_env_path "${stage_dir}")" \ || return 1 candidate_env="${backup_dir}/environment.updated/backend.env.production" manifest="$(update_project_runtime_promotion_manifest_path "${target}")" \ || return 1 validate_prepared_update_project_runtime_environment \ "${prepared_env}" "${transaction_id}" || return 1 [[ -f "${candidate_env}" && ! -L "${candidate_env}" ]] || return 1 policy="$(read_env_value \ "${prepared_env}" PROJECT_RUNTIME_CONFINEMENT_POLICY || true)" [[ "$(read_env_value \ "${candidate_env}" PROJECT_RUNTIME_CONFINEMENT_POLICY || true)" \ == "${policy}" ]] || return 1 for key in \ PROJECT_EGRESS_PROXY_IMAGE_ID \ CODEX_PROJECT_SANDBOX_IMAGE_ID \ CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID \ ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID \ OLLAMA_PROJECT_SANDBOX_IMAGE_ID \ AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID \ OPENCLAW_PROJECT_SANDBOX_IMAGE_ID \ PORTAL_PROJECT_RUNTIME_IMAGE_ID; do value="$(read_env_value "${prepared_env}" "${key}" || true)" [[ "$(read_env_value "${candidate_env}" "${key}" || true)" == "${value}" ]] \ || return 1 values+=("${value}") done if [[ ! -e "${manifest}" && ! -L "${manifest}" ]]; then python3 - "${transaction_dir}" "${manifest}" "${transaction_id}" \ "${policy}" "${values[@]}" <<'PY' import json import os import secrets import stat import sys directory, destination, transaction_id, policy, *values = sys.argv[1:] keys = ( "PROJECT_EGRESS_PROXY_IMAGE_ID", "CODEX_PROJECT_SANDBOX_IMAGE_ID", "CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_ID", "ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_ID", "OLLAMA_PROJECT_SANDBOX_IMAGE_ID", "AGENT_ZERO_PROJECT_SANDBOX_IMAGE_ID", "OPENCLAW_PROJECT_SANDBOX_IMAGE_ID", "PORTAL_PROJECT_RUNTIME_IMAGE_ID", ) if len(values) != len(keys): raise SystemExit(1) directory_details = os.lstat(directory) if ( not stat.S_ISDIR(directory_details.st_mode) or stat.S_ISLNK(directory_details.st_mode) or directory_details.st_uid != os.geteuid() or directory_details.st_gid != os.getegid() or directory_details.st_mode & 0o077 or os.path.dirname(destination) != directory ): raise SystemExit(1) record = { "schema": "bridgesllm-update-project-runtime-images-v1", "transactionId": transaction_id, "policy": policy, "images": dict(zip(keys, values, strict=True)), } payload = ( json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" ).encode("ascii") temporary = os.path.join( directory, f".project-runtime-images.{os.getpid()}.{secrets.token_hex(8)}" ) fd = -1 try: fd = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) os.write(fd, payload) os.fchmod(fd, 0o600) os.fsync(fd) os.close(fd) fd = -1 if os.path.lexists(destination): raise FileExistsError(destination) os.rename(temporary, destination) directory_fd = os.open( directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if fd >= 0: os.close(fd) try: os.unlink(temporary) except FileNotFoundError: pass PY fi mapfile -t values < <( read_update_project_runtime_promotion_manifest "${target}" ) || return 1 [[ "${#values[@]}" -eq 9 && "${values[0]}" == "${policy}" ]] } assert_prepared_update_project_runtime_images_available() { local target="$1" policy value local -a values=() mapfile -t values < <( read_update_project_runtime_promotion_manifest "${target}" ) || return 1 [[ "${#values[@]}" -eq 9 ]] || return 1 policy="${values[0]}" [[ "${policy}" == "${PROJECT_RUNTIME_DISABLED_POLICY}" ]] && return 0 for value in "${values[@]:1}"; do [[ -z "${value}" ]] && continue [[ "$(docker_image_id "${value}" || true)" == "${value}" ]] || return 1 done } verify_update_project_runtime_promotion_complete() { local target="$1" transaction_id manifest marker manifest_sha local policy image_id canonical_tag index=0 local -a values=() transaction_id="$(read_update_transaction_field \ "${target}" transaction_id)" || return 1 manifest="$(update_project_runtime_promotion_manifest_path "${target}")" \ || return 1 marker="$(update_project_runtime_promotion_marker_path "${target}")" \ || return 1 [[ -f "${marker}" && ! -L "${marker}" ]] || return 1 mapfile -t values < <( read_update_project_runtime_promotion_manifest "${target}" ) || return 1 [[ "${#values[@]}" -eq 9 ]] || return 1 manifest_sha="$(sha256sum "${manifest}" | awk '{print $1}')" python3 - "${marker}" "${transaction_id}" "${manifest_sha}" <<'PY' \ || return 1 import json import os import stat import sys path, transaction_id, manifest_sha = sys.argv[1:] flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) fd = os.open(path, flags) try: details = os.fstat(fd) if ( not stat.S_ISREG(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or stat.S_IMODE(details.st_mode) != 0o600 or details.st_nlink != 1 or not 1 <= details.st_size <= 1024 ): raise SystemExit(1) chunks = [] remaining = 1025 while remaining: chunk = os.read(fd, remaining) if not chunk: break chunks.append(chunk) remaining -= len(chunk) raw = b"".join(chunks) after = os.fstat(fd) if ( (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) != (details.st_dev, details.st_ino, details.st_size, details.st_mtime_ns) or len(raw) != details.st_size or len(raw) > 1024 ): raise SystemExit(1) finally: os.close(fd) record = json.loads(raw.decode("ascii")) if record != { "schema": "bridgesllm-update-project-runtime-promotion-v1", "transactionId": transaction_id, "manifestSha256": manifest_sha, }: raise SystemExit(1) canonical = ( json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" ).encode("ascii") if raw != canonical: raise SystemExit(1) PY for canonical_tag in \ "${PROJECT_EGRESS_PROXY_IMAGE_TAG}" \ "${CODEX_PROJECT_SANDBOX_IMAGE_TAG}" \ "${CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_TAG}" \ "${ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_TAG}" \ "${OLLAMA_PROJECT_SANDBOX_IMAGE_TAG}" \ "${AGENT_ZERO_PROJECT_SANDBOX_IMAGE_TAG}" \ "${OPENCLAW_PROJECT_SANDBOX_IMAGE_TAG}" \ "${PORTAL_PROJECT_RUNTIME_IMAGE_TAG}"; do index=$((index + 1)) image_id="${values[index]}" [[ -z "${image_id}" ]] && continue [[ "$(docker_image_id "${canonical_tag}" || true)" == "${image_id}" ]] \ || return 1 done } promote_prepared_update_project_runtime_images() { local target="$1" transaction_id transaction_dir manifest marker manifest_sha local image_id canonical_tag index=0 local -a values=() verify_update_project_runtime_promotion_complete "${target}" \ && return 0 transaction_id="$(read_update_transaction_field \ "${target}" transaction_id)" || return 1 transaction_dir="$(read_update_transaction_field \ "${target}" transaction_dir)" || return 1 manifest="$(update_project_runtime_promotion_manifest_path "${target}")" \ || return 1 marker="$(update_project_runtime_promotion_marker_path "${target}")" \ || return 1 mapfile -t values < <( read_update_project_runtime_promotion_manifest "${target}" ) || return 1 [[ "${#values[@]}" -eq 9 ]] || return 1 for canonical_tag in \ "${PROJECT_EGRESS_PROXY_IMAGE_TAG}" \ "${CODEX_PROJECT_SANDBOX_IMAGE_TAG}" \ "${CLAUDE_CODE_PROJECT_SANDBOX_IMAGE_TAG}" \ "${ANTIGRAVITY_PROJECT_SANDBOX_IMAGE_TAG}" \ "${OLLAMA_PROJECT_SANDBOX_IMAGE_TAG}" \ "${AGENT_ZERO_PROJECT_SANDBOX_IMAGE_TAG}" \ "${OPENCLAW_PROJECT_SANDBOX_IMAGE_TAG}" \ "${PORTAL_PROJECT_RUNTIME_IMAGE_TAG}"; do index=$((index + 1)) image_id="${values[index]}" [[ -z "${image_id}" ]] && continue [[ "$(docker_image_id "${image_id}" || true)" == "${image_id}" ]] \ || return 1 docker image tag "${image_id}" "${canonical_tag}" \ >> "${LOG_FILE}" 2>&1 || return 1 [[ "$(docker_image_id "${canonical_tag}" || true)" == "${image_id}" ]] \ || return 1 done manifest_sha="$(sha256sum "${manifest}" | awk '{print $1}')" if [[ ! -e "${marker}" && ! -L "${marker}" ]]; then python3 - "${transaction_dir}" "${marker}" \ "${transaction_id}" "${manifest_sha}" <<'PY' import json import os import secrets import stat import sys directory, destination, transaction_id, manifest_sha = sys.argv[1:] details = os.lstat(directory) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_mode & 0o077 or os.path.dirname(destination) != directory ): raise SystemExit(1) temporary = os.path.join( directory, f".project-runtime-images-promoted.{os.getpid()}.{secrets.token_hex(8)}", ) record = { "schema": "bridgesllm-update-project-runtime-promotion-v1", "transactionId": transaction_id, "manifestSha256": manifest_sha, } payload = ( json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" ).encode("ascii") fd = -1 try: fd = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) os.write(fd, payload) os.fchmod(fd, 0o600) os.fsync(fd) os.close(fd) fd = -1 if os.path.lexists(destination): raise FileExistsError(destination) os.rename(temporary, destination) directory_fd = os.open( directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if fd >= 0: os.close(fd) try: os.unlink(temporary) except FileNotFoundError: pass PY fi verify_update_project_runtime_promotion_complete "${target}" } cleanup_prepared_update_project_runtime_tags() { local transaction_id="$1" cleanup_policy="${2:-}" component tag [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 case "${cleanup_policy}" in ""|"${PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY}"|\ "${PROJECT_RUNTIME_SECCOMP_ONLY_POLICY}"|\ "${PROJECT_RUNTIME_DISABLED_POLICY}"|\ "${PROJECT_RUNTIME_PREPARATION_NOT_STARTED_POLICY}") ;; *) return 1 ;; esac # A validated disabled preparation exits before every Docker build/tag # operation, so this exact transaction cannot own candidate tags. Do not # turn an installed-but-stopped Docker daemon into a permanent cleanup # blocker for the explicit --skip-project-runtimes path. if [[ "${cleanup_policy}" == "${PROJECT_RUNTIME_DISABLED_POLICY}" \ || "${cleanup_policy}" \ == "${PROJECT_RUNTIME_PREPARATION_NOT_STARTED_POLICY}" ]]; then return 0 fi if ! command -v docker >/dev/null 2>&1; then # For every enabled or indeterminate preparation, absence of the CLI is # not proof that old Docker metadata vanished; retain the transaction # stage until the exact tags can be inspected and removed. return 1 fi docker info >/dev/null 2>&1 || return 1 for component in \ egress codex claude antigravity ollama agent-zero openclaw portal; do tag="$(update_project_runtime_staging_tag \ "${transaction_id}" "${component}")" || return 1 if docker image inspect "${tag}" >/dev/null 2>&1; then docker image rm "${tag}" >> "${LOG_FILE}" 2>&1 || return 1 else # `image inspect` uses the same non-zero status for "not found" and a # daemon failure. Re-prove daemon reachability before accepting absence. docker info >/dev/null 2>&1 || return 1 fi if docker image inspect "${tag}" >/dev/null 2>&1; then return 1 fi docker info >/dev/null 2>&1 || return 1 done } prepared_update_project_runtime_cleanup_policy_from_stage() { local stage_dir="$1" transaction_id="$2" prepared_env marker local preparation_mode="" policy [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ \ && -d "${stage_dir}" && ! -L "${stage_dir}" \ && "$(stat -c '%u:%g:%a' "${stage_dir}")" \ == "$(id -u):$(id -g):700" ]] || return 1 prepared_env="$(update_project_runtime_prepared_env_path "${stage_dir}")" \ || return 1 marker="$(update_project_runtime_preparation_mode_path "${stage_dir}")" \ || return 1 if [[ -e "${marker}" || -L "${marker}" ]]; then preparation_mode="$( read_update_project_runtime_preparation_mode \ "${stage_dir}" "${transaction_id}" )" || return 1 fi if [[ "${preparation_mode}" == "disabled" ]]; then printf '%s\n' "${PROJECT_RUNTIME_DISABLED_POLICY}" return 0 fi if [[ ! -e "${prepared_env}" && ! -L "${prepared_env}" ]]; then printf '%s\n' "${PROJECT_RUNTIME_PREPARATION_NOT_STARTED_POLICY}" return 0 fi validate_prepared_update_project_runtime_environment \ "${prepared_env}" "${transaction_id}" || return 1 policy="$(read_env_value \ "${prepared_env}" PROJECT_RUNTIME_CONFINEMENT_POLICY || true)" if [[ "${preparation_mode}" == "docker" \ && "${policy}" == "${PROJECT_RUNTIME_DISABLED_POLICY}" ]]; then return 1 fi case "${policy}" in "${PROJECT_RUNTIME_APPARMOR_SECCOMP_POLICY}"|\ "${PROJECT_RUNTIME_SECCOMP_ONLY_POLICY}"|\ "${PROJECT_RUNTIME_DISABLED_POLICY}") printf '%s\n' "${policy}" ;; *) return 1 ;; esac } update_project_runtime_cleanup_policy_for_target() { local target="$1" manifest_output stage_dir transaction_id if manifest_output="$( read_update_project_runtime_promotion_manifest "${target}" 2>/dev/null )"; then printf '%s\n' "${manifest_output%%$'\n'*}" return 0 fi stage_dir="$(read_update_transaction_field "${target}" stage_dir)" \ || return 1 transaction_id="$(read_update_transaction_field \ "${target}" transaction_id)" || return 1 prepared_update_project_runtime_cleanup_policy_from_stage \ "${stage_dir}" "${transaction_id}" } resolve_openclaw_pending_input_hotfix_target() { local openclaw_dist="$1" python3 - "${openclaw_dist}" <<'PY' import os from pathlib import Path import stat import sys dist = Path(sys.argv[1]).resolve() markers = ( "function createCodexUserInputBridge(params) {", 'request.method === "item/tool/requestUserInput"', "const activeSteeringQueue = createCodexSteeringQueue({", "setActiveEmbeddedRun(params.sessionId, handle, params.sessionKey, params.sessionFile);", "clearActiveEmbeddedRun(params.sessionId, handle, params.sessionKey, params.sessionFile);", ) matches = [] for candidate in sorted(dist.glob("run-attempt-*.js")): try: metadata = os.lstat(candidate) if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): continue text = candidate.read_text(encoding="utf-8") except (OSError, UnicodeError): continue if all(marker in text for marker in markers): matches.append(candidate.resolve()) if len(matches) != 1: raise SystemExit(1) print(matches[0]) PY } openclaw_pending_input_hotfix_is_applied() { local target="$1" python3 - "${target}" <<'PY' from pathlib import Path import sys marker = 'const BRIDGESLLM_PENDING_INPUT_HOTFIX_MARKER = "bridgesllm-openclaw-pending-input-v1";' try: text = Path(sys.argv[1]).read_text(encoding="utf-8") except (OSError, UnicodeError): raise SystemExit(1) raise SystemExit(0 if text.count(marker) == 1 else 1) PY } arm_openclaw_pending_input_hotfix_rollback() { local target="$1" local backup="${target}.bridgesllm-pending-input-v1.bak" python3 - "${target}" "${backup}" <<'PY' import os from pathlib import Path import stat import subprocess import sys target = Path(sys.argv[1]) backup = Path(sys.argv[2]) marker = 'const BRIDGESLLM_PENDING_INPUT_HOTFIX_MARKER = "bridgesllm-openclaw-pending-input-v1";' if backup != target.with_name(target.name + ".bridgesllm-pending-input-v1.bak"): raise SystemExit(1) for path in (target, backup): try: metadata = os.lstat(path) except OSError: raise SystemExit(1) if ( not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 ): raise SystemExit(1) try: target_text = target.read_text(encoding="utf-8") backup_text = backup.read_text(encoding="utf-8") except (OSError, UnicodeError): raise SystemExit(1) if target_text.count(marker) != 1 or marker in backup_text: raise SystemExit(1) def valid_javascript(text): # The OpenClaw bundle is an ES module, but `node --check -` parses stdin as # CommonJS no matter what the nearest package.json says, so every ESM bundle # fails this check as written. Check the module grammar the bundle actually # uses, and still accept CommonJS so an upstream switch to a script bundle # cannot turn this into a false negative. for module_type in ("module", "commonjs"): if subprocess.run( ["node", f"--input-type={module_type}", "--check", "-"], input=text, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ).returncode == 0: return True return False for text in (target_text, backup_text): if not valid_javascript(text): raise SystemExit(1) PY OPENCLAW_PENDING_INPUT_HOTFIX_TARGET="${target}" OPENCLAW_PENDING_INPUT_HOTFIX_BACKUP="${backup}" OPENCLAW_PENDING_INPUT_HOTFIX_APPLIED=true OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED=false } rollback_openclaw_pending_input_hotfix() { $OPENCLAW_PENDING_INPUT_HOTFIX_APPLIED || return 0 $OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED && return 0 local target="${OPENCLAW_PENDING_INPUT_HOTFIX_TARGET:-}" local backup="${OPENCLAW_PENDING_INPUT_HOTFIX_BACKUP:-}" [[ -n "${target}" && -n "${backup}" ]] || return 1 if ! python3 - "${target}" "${backup}" <<'PY' import os from pathlib import Path import stat import subprocess import sys target = Path(sys.argv[1]) backup = Path(sys.argv[2]) marker = 'const BRIDGESLLM_PENDING_INPUT_HOTFIX_MARKER = "bridgesllm-openclaw-pending-input-v1";' if backup != target.with_name(target.name + ".bridgesllm-pending-input-v1.bak"): raise SystemExit(1) for path in (target, backup): try: metadata = os.lstat(path) except OSError: raise SystemExit(1) if ( not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 ): raise SystemExit(1) try: target_text = target.read_text(encoding="utf-8") backup_text = backup.read_text(encoding="utf-8") except (OSError, UnicodeError): raise SystemExit(1) if target_text.count(marker) != 1 or marker in backup_text: raise SystemExit(1) def valid_javascript(text): # The OpenClaw bundle is an ES module, but `node --check -` parses stdin as # CommonJS no matter what the nearest package.json says, so every ESM bundle # fails this check as written. Check the module grammar the bundle actually # uses, and still accept CommonJS so an upstream switch to a script bundle # cannot turn this into a false negative. for module_type in ("module", "commonjs"): if subprocess.run( ["node", f"--input-type={module_type}", "--check", "-"], input=text, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ).returncode == 0: return True return False if not valid_javascript(backup_text): raise SystemExit(1) os.replace(backup, target) directory_fd = os.open(target.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY then return 1 fi OPENCLAW_PENDING_INPUT_HOTFIX_APPLIED=false OPENCLAW_PENDING_INPUT_HOTFIX_TARGET="" OPENCLAW_PENDING_INPUT_HOTFIX_BACKUP="" OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED=false } commit_openclaw_pending_input_hotfix() { if ! $OPENCLAW_PENDING_INPUT_HOTFIX_APPLIED; then OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED=true return 0 fi $OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED && return 0 local target="${OPENCLAW_PENDING_INPUT_HOTFIX_TARGET:-}" local backup="${OPENCLAW_PENDING_INPUT_HOTFIX_BACKUP:-}" [[ -n "${target}" && -n "${backup}" ]] || return 1 if ! python3 - "${target}" "${backup}" <<'PY' import os from pathlib import Path import stat import subprocess import sys target = Path(sys.argv[1]) backup = Path(sys.argv[2]) marker = 'const BRIDGESLLM_PENDING_INPUT_HOTFIX_MARKER = "bridgesllm-openclaw-pending-input-v1";' if backup != target.with_name(target.name + ".bridgesllm-pending-input-v1.bak"): raise SystemExit(1) for path in (target, backup): try: metadata = os.lstat(path) except OSError: raise SystemExit(1) if ( not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 ): raise SystemExit(1) try: target_text = target.read_text(encoding="utf-8") backup_text = backup.read_text(encoding="utf-8") except (OSError, UnicodeError): raise SystemExit(1) if target_text.count(marker) != 1 or marker in backup_text: raise SystemExit(1) def valid_javascript(text): # The OpenClaw bundle is an ES module, but `node --check -` parses stdin as # CommonJS no matter what the nearest package.json says, so every ESM bundle # fails this check as written. Check the module grammar the bundle actually # uses, and still accept CommonJS so an upstream switch to a script bundle # cannot turn this into a false negative. for module_type in ("module", "commonjs"): if subprocess.run( ["node", f"--input-type={module_type}", "--check", "-"], input=text, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ).returncode == 0: return True return False for text in (target_text, backup_text): if not valid_javascript(text): raise SystemExit(1) backup.unlink() directory_fd = os.open(target.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY then return 1 fi OPENCLAW_PENDING_INPUT_HOTFIX_BACKUP="" OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED=true } resolve_openclaw_claude_ask_user_hotfix_target() { local openclaw_dist="$1" python3 - "${openclaw_dist}" <<'PY' import os from pathlib import Path import stat import sys dist = Path(sys.argv[1]).resolve() backup_suffix = ".bridgesllm-claude-ask-user-route-v2.bak" markers = ( 'const CLAUDE_DISALLOWED_TOOLS_ARG = "--disallowedTools";', "function resolveClaudePermissionMode(context) {", "function normalizeClaudeBackendConfig(config, context) {", ) matches = [] for candidate in sorted(dist.glob("cli-shared-*.js")): try: metadata = os.lstat(candidate) if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): continue except OSError: continue texts = [] try: texts.append(candidate.read_text(encoding="utf-8")) except (OSError, UnicodeError): pass backup = candidate.with_name(candidate.name + backup_suffix) if os.path.lexists(backup): try: backup_metadata = os.lstat(backup) if stat.S_ISREG(backup_metadata.st_mode) and not stat.S_ISLNK(backup_metadata.st_mode): texts.append(backup.read_text(encoding="utf-8")) except (OSError, UnicodeError): pass if any(all(marker in text for marker in markers) for text in texts): matches.append(candidate.resolve()) if len(matches) != 1: raise SystemExit(1) print(matches[0]) PY } openclaw_claude_ask_user_hotfix_is_applied() { local target="$1" python3 - "${target}" <<'PY' from pathlib import Path import sys marker = 'const BRIDGESLLM_CLAUDE_ASK_USER_ROUTE_MARKER = "bridgesllm-openclaw-claude-ask-user-route-v2";' try: text = Path(sys.argv[1]).read_text(encoding="utf-8") except (OSError, UnicodeError): raise SystemExit(1) if ( text.count(marker) != 1 or text.count("function ensureClaudeDisallowedTool(args, toolName) {") != 1 or text.count( "function ensureClaudeDisallowedTool(args, toolName) {\n" "\tif (!args) return args;\n" "\tconst normalized = [...args];" ) != 1 or text.count("args: ensureClaudeDisallowedTool(") != 1 or text.count("resumeArgs: ensureClaudeDisallowedTool(") != 1 ): raise SystemExit(1) PY } prepare_openclaw_claude_ask_user_hotfix_rollback() { local target="$1" local result backup if ! result="$(python3 - "${target}" <<'PY' import os from pathlib import Path import shutil import stat import subprocess import sys import tempfile target = Path(sys.argv[1]) backup = target.with_name(target.name + ".bridgesllm-claude-ask-user-route-v2.bak") marker = 'const BRIDGESLLM_CLAUDE_ASK_USER_ROUTE_MARKER = "bridgesllm-openclaw-claude-ask-user-route-v2";' def regular_single_link(path): metadata = os.lstat(path) return ( stat.S_ISREG(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_nlink == 1 and metadata.st_uid == 0 and metadata.st_gid == 0 ) def valid_javascript(text): for module_type in ("module", "commonjs"): if subprocess.run( ["node", f"--input-type={module_type}", "--check", "-"], input=text, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ).returncode == 0: return True return False def pristine_contract(text): return ( marker not in text and "function ensureClaudeDisallowedTool(args, toolName) {" not in text and text.count("function normalizeClaudeBackendConfig(config, context) {") == 1 and text.count("args: normalizeClaudePermissionArgs(") == 1 and text.count("resumeArgs: normalizeClaudePermissionArgs(") == 1 ) def applied_contract(text): return ( text.count(marker) == 1 and text.count("function ensureClaudeDisallowedTool(args, toolName) {") == 1 and text.count( "function ensureClaudeDisallowedTool(args, toolName) {\n" "\tif (!args) return args;\n" "\tconst normalized = [...args];" ) == 1 and text.count("args: ensureClaudeDisallowedTool(") == 1 and text.count("resumeArgs: ensureClaudeDisallowedTool(") == 1 ) try: if not regular_single_link(target): raise SystemExit(1) except OSError: raise SystemExit(1) try: target_text = target.read_text(encoding="utf-8") except (OSError, UnicodeError): target_text = None backup_exists = os.path.lexists(backup) if backup_exists: try: if not regular_single_link(backup): raise SystemExit(1) backup_text = backup.read_text(encoding="utf-8") except (OSError, UnicodeError): backup_text = None if ( backup_text is not None and valid_javascript(backup_text) and pristine_contract(backup_text) ): # A recovery artifact from an interrupted run owns the baseline even # when the target stopped between write and marker verification. print(backup) raise SystemExit(0) # Older builds wrote directly to the final backup name. A torn backup is # safe to replace only while the live target is still a fully validated, # pristine baseline. Never discard the sole recovery copy of a changed or # unreadable target. if ( target_text is None or not valid_javascript(target_text) or not pristine_contract(target_text) ): raise SystemExit(1) backup.unlink() directory_fd = os.open(target.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) if target_text is None or not valid_javascript(target_text): raise SystemExit(1) if applied_contract(target_text): print("committed") raise SystemExit(0) if not pristine_contract(target_text): raise SystemExit(1) target_mode = target.stat().st_mode & 0o7777 temporary_name = "" try: with tempfile.NamedTemporaryFile( mode="wb", dir=target.parent, prefix=f".{target.name}.bridgesllm-claude-backup-", suffix=".tmp", delete=False, ) as stream: temporary_name = stream.name stream.write(target.read_bytes()) stream.flush() os.fsync(stream.fileno()) shutil.copystat(target, temporary_name, follow_symlinks=True) os.chmod(temporary_name, target_mode) if os.path.lexists(backup): raise FileExistsError(backup) # The package directory is root-owned and the installer operation lock # excludes another writer. Publish only complete, fsynced bytes. os.replace(temporary_name, backup) temporary_name = "" directory_fd = os.open(target.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) except BaseException: if temporary_name: try: Path(temporary_name).unlink() except FileNotFoundError: pass raise print(backup) PY )"; then return 1 fi if [[ "${result}" == "committed" ]]; then OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED=true return 0 fi backup="${target}.bridgesllm-claude-ask-user-route-v2.bak" [[ "${result}" == "${backup}" && -f "${backup}" ]] || return 1 OPENCLAW_CLAUDE_ASK_USER_HOTFIX_TARGET="${target}" OPENCLAW_CLAUDE_ASK_USER_HOTFIX_BACKUP="${backup}" OPENCLAW_CLAUDE_ASK_USER_HOTFIX_APPLIED=true OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED=false } rollback_openclaw_claude_ask_user_hotfix() { $OPENCLAW_CLAUDE_ASK_USER_HOTFIX_APPLIED || return 0 $OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED && return 0 local target="${OPENCLAW_CLAUDE_ASK_USER_HOTFIX_TARGET:-}" local backup="${OPENCLAW_CLAUDE_ASK_USER_HOTFIX_BACKUP:-}" [[ -n "${target}" && -n "${backup}" ]] || return 1 if ! python3 - "${target}" "${backup}" <<'PY' import os from pathlib import Path import stat import subprocess import sys target = Path(sys.argv[1]) backup = Path(sys.argv[2]) marker = 'const BRIDGESLLM_CLAUDE_ASK_USER_ROUTE_MARKER = "bridgesllm-openclaw-claude-ask-user-route-v2";' if backup != target.with_name(target.name + ".bridgesllm-claude-ask-user-route-v2.bak"): raise SystemExit(1) def regular_single_link(path): metadata = os.lstat(path) return ( stat.S_ISREG(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_nlink == 1 and metadata.st_uid == 0 and metadata.st_gid == 0 ) try: if not regular_single_link(target) or not regular_single_link(backup): raise SystemExit(1) backup_text = backup.read_text(encoding="utf-8") except (OSError, UnicodeError): raise SystemExit(1) if ( marker in backup_text or "function ensureClaudeDisallowedTool(args, toolName) {" in backup_text or backup_text.count("function normalizeClaudeBackendConfig(config, context) {") != 1 ): raise SystemExit(1) if not any( subprocess.run( ["node", f"--input-type={module_type}", "--check", "-"], input=backup_text, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ).returncode == 0 for module_type in ("module", "commonjs") ): raise SystemExit(1) os.replace(backup, target) directory_fd = os.open(target.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY then return 1 fi OPENCLAW_CLAUDE_ASK_USER_HOTFIX_APPLIED=false OPENCLAW_CLAUDE_ASK_USER_HOTFIX_TARGET="" OPENCLAW_CLAUDE_ASK_USER_HOTFIX_BACKUP="" OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED=false } commit_openclaw_claude_ask_user_hotfix() { if ! $OPENCLAW_CLAUDE_ASK_USER_HOTFIX_APPLIED; then OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED=true return 0 fi $OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED && return 0 local target="${OPENCLAW_CLAUDE_ASK_USER_HOTFIX_TARGET:-}" local backup="${OPENCLAW_CLAUDE_ASK_USER_HOTFIX_BACKUP:-}" [[ -n "${target}" && -n "${backup}" ]] || return 1 openclaw_claude_ask_user_hotfix_is_applied "${target}" || return 1 if ! python3 - "${target}" "${backup}" <<'PY' import os from pathlib import Path import stat import sys target = Path(sys.argv[1]) backup = Path(sys.argv[2]) marker = 'const BRIDGESLLM_CLAUDE_ASK_USER_ROUTE_MARKER = "bridgesllm-openclaw-claude-ask-user-route-v2";' if backup != target.with_name(target.name + ".bridgesllm-claude-ask-user-route-v2.bak"): raise SystemExit(1) for path in (target, backup): metadata = os.lstat(path) if ( not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 or metadata.st_uid != 0 or metadata.st_gid != 0 ): raise SystemExit(1) target_text = target.read_text(encoding="utf-8") backup_text = backup.read_text(encoding="utf-8") if target_text.count(marker) != 1 or marker in backup_text: raise SystemExit(1) backup.unlink() directory_fd = os.open(target.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY then return 1 fi OPENCLAW_CLAUDE_ASK_USER_HOTFIX_BACKUP="" OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED=true } openclaw_tested_pair_commit_record_matches_target() { local component="$1" target="$2" local record="${OPENCLAW_TESTED_PAIR_COMMIT_RECORD}" python3 - "${record}" "${component}" "${target}" <<'PY' import hashlib import json import os from pathlib import Path import stat import sys record = Path(sys.argv[1]) component = sys.argv[2] target = Path(sys.argv[3]) if component not in {"claudeAskUser", "pendingInput"}: raise SystemExit(1) backup_suffix = { "pendingInput": ".bridgesllm-pending-input-v1.bak", "claudeAskUser": ".bridgesllm-claude-ask-user-route-v2.bak", }[component] backup = target.with_name(target.name + backup_suffix) def safe_file(path): metadata = os.lstat(path) return ( stat.S_ISREG(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_nlink == 1 and metadata.st_uid == 0 and metadata.st_gid == 0 ) def backup_identity(path): metadata = os.lstat(path) return { "path": str(path.resolve()), "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), "device": metadata.st_dev, "inode": metadata.st_ino, "ctimeNs": metadata.st_ctime_ns, "size": metadata.st_size, } try: record_stat = os.lstat(record) if ( not stat.S_ISREG(record_stat.st_mode) or stat.S_ISLNK(record_stat.st_mode) or record_stat.st_nlink != 1 or record_stat.st_uid != 0 or record_stat.st_gid != 0 or stat.S_IMODE(record_stat.st_mode) != 0o600 or not safe_file(target) or not safe_file(backup) ): raise SystemExit(1) value = json.loads(record.read_text(encoding="utf-8")) target_bytes = target.read_bytes() observed_backup = backup_identity(backup) except (OSError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) if not isinstance(value, dict): raise SystemExit(1) entry = value.get(component) if ( value.get("schema") != "bridgesllm-openclaw-tested-pair-commit-v3" or not isinstance(entry, dict) or entry.get("path") != str(target.resolve()) or entry.get("sha256") != hashlib.sha256(target_bytes).hexdigest() or entry.get("backup") != observed_backup ): raise SystemExit(1) PY } openclaw_tested_pair_commit_record_matches_ask_user_transaction() { local transaction_dir="$1" target_dir="$2" config_path="$3" local record="${OPENCLAW_TESTED_PAIR_COMMIT_RECORD}" python3 - \ "${record}" \ "${transaction_dir}" \ "${target_dir}" \ "${config_path}" <<'PY' import hashlib import json import os from pathlib import Path import stat import sys record = Path(sys.argv[1]).absolute() transaction = Path(sys.argv[2]).absolute() target = Path(sys.argv[3]).absolute() config = Path(sys.argv[4]).absolute() manifest = transaction / "manifest.json" def safe_file(path): metadata = os.lstat(path) return ( stat.S_ISREG(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_nlink == 1 and metadata.st_uid == 0 and metadata.st_gid == 0 ) def safe_directory(path): metadata = os.lstat(path) return ( stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_gid == 0 ) def tree_digest(root): digest = hashlib.sha256() for path in [root, *sorted(root.rglob("*"), key=lambda item: str(item.relative_to(root)))]: metadata = os.lstat(path) relative = "." if path == root else str(path.relative_to(root)) digest.update(relative.encode("utf-8") + b"\0") if stat.S_ISDIR(metadata.st_mode): digest.update(b"d\0") elif stat.S_ISREG(metadata.st_mode): digest.update(b"f\0" + path.read_bytes()) elif stat.S_ISLNK(metadata.st_mode): digest.update(b"l\0" + os.readlink(path).encode("utf-8")) else: raise OSError(f"unsupported plugin entry: {path}") return digest.hexdigest() try: if ( not safe_file(record) or stat.S_IMODE(os.lstat(record).st_mode) != 0o600 or not safe_directory(transaction) or not safe_directory(target) or not safe_file(config) or stat.S_IMODE(os.lstat(config).st_mode) != 0o600 or not safe_file(manifest) ): raise OSError("unsafe ask-user commit artifacts") value = json.loads(record.read_text(encoding="utf-8")) ask_user = value.get("askUser") manifest_value = json.loads(manifest.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) if ( value.get("schema") != "bridgesllm-openclaw-tested-pair-commit-v3" or not isinstance(ask_user, dict) or manifest_value.get("schema") != "bridgesllm-ask-user-tested-pair-v1" or Path(manifest_value.get("targetDir", "")).absolute() != target or Path(manifest_value.get("configPath", "")).absolute() != config or ask_user.get("transactionPath") != str(transaction) or ask_user.get("transactionManifestSha256") != hashlib.sha256(manifest.read_bytes()).hexdigest() or ask_user.get("pluginPath") != str(target) or ask_user.get("pluginTreeSha256") != tree_digest(target) or ask_user.get("configPath") != str(config) or ask_user.get("configSha256") != hashlib.sha256(config.read_bytes()).hexdigest() ): raise SystemExit(1) PY } write_openclaw_tested_pair_commit_record() { local pending_target="$1" claude_target="$2" local ask_user_transaction="$3" ask_user_target="$4" ask_user_config="$5" local record="${OPENCLAW_TESTED_PAIR_COMMIT_RECORD}" python3 - \ "${record}" \ "${pending_target}" \ "${claude_target}" \ "${ask_user_transaction}" \ "${ask_user_target}" \ "${ask_user_config}" <<'PY' import hashlib import json import os from pathlib import Path import stat import sys import tempfile record = Path(sys.argv[1]) pending_target = Path(sys.argv[2]) claude_target = Path(sys.argv[3]) ask_user_transaction = Path(sys.argv[4]).absolute() ask_user_target = Path(sys.argv[5]).absolute() ask_user_config = Path(sys.argv[6]).absolute() ask_user_manifest = ask_user_transaction / "manifest.json" parent = record.parent backup_specs = { "pendingInput": ( pending_target, pending_target.with_name( pending_target.name + ".bridgesllm-pending-input-v1.bak" ), ), "claudeAskUser": ( claude_target, claude_target.with_name( claude_target.name + ".bridgesllm-claude-ask-user-route-v2.bak" ), ), } def safe_file(path): metadata = os.lstat(path) return ( stat.S_ISREG(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_nlink == 1 and metadata.st_uid == 0 and metadata.st_gid == 0 ) def safe_directory(path): metadata = os.lstat(path) return ( stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_gid == 0 ) def tree_digest(root): digest = hashlib.sha256() for path in [root, *sorted(root.rglob("*"), key=lambda item: str(item.relative_to(root)))]: metadata = os.lstat(path) relative = "." if path == root else str(path.relative_to(root)) digest.update(relative.encode("utf-8") + b"\0") if stat.S_ISDIR(metadata.st_mode): digest.update(b"d\0") elif stat.S_ISREG(metadata.st_mode): digest.update(b"f\0" + path.read_bytes()) elif stat.S_ISLNK(metadata.st_mode): digest.update(b"l\0" + os.readlink(path).encode("utf-8")) else: raise OSError(f"unsupported plugin entry: {path}") return digest.hexdigest() def component_entry(target, backup): entry = { "path": str(target.resolve()), "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), "backup": None, } if os.path.lexists(backup): if not safe_file(backup): raise OSError(f"unsafe tested-pair backup: {backup}") metadata = os.lstat(backup) entry["backup"] = { "path": str(backup.resolve()), "sha256": hashlib.sha256(backup.read_bytes()).hexdigest(), "device": metadata.st_dev, "inode": metadata.st_ino, "ctimeNs": metadata.st_ctime_ns, "size": metadata.st_size, } return entry try: parent_stat = os.lstat(parent) if ( not stat.S_ISDIR(parent_stat.st_mode) or stat.S_ISLNK(parent_stat.st_mode) or parent_stat.st_uid != 0 or parent_stat.st_gid != 0 ): raise SystemExit(1) for target in (pending_target, claude_target): if not safe_file(target): raise SystemExit(1) if ( not safe_directory(ask_user_transaction) or not safe_directory(ask_user_target) or not safe_file(ask_user_config) or stat.S_IMODE(os.lstat(ask_user_config).st_mode) != 0o600 or not safe_file(ask_user_manifest) ): raise SystemExit(1) ask_user_manifest_value = json.loads( ask_user_manifest.read_text(encoding="utf-8") ) if ( ask_user_manifest_value.get("schema") != "bridgesllm-ask-user-tested-pair-v1" or Path(ask_user_manifest_value.get("targetDir", "")).absolute() != ask_user_target or Path(ask_user_manifest_value.get("configPath", "")).absolute() != ask_user_config ): raise SystemExit(1) if os.path.lexists(record) and not safe_file(record): raise SystemExit(1) value = { "schema": "bridgesllm-openclaw-tested-pair-commit-v3", **{ component: component_entry(target, backup) for component, (target, backup) in backup_specs.items() }, "askUser": { "transactionPath": str(ask_user_transaction), "transactionManifestSha256": hashlib.sha256( ask_user_manifest.read_bytes() ).hexdigest(), "pluginPath": str(ask_user_target), "pluginTreeSha256": tree_digest(ask_user_target), "configPath": str(ask_user_config), "configSha256": hashlib.sha256(ask_user_config.read_bytes()).hexdigest(), }, } except (OSError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) temporary_name = "" try: with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", dir=parent, prefix=".bridgesllm-tested-pair-commit-", suffix=".tmp", delete=False, ) as stream: temporary_name = stream.name json.dump(value, stream, sort_keys=True, separators=(",", ":")) stream.write("\n") stream.flush() os.fsync(stream.fileno()) os.chmod(temporary_name, 0o600) os.replace(temporary_name, record) temporary_name = "" directory_fd = os.open(parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary_name: try: Path(temporary_name).unlink() except FileNotFoundError: pass PY } retire_openclaw_tested_pair_commit_record_if_clean() { local pending_target="$1" claude_target="$2" ask_user_transaction="$3" local record="${OPENCLAW_TESTED_PAIR_COMMIT_RECORD}" python3 - \ "${record}" \ "${pending_target}" \ "${claude_target}" \ "${ask_user_transaction}" <<'PY' import os from pathlib import Path import stat import sys record = Path(sys.argv[1]) pending_target = Path(sys.argv[2]) claude_target = Path(sys.argv[3]) ask_user_transaction = Path(sys.argv[4]) backups = ( pending_target.with_name( pending_target.name + ".bridgesllm-pending-input-v1.bak" ), claude_target.with_name( claude_target.name + ".bridgesllm-claude-ask-user-route-v2.bak" ), ) if any(os.path.lexists(path) for path in backups) or os.path.lexists(ask_user_transaction): raise SystemExit(0) if not os.path.lexists(record): raise SystemExit(0) try: # A cleanup helper can unlink successfully and still fail its directory # fsync. Re-prove durable absence in every artifact directory before # retiring the only record that authorizes a backup which could reappear # after power loss. artifact_parents = { path.parent.resolve() for path in (*backups, ask_user_transaction) } for directory in artifact_parents: directory_metadata = os.lstat(directory) if ( not stat.S_ISDIR(directory_metadata.st_mode) or stat.S_ISLNK(directory_metadata.st_mode) or directory_metadata.st_uid != 0 or directory_metadata.st_gid != 0 ): raise SystemExit(1) directory_fd = os.open(directory, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) metadata = os.lstat(record) if ( not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 or metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) != 0o600 ): raise SystemExit(1) record.unlink() directory_fd = os.open(record.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) except OSError: raise SystemExit(1) PY } auto_apply_openclaw_compatibility_hotfix() { if $SKIP_OPENCLAW || ! command -v openclaw &>/dev/null; then return 0 fi if ! $OPENCLAW_ASK_USER_BASE_ATTESTED; then fail "Native Claude questions may only be suppressed after the replacement ask-user tool and all settlement methods pass live base probes." fi local hotfix_script="${PORTAL_DIR}/scripts/patch-openclaw-long-run-relay-hotfix.sh" local pending_input_hotfix_script="${PORTAL_DIR}/scripts/patch-openclaw-codex-pending-input-hotfix.sh" local openclaw_package_dir openclaw_dist pending_input_target local claude_ask_user_target local pending_input_backup pending_input_hotfix_status=0 local pending_input_hotfix_preexisted=false openclaw_package_dir="$(openclaw_core_package_dir || true)" openclaw_dist="${openclaw_package_dir}/dist" if [[ ! -f "${hotfix_script}" ]]; then fail "Bundled OpenClaw compatibility hotfix script is missing; refusing to start the tested runtime without its required compatibility markers." fi if [[ ! -f "${pending_input_hotfix_script}" ]]; then fail "Bundled OpenClaw native pending-input hotfix is missing; refusing to start a runtime that cannot attest exact Codex request identity." fi if [[ -z "${openclaw_package_dir}" \ || "$(node_package_name_from_dir "${openclaw_package_dir}" || true)" != "openclaw" \ || "$(node_package_version_from_dir "${openclaw_package_dir}" || true)" != "${PIN_OPENCLAW_CORE_PACKAGE_VERSION}" \ || ! -d "${openclaw_dist}" ]]; then fail "Could not resolve the exact OpenClaw ${PIN_OPENCLAW_CORE_PACKAGE_VERSION} package directory for required compatibility preparation." fi claude_ask_user_target="$( resolve_openclaw_claude_ask_user_hotfix_target "${openclaw_dist}" || true )" if [[ -z "${claude_ask_user_target}" ]]; then fail "Could not resolve exactly one Claude CLI normalization bundle before compatibility preparation." fi if ! prepare_openclaw_claude_ask_user_hotfix_rollback \ "${claude_ask_user_target}"; then fail "Could not preserve an exact Claude CLI normalization bundle before compatibility preparation." fi if $OPENCLAW_CLAUDE_ASK_USER_HOTFIX_APPLIED \ && openclaw_claude_ask_user_hotfix_is_applied \ "${claude_ask_user_target}" \ && openclaw_tested_pair_commit_record_matches_target \ claudeAskUser "${claude_ask_user_target}"; then if ! commit_openclaw_claude_ask_user_hotfix; then fail "Could not retire a Claude ask-user recovery artifact already covered by the durable tested-pair commit record." fi fi pending_input_target="$( resolve_openclaw_pending_input_hotfix_target "${openclaw_dist}" || true )" if [[ -z "${pending_input_target}" ]]; then fail "Could not resolve exactly one native Codex pending-input bundle before compatibility preparation." fi pending_input_backup="${pending_input_target}.bridgesllm-pending-input-v1.bak" if openclaw_pending_input_hotfix_is_applied "${pending_input_target}"; then pending_input_hotfix_preexisted=true # A patched target plus our exact backup is an interrupted prior # transaction, not a committed baseline. Re-arm it so this run either # restores the original bytes on failure or retires the backup on commit. if [[ -e "${pending_input_backup}" || -L "${pending_input_backup}" ]] \ && ! arm_openclaw_pending_input_hotfix_rollback \ "${pending_input_target}"; then fail "The existing native pending-input hotfix recovery artifact is unsafe at ${pending_input_backup}; refusing to discard or overwrite it." fi if $OPENCLAW_PENDING_INPUT_HOTFIX_APPLIED \ && openclaw_tested_pair_commit_record_matches_target \ pendingInput "${pending_input_target}"; then if ! commit_openclaw_pending_input_hotfix; then fail "Could not retire a native pending-input recovery artifact already covered by the durable tested-pair commit record." fi fi elif [[ -e "${pending_input_backup}" || -L "${pending_input_backup}" ]]; then fail "A stale native pending-input hotfix backup already exists at ${pending_input_backup}; refusing to overwrite an uncommitted recovery artifact." fi # The commit record is needed only while at least one exact cleanup artifact # survives. Retire it once both are gone. A later same-pin reinstall can # therefore never reuse an old decision, and v3 artifact identities make even # an interrupted cleanup record fail closed against newly created backups. if ! retire_openclaw_tested_pair_commit_record_if_clean \ "${pending_input_target}" \ "${claude_ask_user_target}" \ "/root/.openclaw/.bridgesllm-ask-user-tested-pair-v1"; then fail "Could not safely retire the completed OpenClaw tested-pair commit record." fi chmod 755 "${hotfix_script}" "${pending_input_hotfix_script}" 2>/dev/null || true info "Applying OpenClaw compatibility hotfix (if needed)..." if ! run_openclaw_compatibility_hotfix \ "${hotfix_script}" "${openclaw_dist}"; then fail "OpenClaw compatibility preparation failed or required markers were not verified; the tested-pair rollback remains armed." fi if ! openclaw_claude_ask_user_hotfix_is_applied \ "${claude_ask_user_target}"; then fail "OpenClaw compatibility preparation returned without the required post-merge Claude ask-user routing contract." fi if spin "Applying OpenClaw native pending-input hotfix (if needed)" \ "PORTAL_OPENCLAW_PENDING_INPUT_STRICT=1 PORTAL_REQUIRED_OPENCLAW_PACKAGE_NAME='openclaw' PORTAL_REQUIRED_OPENCLAW_PACKAGE_VERSION='${PIN_OPENCLAW_CORE_PACKAGE_VERSION}' bash '${pending_input_hotfix_script}' '${openclaw_dist}'"; then pending_input_hotfix_status=0 else pending_input_hotfix_status=$? fi if ! $pending_input_hotfix_preexisted \ && openclaw_pending_input_hotfix_is_applied "${pending_input_target}"; then if ! arm_openclaw_pending_input_hotfix_rollback "${pending_input_target}"; then fail "OpenClaw native pending-input preparation changed the runtime, but its exact rollback artifact could not be proven safe. Manual recovery is required from ${pending_input_backup}." fi fi if (( pending_input_hotfix_status != 0 )); then fail "OpenClaw native pending-input preparation failed or exact run/request markers were not verified; the tested-pair rollback remains armed." fi if ! openclaw_pending_input_hotfix_is_applied "${pending_input_target}"; then fail "OpenClaw native pending-input preparation returned without the required exact runtime marker." fi # The replacement ask-user bridge was installed and semantically attested # before this function was allowed to patch native Claude routing. Restart # now so no running gateway can retain a half-old compatibility generation. if systemctl is-enabled openclaw-gateway &>/dev/null 2>&1; then if ! spin "Restarting OpenClaw gateway after compatibility hotfix" "systemctl restart openclaw-gateway"; then fail "OpenClaw gateway restart after required compatibility preparation failed." fi sleep 3 fi ok "OpenClaw compatibility hotfix checked" } run_openclaw_state_repair_notice() { if $SKIP_OPENCLAW || ! command -v openclaw &>/dev/null; then return 0 fi ok "OpenClaw state repair skipped (manual/admin action only)" } repair_openclaw_portal_model_config() { if $SKIP_OPENCLAW || ! command -v node &>/dev/null; then return 0 fi local helper="${PORTAL_DIR}/backend/dist/utils/openclawCli.js" if [[ ! -f "${helper}" ]]; then warn "Portal OpenClaw model repair helper is missing. Skipping model config normalization." return 0 fi if ! spin "Normalizing OpenClaw model configuration" "OPENCLAW_ALLOW_ROOT=1 NODE_PATH='${PORTAL_DIR}/backend/node_modules' node -e 'const helper = process.argv[1]; const mod = require(helper); const result = mod.repairClaudeSubscriptionConfig(); console.log(JSON.stringify(result));' '${helper}'"; then warn "OpenClaw model configuration normalization failed. Continuing install/update; check ${LOG_FILE}." return 0 fi ok "OpenClaw model configuration checked" } bridge_openclaw_codex_cli_auth() { local portal_root="${1:-${PORTAL_DIR}}" if $SKIP_OPENCLAW || ! command -v node &>/dev/null; then return 0 fi local helper="${portal_root}/backend/dist/services/openclawConfigManager.js" if [[ ! -f "${helper}" ]]; then warn "Portal OpenClaw Codex auth bridge helper is missing. Skipping Codex auth bridge." return 0 fi local portal_service_codex_home="" local portal_env_file="${portal_root}/backend/.env.production" if [[ -f "${portal_env_file}" ]]; then portal_service_codex_home="$( read_env_value "${portal_env_file}" CODEX_HOME 2>/dev/null || true )" fi # The Portal service EnvironmentFile is the runtime authority. Never borrow # a transient installer-shell CODEX_HOME: an update launched from an agent # can inherit that agent's private Codex home, which the Portal service will # not use after restart. local portal_codex_home="${portal_service_codex_home:-${HOME}/.codex}" if [[ -n "${CODEX_HOME:-}" && "${CODEX_HOME}" != "${portal_codex_home}" ]]; then warn "Ignoring transient installer CODEX_HOME; using the Portal service credential path" fi local -x CODEX_HOME="${portal_codex_home}" if [[ ! -f "${portal_codex_home}/auth.json" ]]; then ok "OpenClaw Codex auth bridge skipped (no external Codex CLI auth found)" return 0 fi if ! spin "Bridging Codex CLI auth into OpenClaw" "OPENCLAW_ALLOW_ROOT=1 NODE_PATH='${PORTAL_DIR}/backend/node_modules' node -e 'const helper = process.argv[1]; const mod = require(helper); const result = mod.pinCodexExternalCliAuthProfile(); console.log(JSON.stringify(result));' '${helper}'"; then warn "OpenClaw Codex auth bridge failed. Continuing update; re-run Codex setup in Settings if needed." return 0 fi ok "OpenClaw Codex auth bridge checked" } openclaw_codex_plugin_package_dir() { local source_path="$1" candidate package_name depth [[ -n "${source_path}" ]] || return 1 source_path="${source_path/#\~/${HOME}}" if [[ -d "${source_path}" ]]; then candidate="${source_path}" else candidate="$(dirname "${source_path}")" fi for depth in 1 2 3 4 5 6; do package_name="$(node_package_name_from_dir "${candidate}" || true)" if [[ "${package_name}" == "@openclaw/codex" ]]; then printf '%s\n' "${candidate}" return 0 fi [[ "${candidate}" != "/" ]] || break candidate="$(dirname "${candidate}")" done return 1 } openclaw_codex_plugin_details() { local inspect_json list_json presence inspect_json="$(OPENCLAW_ALLOW_ROOT=1 openclaw plugins inspect codex --json 2>>"${LOG_FILE}" || true)" if [[ -n "${inspect_json}" ]] && printf '%s' "${inspect_json}" | node -e ' let raw = ""; process.stdin.on("data", chunk => raw += chunk); process.stdin.on("end", () => { try { const row = JSON.parse(raw); const plugin = row && typeof row === "object" ? row.plugin || {} : {}; const install = row && typeof row === "object" ? row.install || {} : {}; if (plugin.id !== "codex") process.exit(1); for (const value of [ plugin.version, plugin.source, plugin.origin, install.source, install.spec, install.installPath, install.version, ]) console.log(typeof value === "string" ? value : ""); } catch (_) { process.exit(1); } }); '; then return 0 fi # `plugins inspect` exits non-zero when codex is absent. Prove absence from # the machine-readable registry rather than treating every CLI failure as an # empty baseline that is safe to overwrite. list_json="$(OPENCLAW_ALLOW_ROOT=1 openclaw plugins list --json 2>>"${LOG_FILE}" || true)" presence="$(printf '%s' "${list_json}" | node -e ' let raw = ""; process.stdin.on("data", chunk => raw += chunk); process.stdin.on("end", () => { try { const row = JSON.parse(raw); const plugins = Array.isArray(row?.plugins) ? row.plugins : null; if (!plugins) process.exit(1); process.stdout.write(plugins.some(plugin => plugin?.id === "codex") ? "present" : "absent"); } catch (_) { process.exit(1); } }); ' 2>/dev/null || true)" [[ "${presence}" == "absent" ]] && return 2 return 1 } openclaw_codex_plugin_runtime_details() { local inspect_json list_json presence inspect_json="$(OPENCLAW_ALLOW_ROOT=1 openclaw plugins inspect codex --json --runtime 2>>"${LOG_FILE}" || true)" if [[ -n "${inspect_json}" ]] && printf '%s' "${inspect_json}" | node -e ' let raw = ""; process.stdin.on("data", chunk => raw += chunk); process.stdin.on("end", () => { try { const row = JSON.parse(raw); const plugin = row && typeof row === "object" ? row.plugin || {} : {}; const install = row && typeof row === "object" ? row.install || {} : {}; if ( plugin.id !== "codex" || plugin.status !== "loaded" || plugin.enabled !== true || plugin.activated !== true || !Array.isArray(plugin.agentHarnessIds) || !plugin.agentHarnessIds.includes("codex") ) process.exit(1); for (const value of [ plugin.version, plugin.source, plugin.origin, install.source, install.spec, install.installPath, install.version, plugin.rootDir, ]) console.log(typeof value === "string" ? value : ""); } catch (_) { process.exit(1); } }); '; then return 0 fi list_json="$(OPENCLAW_ALLOW_ROOT=1 openclaw plugins list --json 2>>"${LOG_FILE}" || true)" presence="$(printf '%s' "${list_json}" | node -e ' let raw = ""; process.stdin.on("data", chunk => raw += chunk); process.stdin.on("end", () => { try { const row = JSON.parse(raw); const plugins = Array.isArray(row?.plugins) ? row.plugins : null; if (!plugins) process.exit(1); process.stdout.write(plugins.some(plugin => plugin?.id === "codex") ? "present" : "absent"); } catch (_) { process.exit(1); } }); ' 2>/dev/null || true)" [[ "${presence}" == "absent" ]] && return 2 return 1 } openclaw_codex_plugin_catalog_package_dir() { local details="${1:-}" local package_dir package_real source_path local -a fields=() [[ -n "${details}" ]] || details="$(openclaw_codex_plugin_details)" || return 1 mapfile -t fields <<< "${details}" source_path="${fields[1]:-${fields[5]:-}}" package_dir="$(openclaw_codex_plugin_package_dir "${source_path}" || true)" package_real="$(readlink -f -- "${package_dir}" 2>/dev/null || true)" [[ -n "${package_real}" && -d "${package_real}" ]] || return 1 printf '%s\n' "${package_real}" } openclaw_codex_plugin_active_package_dir() { local details="${1:-}" local source_package_dir root_package_dir local source_real root_real local -a fields=() [[ -n "${details}" ]] || details="$(openclaw_codex_plugin_runtime_details)" || return 1 mapfile -t fields <<< "${details}" source_package_dir="$(openclaw_codex_plugin_package_dir \ "${fields[1]:-}" || true)" root_package_dir="$(openclaw_codex_plugin_package_dir \ "${fields[7]:-}" || true)" [[ -n "${source_package_dir}" \ && -n "${root_package_dir}" ]] || return 1 source_real="$(readlink -f -- "${source_package_dir}" 2>/dev/null || true)" root_real="$(readlink -f -- "${root_package_dir}" 2>/dev/null || true)" [[ -n "${source_real}" \ && "${source_real}" == "${root_real}" \ && -d "${source_real}" ]] || return 1 printf '%s\n' "${source_real}" } openclaw_codex_plugin_attested_package_dir() { local details="${1:-}" local active_package_dir install_package_dir local active_real install_real local -a fields=() [[ -n "${details}" ]] || details="$(openclaw_codex_plugin_runtime_details)" || return 1 mapfile -t fields <<< "${details}" active_package_dir="$(openclaw_codex_plugin_active_package_dir \ "${details}" || true)" install_package_dir="$(openclaw_codex_plugin_package_dir \ "${fields[5]:-}" || true)" [[ -n "${active_package_dir}" \ && -n "${install_package_dir}" ]] || return 1 active_real="$(readlink -f -- "${active_package_dir}" 2>/dev/null || true)" install_real="$(readlink -f -- "${install_package_dir}" 2>/dev/null || true)" [[ -n "${active_real}" \ && "${active_real}" == "${install_real}" \ && -d "${active_real}" ]] || return 1 printf '%s\n' "${active_real}" } verify_openclaw_codex_plugin_pin() { local expected_version="${1:-${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}}" local details package_dir package_name package_version local -a fields=() if ! details="$(openclaw_codex_plugin_runtime_details)"; then return 1 fi mapfile -t fields <<< "${details}" local plugin_version="${fields[0]:-}" local plugin_source="${fields[1]:-}" local install_source="${fields[3]:-}" local install_spec="${fields[4]:-}" local recorded_version="${fields[6]:-}" package_dir="$(openclaw_codex_plugin_attested_package_dir \ "${details}" || true)" package_name="$(node_package_name_from_dir "${package_dir}" || true)" package_version="$(node_package_version_from_dir "${package_dir}" || true)" # Registries upgraded from 3.26-era installs keep their legacy bare # "@openclaw/codex" spec even after an exact pinned reinstall; the exact # version is still enforced through plugin, recorded, and package versions. [[ "${plugin_version}" == "${expected_version}" \ && "${install_source}" == "npm" \ && ( "${install_spec}" == "@openclaw/codex@${expected_version}" \ || "${install_spec}" == "@openclaw/codex" ) \ && "${recorded_version}" == "${expected_version}" \ && "${package_name}" == "@openclaw/codex" \ && "${package_version}" == "${expected_version}" \ && -n "${plugin_source}" \ && "${plugin_source}" != *"/.openclaw/npm/node_modules/@openclaw/codex/"* \ && "${plugin_source}" != "~/.openclaw/npm/node_modules/@openclaw/codex/"* ]] } resolve_openclaw_codex_plugin_pending_input_hotfix_target() { local details package_dir package_name package_version target verify_openclaw_codex_plugin_pin "${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" \ || return 1 details="$(openclaw_codex_plugin_runtime_details)" || return 1 package_dir="$(openclaw_codex_plugin_attested_package_dir \ "${details}" || true)" package_name="$(node_package_name_from_dir "${package_dir}" || true)" package_version="$(node_package_version_from_dir "${package_dir}" || true)" [[ "${package_name}" == "@openclaw/codex" \ && "${package_version}" == "${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" \ && -d "${package_dir}/dist" ]] || return 1 target="$(resolve_openclaw_pending_input_hotfix_target \ "${package_dir}/dist" || true)" [[ -n "${target}" ]] || return 1 printf '%s\n' "${target}" } openclaw_codex_plugin_pending_input_hotfix_is_applied() { local target="$1" python3 - "${target}" <<'PY' from pathlib import Path import sys marker = 'const BRIDGESLLM_PENDING_INPUT_HOTFIX_MARKER = "bridgesllm-openclaw-pending-input-v1";' symbol = 'Symbol.for("bridgesllm.openclaw.pending-input.codex-plugin.v1")' try: text = Path(sys.argv[1]).read_text(encoding="utf-8") except (OSError, UnicodeError): raise SystemExit(1) raise SystemExit(0 if text.count(marker) == 1 and text.count(symbol) == 1 else 1) PY } manage_openclaw_codex_plugin_pending_input_backup() { local target="$1" local hotfix_script="$2" local action="$3" local backup="${target}.bridgesllm-pending-input-v1.bak" python3 - \ "${target}" \ "${backup}" \ "${hotfix_script}" \ "${action}" \ "${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" <<'PY' import os from pathlib import Path import stat import subprocess import sys import tempfile target = Path(sys.argv[1]) backup = Path(sys.argv[2]) hotfix_script = Path(sys.argv[3]) action = sys.argv[4] expected_version = sys.argv[5] marker = 'const BRIDGESLLM_PENDING_INPUT_HOTFIX_MARKER = "bridgesllm-openclaw-pending-input-v1";' symbol = 'Symbol.for("bridgesllm.openclaw.pending-input.codex-plugin.v1")' if action not in {"normalize", "retire"}: raise SystemExit(1) if backup != target.with_name(target.name + ".bridgesllm-pending-input-v1.bak"): raise SystemExit(1) for path in (target, backup, hotfix_script): try: metadata = os.lstat(path) except OSError: raise SystemExit(1) if ( not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 ): raise SystemExit(1) try: package_path = target.parent.parent / "package.json" package_metadata = os.lstat(package_path) if ( not stat.S_ISREG(package_metadata.st_mode) or stat.S_ISLNK(package_metadata.st_mode) or package_metadata.st_nlink != 1 ): raise OSError package_bytes = package_path.read_bytes() import json package = json.loads(package_bytes.decode("utf-8")) if ( package.get("name") != "@openclaw/codex" or package.get("version") != expected_version ): raise OSError target_bytes = target.read_bytes() backup_bytes = backup.read_bytes() target_text = target.read_text(encoding="utf-8") backup_text = backup.read_text(encoding="utf-8") except (OSError, UnicodeError, ValueError): raise SystemExit(1) if ( marker in backup_text or symbol in backup_text ): raise SystemExit(1) # The recovery artifact is trusted only if the exact bundled patcher accepts # it as the pristine pinned provider source and deterministically reproduces # the current patched bytes. Syntax alone is not provenance. with tempfile.TemporaryDirectory(prefix="bridgesllm-codex-pending-proof-") as temporary: package_root = Path(temporary) / "package" dist = package_root / "dist" dist.mkdir(parents=True) (package_root / "package.json").write_bytes(package_bytes) fixture = dist / target.name fixture.write_bytes(backup_bytes) environment = os.environ.copy() environment.update({ "PORTAL_OPENCLAW_PENDING_INPUT_STRICT": "1", "PORTAL_REQUIRED_OPENCLAW_PACKAGE_NAME": "@openclaw/codex", "PORTAL_REQUIRED_OPENCLAW_PACKAGE_VERSION": expected_version, }) proof = subprocess.run( ["bash", str(hotfix_script), str(dist)], env=environment, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ) if proof.returncode != 0: raise SystemExit(1) reproduced_bytes = fixture.read_bytes() target_is_patched = ( target_text.count(marker) == 1 and target_text.count(symbol) == 1 and reproduced_bytes == target_bytes ) target_is_pristine = ( marker not in target_text and symbol not in target_text and target_bytes == backup_bytes ) if action == "retire": if not target_is_patched: raise SystemExit(1) backup.unlink() elif target_is_patched: # Interrupted after publishing the exact patched bytes: restore the proven # original before capturing a fresh package-level rollback baseline. os.replace(backup, target) elif target_is_pristine: # Interrupted before publication. The target already is the proven # original, so remove only the byte-identical redundant artifact. backup.unlink() else: raise SystemExit(1) directory_fd = os.open(target.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY } normalize_openclaw_codex_plugin_pending_input_recovery_artifact() { manage_openclaw_codex_plugin_pending_input_backup \ "$1" "$2" normalize } retire_openclaw_codex_plugin_pending_input_backup() { manage_openclaw_codex_plugin_pending_input_backup \ "$1" "$2" retire } verify_openclaw_codex_plugin_pending_input_hotfix() { if $SKIP_OPENCLAW || ! command -v openclaw &>/dev/null; then return 0 fi local target backup target="$(resolve_openclaw_codex_plugin_pending_input_hotfix_target || true)" [[ -n "${target}" ]] || return 1 backup="${target}.bridgesllm-pending-input-v1.bak" openclaw_codex_plugin_pending_input_hotfix_is_applied "${target}" \ && [[ ! -e "${backup}" && ! -L "${backup}" ]] \ && node --check "${target}" >/dev/null 2>&1 } stage_openclaw_codex_plugin_rollback_package() { local expected_version="$1" source_path="$2" local package_dir backup_dir packed_name packed_path packed_version package_dir="$(openclaw_codex_plugin_package_dir "${source_path}" || true)" [[ -n "${package_dir}" && -f "${package_dir}/package.json" ]] || return 1 backup_dir="${UPDATE_RECOVERY_BACKUP_DIR:-/tmp}/openclaw-codex-plugin" mkdir -p "${backup_dir}" packed_name="$(cd "${backup_dir}" && npm pack --ignore-scripts --silent "${package_dir}" 2>> "$LOG_FILE" | tail -1)" packed_path="${backup_dir}/${packed_name}" [[ -n "${packed_name}" && -f "${packed_path}" ]] || return 1 packed_version="$(tar -xOf "${packed_path}" package/package.json 2>/dev/null \ | node -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { console.log(JSON.parse(s).version || ""); } catch {} });' \ | head -1)" if [[ "${packed_version}" != "${expected_version}" ]]; then rm -f "${packed_path}" return 1 fi chmod 600 "${packed_path}" 2>/dev/null || true OPENCLAW_CODEX_PLUGIN_ROLLBACK_TARBALL="${packed_path}" } capture_openclaw_codex_plugin_baseline() { $OPENCLAW_CODEX_PLUGIN_BASELINE_CAPTURED && return 0 local details status package_source local -a fields=() if details="$(openclaw_codex_plugin_details)"; then mapfile -t fields <<< "${details}" local plugin_version="${fields[0]:-}" [[ -n "${plugin_version}" ]] || return 1 # This capture runs before old OpenClaw cores are replaced, so it may not # depend on the new runtime-inspection schema. Prefer the catalog's loaded # source over a stale installPath and prove the packed package/version. package_source="$(openclaw_codex_plugin_catalog_package_dir \ "${details}" || true)" [[ -n "${package_source}" ]] || return 1 stage_openclaw_codex_plugin_rollback_package "${plugin_version}" "${package_source}" || return 1 OPENCLAW_CODEX_PLUGIN_PREEXISTED=true OPENCLAW_CODEX_PLUGIN_PREUPDATE_VERSION="${plugin_version}" else status=$? [[ ${status} -eq 2 ]] || return 1 OPENCLAW_CODEX_PLUGIN_PREEXISTED=false OPENCLAW_CODEX_PLUGIN_PREUPDATE_VERSION="" fi OPENCLAW_CODEX_PLUGIN_BASELINE_CAPTURED=true } rollback_openclaw_codex_plugin() { local defer_gateway_restart="${1:-false}" local rollback_ok=true details status package_dir package_version local -a fields=() if $OPENCLAW_CODEX_PLUGIN_PREEXISTED; then if [[ -z "${OPENCLAW_CODEX_PLUGIN_ROLLBACK_TARBALL:-}" \ || ! -f "${OPENCLAW_CODEX_PLUGIN_ROLLBACK_TARBALL}" ]] \ || ! OPENCLAW_ALLOW_ROOT=1 openclaw plugins install "${OPENCLAW_CODEX_PLUGIN_ROLLBACK_TARBALL}" --force --pin >> "$LOG_FILE" 2>&1; then rollback_ok=false else if details="$(openclaw_codex_plugin_details)"; then mapfile -t fields <<< "${details}" package_dir="$(openclaw_codex_plugin_catalog_package_dir \ "${details}" || true)" package_version="$(node_package_version_from_dir "${package_dir}" || true)" [[ "${fields[0]:-}" == "${OPENCLAW_CODEX_PLUGIN_PREUPDATE_VERSION}" \ && "${package_version}" == "${OPENCLAW_CODEX_PLUGIN_PREUPDATE_VERSION}" ]] || rollback_ok=false else rollback_ok=false fi fi else if details="$(openclaw_codex_plugin_details)"; then OPENCLAW_ALLOW_ROOT=1 openclaw plugins uninstall codex --force >> "$LOG_FILE" 2>&1 || rollback_ok=false else status=$? [[ ${status} -eq 2 ]] || rollback_ok=false fi fi if [[ "${defer_gateway_restart}" != "true" ]] \ && systemctl is-enabled openclaw-gateway >/dev/null 2>&1; then systemctl restart openclaw-gateway >> "$LOG_FILE" 2>&1 || rollback_ok=false verify_openclaw_gateway_stable "${PIN_OPENCLAW_RUNTIME_VERSION}" 18 || rollback_ok=false fi $rollback_ok && OPENCLAW_CODEX_PLUGIN_UPDATE_ATTEMPTED=false $rollback_ok } rollback_openclaw_tested_pair() { local core_rollback_pending=false local hotfix_rollback_pending=false local claude_ask_user_rollback_pending=false local ask_user_transaction_rollback_pending=false local core_rollback_ok=true local hotfix_rollback_ok=true local claude_ask_user_rollback_ok=true local ask_user_transaction_rollback_ok=true local plugin_rollback_ok=true local defer_plugin_restart=false local plugin_retry_restarted=false if ! $OPENCLAW_UPGRADE_COMMITTED \ && { $OPENCLAW_PACKAGE_UPDATE_ATTEMPTED \ || $OPENCLAW_PACKAGE_UPDATED \ || [[ -n "${OPENCLAW_UPGRADE_STATE_MANIFEST:-}" ]]; }; then core_rollback_pending=true fi if $OPENCLAW_PENDING_INPUT_HOTFIX_APPLIED \ && ! $OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED; then hotfix_rollback_pending=true fi if $OPENCLAW_CLAUDE_ASK_USER_HOTFIX_APPLIED \ && ! $OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED; then claude_ask_user_rollback_pending=true fi if $OPENCLAW_ASK_USER_TRANSACTION_ARMED \ && ! $OPENCLAW_ASK_USER_TRANSACTION_COMMITTED; then ask_user_transaction_rollback_pending=true fi if $core_rollback_pending || $hotfix_rollback_pending \ || $claude_ask_user_rollback_pending \ || $ask_user_transaction_rollback_pending; then defer_plugin_restart=true fi # Keep the replacement ask-user bridge and its durable rollback journal in # place until native Claude prompting is restored. If rollback itself loses # power, that ordering guarantees at least one working question path rather # than deleting the replacement while native AskUserQuestion is suppressed. if $OPENCLAW_CODEX_PLUGIN_UPDATE_ATTEMPTED; then rollback_openclaw_codex_plugin "${defer_plugin_restart}" || plugin_rollback_ok=false fi # The native pending-input patch has its own byte-exact recovery artifact. # Restore it before any package-level rollback so every later preparation # failure removes the Portal mutation even when the pinned core preexisted. if $hotfix_rollback_pending; then rollback_openclaw_pending_input_hotfix || hotfix_rollback_ok=false fi if $claude_ask_user_rollback_pending; then rollback_openclaw_claude_ask_user_hotfix \ || claude_ask_user_rollback_ok=false fi if $core_rollback_pending; then # Do not boot the restored core until the matching plugin/config baseline # is back. One final restart below crosses the complete rollback boundary. rollback_openclaw_package_update true || core_rollback_ok=false fi # If the new-core CLI could not restore the old plugin, retry after the old # core is back. This makes recovery resilient to cross-revision plugin CLI # incompatibility while preserving the original rollback artifacts. if $OPENCLAW_CODEX_PLUGIN_UPDATE_ATTEMPTED; then if rollback_openclaw_codex_plugin "${ask_user_transaction_rollback_pending}"; then plugin_rollback_ok=true if ! $ask_user_transaction_rollback_pending; then plugin_retry_restarted=true fi else plugin_rollback_ok=false fi fi # Native/core bytes are safe now. Restore the previous bridge/config while # preserving the journal until that restoration and the final gateway boot # both succeed. The helper itself is idempotent, so a later recovery can # repeat this exact step after interruption. if $ask_user_transaction_rollback_pending; then rollback_bridgesllm_ask_user_transaction true \ || ask_user_transaction_rollback_ok=false fi # A package rollback performs its own final restart. When only the exact # bundle bytes changed, restart an already-running gateway once after those # bytes and any plugin baseline are back. Never start a gateway that was # intentionally stopped. if { $core_rollback_pending || $hotfix_rollback_pending \ || $claude_ask_user_rollback_pending \ || $ask_user_transaction_rollback_pending; } \ && $hotfix_rollback_ok && $claude_ask_user_rollback_ok \ && $ask_user_transaction_rollback_ok \ && ! $plugin_retry_restarted \ && { $OPENCLAW_GATEWAY_WAS_ACTIVE \ || systemctl is-active --quiet openclaw-gateway >/dev/null 2>&1; }; then local rollback_gateway_version="${PIN_OPENCLAW_RUNTIME_VERSION}" $core_rollback_pending \ && rollback_gateway_version="${OPENCLAW_PREUPDATE_RUNTIME_VERSION:-}" if [[ -z "${rollback_gateway_version}" ]] \ || ! systemctl restart openclaw-gateway >> "$LOG_FILE" 2>&1 \ || ! verify_openclaw_gateway_stable "${rollback_gateway_version}" 18; then hotfix_rollback_ok=false claude_ask_user_rollback_ok=false ask_user_transaction_rollback_ok=false fi fi $core_rollback_ok && $hotfix_rollback_ok \ && $claude_ask_user_rollback_ok && $ask_user_transaction_rollback_ok \ && $plugin_rollback_ok } verify_openclaw_tested_pair() { # The gateway may still be settling from the plugin-load restart, and the # deep status probe has no retry of its own, so a single silent flake here # must not unwind an otherwise verified core/plugin migration. Retry the # runtime-facing probes over a bounded window and log what was observed. local attempt observed_gateway="" openclaw_package_dir="" local claude_ask_user_target="" verify_openclaw_core_package_pin || return 1 openclaw_package_dir="$(openclaw_core_package_dir || true)" claude_ask_user_target="$( resolve_openclaw_claude_ask_user_hotfix_target \ "${openclaw_package_dir}/dist" || true )" [[ -n "${claude_ask_user_target}" ]] || return 1 openclaw_claude_ask_user_hotfix_is_applied \ "${claude_ask_user_target}" || return 1 for attempt in 1 2 3 4 5 6; do observed_gateway="$(openclaw_gateway_version || true)" if [[ "${observed_gateway}" == "${PIN_OPENCLAW_RUNTIME_VERSION}" ]] \ && OPENCLAW_ALLOW_ROOT=1 openclaw gateway status --require-rpc --timeout 10000 >> "$LOG_FILE" 2>&1 \ && verify_openclaw_codex_plugin_pin "${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" \ && verify_openclaw_codex_plugin_pending_input_hotfix \ && verify_bridgesllm_ask_user_tested_pair; then return 0 fi echo "tested-pair verify attempt ${attempt}: gateway version '${observed_gateway}' (expected '${PIN_OPENCLAW_RUNTIME_VERSION}')" >> "$LOG_FILE" sleep 5 done return 1 } verify_bridgesllm_ask_user_plugin_config() { local config_path="$1" python3 - "${config_path}" <<'PY' import json import os import pathlib import stat import sys path = pathlib.Path(sys.argv[1]) try: metadata = os.lstat(path) value = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) if ( not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != 0 or metadata.st_gid != 0 or metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) != 0o600 or not isinstance(value, dict) ): raise SystemExit(1) plugins = value.get("plugins") entries = plugins.get("entries") if isinstance(plugins, dict) else None entry = entries.get("bridgesllm-ask-user") if isinstance(entries, dict) else None if not isinstance(entry, dict) or entry.get("enabled") is not True: raise SystemExit(1) allow = plugins.get("allow") if allow is not None and ( not isinstance(allow, list) or "bridgesllm-ask-user" not in allow ): raise SystemExit(1) agents = value.get("agents") defaults = agents.get("defaults") if isinstance(agents, dict) else None cli_backends = defaults.get("cliBackends") if isinstance(defaults, dict) else None claude_backend = cli_backends.get("claude-cli") if isinstance(cli_backends, dict) else None claude_env = claude_backend.get("env") if isinstance(claude_backend, dict) else None if ( not isinstance(claude_backend, dict) or not isinstance(claude_backend.get("command"), str) or not claude_backend["command"].strip() or not isinstance(claude_env, dict) or claude_env.get("MCP_TOOL_TIMEOUT") != "660000" or claude_env.get("CLAUDE_CODE_MCP_TOOL_IDLE_TIMEOUT") != "660000" ): raise SystemExit(1) PY } verify_bridgesllm_ask_user_plugin_runtime() { local target_dir="$1" local output_path="$2" if ! OPENCLAW_ALLOW_ROOT=1 openclaw plugins inspect \ bridgesllm-ask-user --json --runtime > "${output_path}" \ 2>> "${LOG_FILE}"; then return 1 fi python3 - \ "${output_path}" \ "${target_dir}" \ "${PIN_BRIDGESLLM_ASK_USER_PLUGIN_VERSION}" <<'PY' import json import os import pathlib import sys report_path = pathlib.Path(sys.argv[1]) target = pathlib.Path(sys.argv[2]) expected_version = sys.argv[3] try: report = json.loads(report_path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) plugin = report.get("plugin") if isinstance(report, dict) else None diagnostics = report.get("diagnostics") if isinstance(report, dict) else None methods = report.get("gatewayMethods") if isinstance(report, dict) else None tool_names = plugin.get("toolNames") if isinstance(plugin, dict) else None typed_hooks = report.get("typedHooks") if isinstance(report, dict) else None if ( not isinstance(plugin, dict) or plugin.get("id") != "bridgesllm-ask-user" or plugin.get("version") != expected_version or plugin.get("status") != "loaded" or plugin.get("enabled") is not True or plugin.get("activated") is not True or plugin.get("error") or os.path.realpath(plugin.get("rootDir", "")) != os.path.realpath(target) or os.path.realpath(plugin.get("source", "")) != os.path.realpath(target / "index.js") or not isinstance(tool_names, list) or "ask_user_question" not in tool_names or plugin.get("hookCount") != 1 or not isinstance(typed_hooks, list) or not any( isinstance(item, dict) and item.get("name") == "before_tool_call" for item in typed_hooks ) or not isinstance(methods, list) or not { "bridgesllm.ask_user.probe", "bridgesllm.ask_user.pending", "bridgesllm.ask_user.answer", "bridgesllm.ask_user.dismiss", "bridgesllm.ask_user.steer", }.issubset(set(methods)) or not isinstance(diagnostics, list) or any( isinstance(item, dict) and item.get("level") == "error" for item in diagnostics ) ): raise SystemExit(1) PY } verify_bridgesllm_ask_user_gateway_method() { local output_path="$1" local mode="${2:-full}" local nonce session_key run_id request_id method params probe_kind case "${mode}" in full|bootstrap) ;; *) return 1 ;; esac nonce="$(rand_hex 12)" || return 1 session_key="agent:main:bridgesllm-install-probe-${nonce}" run_id="bridgesllm-install-probe-${nonce}" request_id="bridgesllm-install-request-${nonce}" for probe_kind in probe pending answer dismiss steer; do case "${probe_kind}" in probe) method="bridgesllm.ask_user.probe" params="{\"nonce\":\"${nonce}\"}" ;; pending) method="bridgesllm.ask_user.pending" params="{\"sessionKey\":\"${session_key}\",\"expectedRunId\":\"${run_id}\"}" ;; answer) method="bridgesllm.ask_user.answer" params="{\"sessionKey\":\"${session_key}\",\"expectedRunId\":\"${run_id}\",\"requestId\":\"${request_id}\",\"text\":\"BridgesLLM readiness probe.\"}" ;; dismiss) method="bridgesllm.ask_user.dismiss" params="{\"sessionKey\":\"${session_key}\",\"expectedRunId\":\"${run_id}\",\"requestId\":\"${request_id}\"}" ;; steer) method="bridgesllm.ask_user.steer" params="{\"sessionKey\":\"${session_key}\",\"expectedRunId\":\"${run_id}\",\"requestId\":\"${request_id}\",\"text\":\"BridgesLLM readiness probe.\"}" ;; esac if ! OPENCLAW_ALLOW_ROOT=1 openclaw gateway call \ "${method}" \ --json \ --timeout 10000 \ --params "${params}" > "${output_path}" 2>> "${LOG_FILE}"; then return 1 fi if ! python3 - \ "${output_path}" "${probe_kind}" "${request_id}" "${mode}" <<'PY' import json import pathlib import sys try: response = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) kind = sys.argv[2] request_id = sys.argv[3] mode = sys.argv[4] if mode not in {"full", "bootstrap"}: raise SystemExit(1) if not isinstance(response, dict): raise SystemExit(1) if kind == "probe": full_valid = ( response.get("ok") is True and response.get("code") == "SEMANTIC_PROBE_OK" and response.get("toolName") == "ask_user_question" and response.get("answer") is True and response.get("dismiss") is True and response.get("steer") is True and response.get("activeRunSteer") is True ) bootstrap_valid = ( response.get("ok") is False and response.get("code") == "SEMANTIC_PROBE_FAILED" and response.get("toolName") == "ask_user_question" and response.get("answer") is True and response.get("dismiss") is True and response.get("steer") is True and response.get("activeRunSteer") is False ) valid = full_valid or (mode == "bootstrap" and bootstrap_valid) elif kind == "pending": valid = ( response.get("pending") is False and response.get("code") == "NO_ACTIVE_RUN" ) else: valid = ( response.get("accepted") is False and response.get("code") == "NO_ACTIVE_RUN" and response.get("requestId") == request_id ) if not valid: raise SystemExit(1) PY then return 1 fi done } verify_bridgesllm_ask_user_tested_pair() { local openclaw_state_dir="${1:-/root/.openclaw}" local mode="${2:-full}" local target_dir="${openclaw_state_dir}/extensions/bridgesllm-ask-user" local config_path="${openclaw_state_dir}/openclaw.json" local output_path status=0 output_path="$(mktemp /tmp/bridgesllm-ask-user-verify.XXXXXX)" \ || return 1 chmod 600 "${output_path}" 2>/dev/null || true verify_bridgesllm_ask_user_plugin_config "${config_path}" \ && verify_bridgesllm_ask_user_plugin_runtime \ "${target_dir}" "${output_path}" \ && verify_bridgesllm_ask_user_gateway_method "${output_path}" "${mode}" \ || status=$? rm -f -- "${output_path}" return "${status}" } write_bridgesllm_ask_user_transaction_manifest() { local transaction_dir="$1" openclaw_state_dir="$2" target_dir="$3" local config_path="$4" target_preexisted="$5" config_preexisted="$6" local gateway_was_active="$7" python3 - \ "${transaction_dir}" \ "${openclaw_state_dir}" \ "${target_dir}" \ "${config_path}" \ "${target_preexisted}" \ "${config_preexisted}" \ "${gateway_was_active}" <<'PY' import hashlib import json import os from pathlib import Path import stat import sys import tempfile transaction = Path(sys.argv[1]).absolute() state_root = Path(sys.argv[2]).absolute() target = Path(sys.argv[3]).absolute() config = Path(sys.argv[4]).absolute() target_preexisted = sys.argv[5] == "true" config_preexisted = sys.argv[6] == "true" gateway_was_active = sys.argv[7] == "true" manifest = transaction / "manifest.json" previous_config = transaction / "previous-openclaw.json" def safe_directory(path): metadata = os.lstat(path) return ( stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_gid == 0 ) def safe_file(path): metadata = os.lstat(path) return ( stat.S_ISREG(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_nlink == 1 and metadata.st_uid == 0 and metadata.st_gid == 0 ) def tree_digest(root): digest = hashlib.sha256() for path in [root, *sorted(root.rglob("*"), key=lambda item: str(item.relative_to(root)))]: metadata = os.lstat(path) relative = "." if path == root else str(path.relative_to(root)) digest.update(relative.encode("utf-8") + b"\0") if stat.S_ISDIR(metadata.st_mode): digest.update(b"d\0") elif stat.S_ISREG(metadata.st_mode): digest.update(b"f\0" + path.read_bytes()) elif stat.S_ISLNK(metadata.st_mode): digest.update(b"l\0" + os.readlink(path).encode("utf-8")) else: raise OSError(f"unsupported plugin entry: {path}") return digest.hexdigest() try: if ( transaction != state_root / ".bridgesllm-ask-user-tested-pair-v1" or target != state_root / "extensions" / "bridgesllm-ask-user" or config != state_root / "openclaw.json" or not safe_directory(state_root) or not safe_directory(transaction) or not safe_directory(target.parent) or os.path.lexists(manifest) ): raise OSError("unsafe ask-user transaction paths") if target_preexisted and not safe_directory(target): raise OSError("unsafe previous ask-user plugin") if config_preexisted and not safe_file(previous_config): raise OSError("unsafe previous OpenClaw config") value = { "schema": "bridgesllm-ask-user-tested-pair-v1", "stateRoot": str(state_root), "targetDir": str(target), "configPath": str(config), "targetPreexisted": target_preexisted, "configPreexisted": config_preexisted, "gatewayWasActive": gateway_was_active, "previousPluginSha256": tree_digest(target) if target_preexisted else None, "previousConfigSha256": ( hashlib.sha256(previous_config.read_bytes()).hexdigest() if config_preexisted else None ), } except OSError: raise SystemExit(1) temporary_name = "" try: with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", dir=transaction, prefix=".manifest-", suffix=".tmp", delete=False, ) as stream: temporary_name = stream.name json.dump(value, stream, sort_keys=True, separators=(",", ":")) stream.write("\n") stream.flush() os.fsync(stream.fileno()) os.chmod(temporary_name, 0o600) os.replace(temporary_name, manifest) temporary_name = "" directory_fd = os.open(transaction, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary_name: try: Path(temporary_name).unlink() except FileNotFoundError: pass PY } rollback_bridgesllm_ask_user_transaction() { $OPENCLAW_ASK_USER_TRANSACTION_ARMED || return 0 $OPENCLAW_ASK_USER_TRANSACTION_COMMITTED && return 0 local defer_gateway_restart="${1:-false}" local transaction_dir="${OPENCLAW_ASK_USER_TRANSACTION_DIR:-}" local gateway_was_active="" [[ -n "${transaction_dir}" ]] || return 1 if ! gateway_was_active="$(python3 - "${transaction_dir}" <<'PY' import hashlib import json import os from pathlib import Path import shutil import stat import sys import tempfile transaction = Path(sys.argv[1]).absolute() manifest_path = transaction / "manifest.json" previous_plugin = transaction / "previous-plugin" previous_config = transaction / "previous-openclaw.json" failed_plugin = transaction / "failed-plugin" failed_config = transaction / "failed-openclaw.json" def safe_directory(path): metadata = os.lstat(path) return ( stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_gid == 0 ) def safe_file(path): metadata = os.lstat(path) return ( stat.S_ISREG(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode) and metadata.st_nlink == 1 and metadata.st_uid == 0 and metadata.st_gid == 0 ) def tree_digest(root): digest = hashlib.sha256() for path in [root, *sorted(root.rglob("*"), key=lambda item: str(item.relative_to(root)))]: metadata = os.lstat(path) relative = "." if path == root else str(path.relative_to(root)) digest.update(relative.encode("utf-8") + b"\0") if stat.S_ISDIR(metadata.st_mode): digest.update(b"d\0") elif stat.S_ISREG(metadata.st_mode): digest.update(b"f\0" + path.read_bytes()) elif stat.S_ISLNK(metadata.st_mode): digest.update(b"l\0" + os.readlink(path).encode("utf-8")) else: raise OSError(f"unsupported plugin entry: {path}") return digest.hexdigest() try: if not safe_directory(transaction) or not safe_file(manifest_path): raise OSError("unsafe transaction") value = json.loads(manifest_path.read_text(encoding="utf-8")) state_root = Path(value["stateRoot"]).absolute() target = Path(value["targetDir"]).absolute() config = Path(value["configPath"]).absolute() if ( value.get("schema") != "bridgesllm-ask-user-tested-pair-v1" or transaction != state_root / ".bridgesllm-ask-user-tested-pair-v1" or target != state_root / "extensions" / "bridgesllm-ask-user" or config != state_root / "openclaw.json" or not safe_directory(state_root) or not safe_directory(target.parent) ): raise OSError("invalid transaction manifest") if value.get("targetPreexisted") is True: expected_plugin_hash = value.get("previousPluginSha256") if os.path.lexists(previous_plugin): if not safe_directory(previous_plugin) or tree_digest(previous_plugin) != expected_plugin_hash: raise OSError("previous plugin does not match transaction manifest") if os.path.lexists(target): if not safe_directory(target) or os.path.lexists(failed_plugin): raise OSError("unsafe candidate plugin") os.replace(target, failed_plugin) os.replace(previous_plugin, target) elif not safe_directory(target) or tree_digest(target) != expected_plugin_hash: raise OSError("previous plugin is unavailable") elif os.path.lexists(target): if not safe_directory(target) or os.path.lexists(failed_plugin): raise OSError("unsafe fresh candidate plugin") os.replace(target, failed_plugin) if value.get("configPreexisted") is True: expected_config_hash = value.get("previousConfigSha256") if ( not safe_file(previous_config) or hashlib.sha256(previous_config.read_bytes()).hexdigest() != expected_config_hash ): raise OSError("previous config does not match transaction manifest") temporary_name = "" try: with tempfile.NamedTemporaryFile( mode="wb", dir=state_root, prefix=".bridgesllm-ask-user-config-restore-", suffix=".tmp", delete=False, ) as stream: temporary_name = stream.name stream.write(previous_config.read_bytes()) stream.flush() os.fsync(stream.fileno()) shutil.copystat(previous_config, temporary_name, follow_symlinks=True) os.replace(temporary_name, config) temporary_name = "" finally: if temporary_name: try: Path(temporary_name).unlink() except FileNotFoundError: pass elif os.path.lexists(config): if not safe_file(config) or os.path.lexists(failed_config): raise OSError("unsafe fresh config") os.replace(config, failed_config) for directory in {state_root, target.parent}: directory_fd = os.open(directory, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) except (OSError, KeyError, TypeError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) print("true" if value.get("gatewayWasActive") is True else "false") PY )"; then return 1 fi if [[ "${gateway_was_active}" == "true" \ && "${defer_gateway_restart}" != "true" ]]; then systemctl restart openclaw-gateway >> "${LOG_FILE}" 2>&1 || return 1 verify_openclaw_gateway_stable \ "${PIN_OPENCLAW_RUNTIME_VERSION}" 18 || return 1 fi rm -rf -- "${transaction_dir}" || return 1 sync -f "$(dirname "${transaction_dir}")" 2>/dev/null || return 1 OPENCLAW_ASK_USER_TRANSACTION_ARMED=false OPENCLAW_ASK_USER_TRANSACTION_COMMITTED=false OPENCLAW_ASK_USER_TRANSACTION_DIR="" OPENCLAW_ASK_USER_STATE_DIR="" OPENCLAW_ASK_USER_GATEWAY_WAS_ACTIVE=false OPENCLAW_ASK_USER_BASE_ATTESTED=false OPENCLAW_ASK_USER_LIVE_ATTESTED=false } commit_bridgesllm_ask_user_transaction() { local transaction_dir="${OPENCLAW_ASK_USER_TRANSACTION_DIR:-}" local state_root="${OPENCLAW_ASK_USER_STATE_DIR:-}" [[ -n "${transaction_dir}" && -n "${state_root}" ]] || return 1 $OPENCLAW_ASK_USER_TRANSACTION_COMMITTED || return 1 openclaw_tested_pair_commit_record_matches_ask_user_transaction \ "${transaction_dir}" \ "${state_root}/extensions/bridgesllm-ask-user" \ "${state_root}/openclaw.json" || return 1 local legacy_dir="${state_root}/plugins/bridgesllm-ask-user" if [[ -d "${state_root}/plugins" && ! -L "${state_root}/plugins" \ && -d "${legacy_dir}" && ! -L "${legacy_dir}" \ && -f "${legacy_dir}/index.js" && ! -L "${legacy_dir}/index.js" ]] \ && ! mountpoint -q -- "${legacy_dir}"; then rm -rf -- "${legacy_dir}" || return 1 fi rm -rf -- "${transaction_dir}" || return 1 sync -f "${state_root}" 2>/dev/null || return 1 OPENCLAW_ASK_USER_TRANSACTION_ARMED=false OPENCLAW_ASK_USER_TRANSACTION_DIR="" OPENCLAW_ASK_USER_STATE_DIR="" return 0 } recover_or_retire_bridgesllm_ask_user_transaction() { local openclaw_state_dir="$1" local transaction_dir="${openclaw_state_dir}/.bridgesllm-ask-user-tested-pair-v1" [[ -e "${transaction_dir}" || -L "${transaction_dir}" ]] || return 0 if [[ ! -d "${transaction_dir}" || -L "${transaction_dir}" \ || "$(stat -c '%u:%g' -- "${transaction_dir}" 2>/dev/null || true)" != "0:0" ]]; then return 1 fi if [[ ! -e "${transaction_dir}/manifest.json" ]]; then rm -rf -- "${transaction_dir}" || return 1 sync -f "${openclaw_state_dir}" 2>/dev/null || return 1 return 0 fi OPENCLAW_ASK_USER_TRANSACTION_ARMED=true OPENCLAW_ASK_USER_TRANSACTION_COMMITTED=false OPENCLAW_ASK_USER_TRANSACTION_DIR="${transaction_dir}" OPENCLAW_ASK_USER_STATE_DIR="${openclaw_state_dir}" if openclaw_tested_pair_commit_record_matches_ask_user_transaction \ "${transaction_dir}" \ "${openclaw_state_dir}/extensions/bridgesllm-ask-user" \ "${openclaw_state_dir}/openclaw.json"; then OPENCLAW_ASK_USER_TRANSACTION_COMMITTED=true commit_bridgesllm_ask_user_transaction else rollback_bridgesllm_ask_user_transaction fi } install_bridgesllm_ask_user_plugin() { # The ask-user bridge is a required part of the Portal/OpenClaw runtime. It # provides non-Codex providers with a real ask-user tool and registers # gateway methods that settle either that tool or native Codex input only on # the exact attested run. Treat its directory and the shared OpenClaw config # as one local transaction: either the new plugin is loaded and callable, or # both are restored before this function fails. if $SKIP_OPENCLAW; then return 0 fi if ! command -v openclaw &>/dev/null; then warn "OpenClaw CLI is missing; the required ask-question plugin cannot be installed." return 1 fi local source_dir="${1:-${PORTAL_DIR}/installer/openclaw-ask-user-plugin}" local openclaw_state_dir="${2:-/root/.openclaw}" # OpenClaw scans `/extensions`, NOT `/plugins`. The # `plugins` directory holds the install registry, not loadable plugin trees, # so a plugin dropped there is never discovered and the hook never runs. local extensions_dir="${openclaw_state_dir}/extensions" local target_dir="${extensions_dir}/bridgesllm-ask-user" local legacy_dir="${openclaw_state_dir}/plugins/bridgesllm-ask-user" local config_path="${openclaw_state_dir}/openclaw.json" local required_file for required_file in index.js package.json openclaw.plugin.json register.py; do if [[ ! -f "${source_dir}/${required_file}" \ || -L "${source_dir}/${required_file}" ]]; then warn "Ask-question plugin source is missing or unsafe: ${required_file}." return 1 fi done if [[ ! -d "${openclaw_state_dir}" || -L "${openclaw_state_dir}" ]]; then warn "OpenClaw state root is missing or unsafe; refusing plugin mutation." return 1 fi if ! recover_or_retire_bridgesllm_ask_user_transaction \ "${openclaw_state_dir}"; then warn "An interrupted ask-question transaction could not be reconciled safely." return 1 fi if [[ -e "${extensions_dir}" || -L "${extensions_dir}" ]] \ && { [[ ! -d "${extensions_dir}" ]] || [[ -L "${extensions_dir}" ]]; }; then warn "OpenClaw extensions root is not a real directory; refusing plugin mutation." return 1 fi if [[ -e "${config_path}" || -L "${config_path}" ]]; then if [[ ! -f "${config_path}" || -L "${config_path}" \ || "$(stat -c '%u:%g:%h' -- "${config_path}" 2>/dev/null || true)" != "0:0:1" ]]; then warn "OpenClaw config is not a single root-owned regular file; refusing plugin mutation." return 1 fi local config_mode="" config_mode="$(stat -c '%a' -- "${config_path}" 2>/dev/null || true)" if [[ ! "${config_mode}" =~ ^[0-7]{3,4}$ ]] \ || (( (8#${config_mode}) & 8#022 )); then warn "OpenClaw config permissions are unsafe; refusing plugin mutation." return 1 fi fi if [[ -e "${target_dir}" || -L "${target_dir}" ]] \ && { [[ ! -d "${target_dir}" ]] || [[ -L "${target_dir}" ]]; }; then warn "Existing ask-question plugin path is not a real directory; refusing replacement." return 1 fi # Create the shared extensions directory without retightening it if OpenClaw # already made it; only our own subtree is forced to 700. if ! mkdir -p "${extensions_dir}"; then return 1 fi local transaction_dir="${openclaw_state_dir}/.bridgesllm-ask-user-tested-pair-v1" if [[ -e "${transaction_dir}" || -L "${transaction_dir}" ]] \ || ! install -d -m 700 "${transaction_dir}"; then return 1 fi local stage_dir="${transaction_dir}/candidate" local previous_dir="${transaction_dir}/previous-plugin" local previous_config="${transaction_dir}/previous-openclaw.json" local inspect_output="${transaction_dir}/inspect.json" local rpc_output="${transaction_dir}/rpc.json" local target_preexisted=false local config_preexisted=false local config_registration_attempted=false local target_displaced=false local plugin_installed=false local gateway_was_active=false local install_succeeded=false local failure_reason="" systemctl is-active --quiet openclaw-gateway && gateway_was_active=true if [[ -e "${target_dir}" || -L "${target_dir}" ]]; then target_preexisted=true fi if [[ -e "${config_path}" || -L "${config_path}" ]]; then config_preexisted=true fi if ! install -d -m 700 "${stage_dir}"; then failure_reason="could not create the staged plugin directory" else for required_file in index.js package.json openclaw.plugin.json; do if ! install -m 600 "${source_dir}/${required_file}" \ "${stage_dir}/${required_file}"; then failure_reason="could not stage ${required_file}" break fi done if [[ -z "${failure_reason}" ]]; then sync -f "${stage_dir}" 2>/dev/null \ || failure_reason="could not durably stage the plugin directory" fi fi if [[ -z "${failure_reason}" ]] && $config_preexisted \ && ! cp -a -- "${config_path}" "${previous_config}"; then failure_reason="could not preserve the OpenClaw config" fi if [[ -z "${failure_reason}" ]] && $config_preexisted \ && ! sync -f "${previous_config}" 2>/dev/null; then failure_reason="could not durably preserve the OpenClaw config" fi if [[ -z "${failure_reason}" ]]; then if ! write_bridgesllm_ask_user_transaction_manifest \ "${transaction_dir}" \ "${openclaw_state_dir}" \ "${target_dir}" \ "${config_path}" \ "${target_preexisted}" \ "${config_preexisted}" \ "${gateway_was_active}"; then failure_reason="could not arm the durable ask-user rollback transaction" else OPENCLAW_ASK_USER_TRANSACTION_ARMED=true OPENCLAW_ASK_USER_TRANSACTION_COMMITTED=false OPENCLAW_ASK_USER_TRANSACTION_DIR="${transaction_dir}" OPENCLAW_ASK_USER_STATE_DIR="${openclaw_state_dir}" OPENCLAW_ASK_USER_GATEWAY_WAS_ACTIVE="${gateway_was_active}" OPENCLAW_ASK_USER_BASE_ATTESTED=false OPENCLAW_ASK_USER_LIVE_ATTESTED=false fi fi if [[ -z "${failure_reason}" ]] && $target_preexisted; then if mv -T -- "${target_dir}" "${previous_dir}"; then target_displaced=true else failure_reason="could not preserve the previous plugin directory" fi fi if [[ -z "${failure_reason}" ]]; then if mv -T -- "${stage_dir}" "${target_dir}"; then plugin_installed=true sync -f "${extensions_dir}" 2>/dev/null \ || failure_reason="could not durably publish the staged plugin directory" else failure_reason="could not publish the staged plugin directory" fi fi if [[ -z "${failure_reason}" ]]; then config_registration_attempted=true if ! python3 "${source_dir}/register.py" >> "${LOG_FILE}" 2>&1; then failure_reason="could not register the plugin in OpenClaw config" fi fi if [[ -z "${failure_reason}" ]] \ && ! verify_bridgesllm_ask_user_plugin_config "${config_path}"; then failure_reason="OpenClaw config did not retain the enabled plugin registration" fi if [[ -z "${failure_reason}" ]] \ && ! verify_bridgesllm_ask_user_plugin_runtime \ "${target_dir}" "${inspect_output}"; then failure_reason="OpenClaw could not load the plugin or register its ask-user capabilities" fi if [[ -z "${failure_reason}" ]] && $gateway_was_active; then if ! systemctl restart openclaw-gateway >> "${LOG_FILE}" 2>&1; then failure_reason="the active OpenClaw gateway could not restart with the plugin" elif ! verify_openclaw_gateway_stable \ "${PIN_OPENCLAW_RUNTIME_VERSION}" 18; then failure_reason="the OpenClaw gateway was not stable after plugin activation" elif ! verify_bridgesllm_ask_user_gateway_method \ "${rpc_output}" bootstrap; then failure_reason="the running OpenClaw gateway failed ask-user execute and settlement semantics" fi fi [[ -z "${failure_reason}" ]] && install_succeeded=true if ! $install_succeeded; then local rollback_ok=true if $OPENCLAW_ASK_USER_TRANSACTION_ARMED; then rollback_bridgesllm_ask_user_transaction || rollback_ok=false else rm -rf -- "${transaction_dir}" || rollback_ok=false fi if ! $rollback_ok; then warn "Ask-question plugin rollback needs manual recovery from ${transaction_dir}." fi warn "Ask-question plugin activation failed: ${failure_reason}." return 1 fi $gateway_was_active && OPENCLAW_ASK_USER_BASE_ATTESTED=true ok "Ask-question answer channel installed and verified; tested-pair rollback remains armed" } ensure_openclaw_codex_plugin_compatible() { if $SKIP_OPENCLAW || ! command -v openclaw &>/dev/null; then return 0 fi if verify_openclaw_codex_plugin_pin "${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}"; then ok "OpenClaw Codex plugin ${PIN_OPENCLAW_CODEX_PLUGIN_VERSION} checked (exact pinned npm record)" return 0 fi if ! capture_openclaw_codex_plugin_baseline; then warn "Could not prove and preserve the existing OpenClaw Codex plugin baseline." return 1 fi OPENCLAW_CODEX_PLUGIN_UPDATE_ATTEMPTED=true local helper="${PORTAL_DIR}/backend/dist/services/openclawConfigManager.js" if [[ -f "${helper}" ]] && command -v node &>/dev/null; then info "Repairing stale OpenClaw Codex plugin state..." if ! OPENCLAW_ALLOW_ROOT=1 \ PORTAL_OPENCLAW_CODEX_PLUGIN_VERSION="${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" \ NODE_PATH="${PORTAL_DIR}/backend/node_modules" \ node -e 'const helper = process.argv[1]; const expected = process.argv[2]; const mod = require(helper); const result = mod.repairOpenClawCodexPluginInstallState(expected); console.log(JSON.stringify(result));' \ "${helper}" "${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" >> "$LOG_FILE" 2>&1; then warn "OpenClaw Codex plugin state repair failed. Continuing with compatibility check." else ok "OpenClaw Codex plugin state repaired" fi fi info "Installing tested OpenClaw Codex plugin ${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}..." if ! OPENCLAW_ALLOW_ROOT=1 openclaw plugins install \ "@openclaw/codex@${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" --force --pin >> "$LOG_FILE" 2>&1; then warn "OpenClaw Codex plugin install failed; the atomic core/plugin rollback remains armed." return 1 fi ok "OpenClaw Codex plugin package installed" if ! verify_openclaw_codex_plugin_pin "${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}"; then warn "OpenClaw Codex plugin did not produce the exact pinned npm install record and package revision; the atomic rollback remains armed." return 1 fi if systemctl is-enabled openclaw-gateway >/dev/null 2>&1; then if ! systemctl restart openclaw-gateway >> "$LOG_FILE" 2>&1 \ || ! verify_openclaw_gateway_stable "${PIN_OPENCLAW_RUNTIME_VERSION}" 18; then warn "The gateway did not load the tested Codex plugin cleanly; the atomic core/plugin rollback remains armed." return 1 fi fi # Keep the rollback armed until the exact core + authenticated gateway RPC + # plugin package/install-record pair is verified and committed together. ok "OpenClaw Codex plugin ${PIN_OPENCLAW_CODEX_PLUGIN_VERSION} loaded; pair commit pending" } apply_openclaw_codex_plugin_pending_input_hotfix() { if $SKIP_OPENCLAW || ! command -v openclaw &>/dev/null; then return 0 fi local hotfix_script="${PORTAL_DIR}/scripts/patch-openclaw-codex-pending-input-hotfix.sh" local target backup if [[ ! -f "${hotfix_script}" || -L "${hotfix_script}" ]]; then warn "Bundled Codex provider pending-input hotfix is missing or unsafe." return 1 fi target="$(resolve_openclaw_codex_plugin_pending_input_hotfix_target || true)" if [[ -z "${target}" ]]; then warn "Could not resolve exactly one run bundle in the pinned OpenClaw Codex provider." return 1 fi backup="${target}.bridgesllm-pending-input-v1.bak" # A patcher-owned backup means an earlier process stopped between preserving # and committing bytes. Normalize back to the pristine package first; only # then capture the package-level rollback baseline used by this transaction. if [[ -e "${backup}" || -L "${backup}" ]]; then if ! normalize_openclaw_codex_plugin_pending_input_recovery_artifact \ "${target}" "${hotfix_script}"; then warn "Codex provider pending-input recovery artifact is ambiguous or unsafe at ${backup}." return 1 fi fi if verify_openclaw_codex_plugin_pending_input_hotfix; then ok "OpenClaw Codex provider pending-input bridge checked" return 0 fi if ! capture_openclaw_codex_plugin_baseline; then warn "Could not preserve the Codex provider before pending-input preparation." return 1 fi OPENCLAW_CODEX_PLUGIN_UPDATE_ATTEMPTED=true chmod 755 "${hotfix_script}" 2>/dev/null || true info "Applying Codex provider pending-input hotfix (if needed)..." if ! PORTAL_OPENCLAW_PENDING_INPUT_STRICT=1 \ PORTAL_REQUIRED_OPENCLAW_PACKAGE_NAME='@openclaw/codex' \ PORTAL_REQUIRED_OPENCLAW_PACKAGE_VERSION="${PIN_OPENCLAW_CODEX_PLUGIN_VERSION}" \ bash "${hotfix_script}" "$(dirname "${target}")" \ >> "${LOG_FILE}" 2>&1 \ || ! openclaw_codex_plugin_pending_input_hotfix_is_applied "${target}"; then warn "Codex provider pending-input preparation failed; the package-level rollback remains armed." return 1 fi if ! retire_openclaw_codex_plugin_pending_input_backup \ "${target}" "${hotfix_script}"; then warn "Codex provider was patched, but its temporary byte backup could not be retired safely; the package-level rollback remains armed." return 1 fi if ! verify_openclaw_codex_plugin_pending_input_hotfix; then warn "Codex provider pending-input preparation returned without its exact external-runtime Symbol contract." return 1 fi # install_bridgesllm_ask_user_plugin performs the transaction's active- # gateway restart after publishing the matching multi-runtime bridge. Until # that restart and tested-pair verification succeed, plugin rollback stays # armed and restores the exact package tarball captured above. ok "OpenClaw Codex provider pending-input bridge prepared; gateway load pending" } commit_openclaw_tested_pair() { local openclaw_package_dir="" pending_target="" claude_target="" local ask_user_transaction="" ask_user_state_root="" local ask_user_target="" ask_user_config="" local sigint_trap="" sigterm_trap="" sighup_trap="" verify_openclaw_tested_pair || return 1 openclaw_package_dir="$(openclaw_core_package_dir || true)" pending_target="${OPENCLAW_PENDING_INPUT_HOTFIX_TARGET:-}" claude_target="${OPENCLAW_CLAUDE_ASK_USER_HOTFIX_TARGET:-}" if [[ -z "${pending_target}" ]]; then pending_target="$( resolve_openclaw_pending_input_hotfix_target \ "${openclaw_package_dir}/dist" || true )" fi if [[ -z "${claude_target}" ]]; then claude_target="$( resolve_openclaw_claude_ask_user_hotfix_target \ "${openclaw_package_dir}/dist" || true )" fi [[ -n "${pending_target}" && -n "${claude_target}" ]] || return 1 openclaw_pending_input_hotfix_is_applied "${pending_target}" || return 1 openclaw_claude_ask_user_hotfix_is_applied "${claude_target}" || return 1 $OPENCLAW_ASK_USER_TRANSACTION_ARMED || return 1 $OPENCLAW_ASK_USER_LIVE_ATTESTED || return 1 ask_user_transaction="${OPENCLAW_ASK_USER_TRANSACTION_DIR:-}" ask_user_state_root="${OPENCLAW_ASK_USER_STATE_DIR:-}" ask_user_target="${ask_user_state_root}/extensions/bridgesllm-ask-user" ask_user_config="${ask_user_state_root}/openclaw.json" [[ -n "${ask_user_transaction}" && -n "${ask_user_state_root}" ]] || return 1 # One fsynced record is the commit decision for both byte-level mutations. # Block termination only across that short decision boundary. Once the # record and in-memory flag exist, cleanup can be retried without rolling one # half of the tested pair back independently. sigint_trap="$(trap -p SIGINT || true)" sigterm_trap="$(trap -p TERM || true)" sighup_trap="$(trap -p HUP || true)" trap '' SIGINT TERM HUP if ! write_openclaw_tested_pair_commit_record \ "${pending_target}" \ "${claude_target}" \ "${ask_user_transaction}" \ "${ask_user_target}" \ "${ask_user_config}"; then [[ -n "${sigint_trap}" ]] && eval "${sigint_trap}" || trap - SIGINT [[ -n "${sigterm_trap}" ]] && eval "${sigterm_trap}" || trap - TERM [[ -n "${sighup_trap}" ]] && eval "${sighup_trap}" || trap - HUP return 1 fi OPENCLAW_UPGRADE_COMMITTED=true OPENCLAW_CODEX_PLUGIN_UPDATE_ATTEMPTED=false OPENCLAW_ASK_USER_TRANSACTION_COMMITTED=true [[ -n "${sigint_trap}" ]] && eval "${sigint_trap}" || trap - SIGINT [[ -n "${sigterm_trap}" ]] && eval "${sigterm_trap}" || trap - TERM [[ -n "${sighup_trap}" ]] && eval "${sighup_trap}" || trap - HUP if ! commit_openclaw_pending_input_hotfix; then warn "The tested pair committed, but the native pending-input backup could not be retired; the durable commit record will reconcile it on the next run." fi if ! commit_openclaw_claude_ask_user_hotfix; then warn "The tested pair committed, but the Claude ask-user backup could not be retired; the durable commit record will reconcile it on the next run." fi if ! commit_bridgesllm_ask_user_transaction; then warn "The tested pair committed, but the ask-user plugin/config rollback directory could not be retired; the durable commit record will reconcile it on the next run." fi OPENCLAW_PENDING_INPUT_HOTFIX_COMMITTED=true OPENCLAW_CLAUDE_ASK_USER_HOTFIX_COMMITTED=true if ! retire_openclaw_tested_pair_commit_record_if_clean \ "${pending_target}" \ "${claude_target}" \ "${ask_user_transaction}"; then warn "The tested pair committed, but its completed decision record could not be retired; backup-generation binding prevents stale replay." fi return 0 } configure_openclaw_codex_harness_defaults() { if $SKIP_OPENCLAW || ! command -v openclaw &>/dev/null; then return 0 fi local oc_config="${HOME}/.openclaw/openclaw.json" [[ -f "${oc_config}" ]] || return 0 if python3 - "$oc_config" >> "$LOG_FILE" 2>&1 <<'PY' import json import sys from pathlib import Path path = Path(sys.argv[1]) try: data = json.loads(path.read_text()) except Exception: sys.exit(0) plugins = data.setdefault("plugins", {}) changed = False if plugins.get("enabled") is not True: plugins["enabled"] = True changed = True entries = plugins.setdefault("entries", {}) codex = entries.setdefault("codex", {}) if codex.get("enabled") is not True: codex["enabled"] = True changed = True config = codex.setdefault("config", {}) app_server = config.setdefault("appServer", {}) for key, value in { "turnCompletionIdleTimeoutMs": 180000, "postToolRawAssistantCompletionIdleTimeoutMs": 180000, }.items(): if app_server.get(key) != value: app_server[key] = value changed = True if changed: path.write_text(json.dumps(data, indent=2) + "\n") PY then ok "OpenClaw Codex harness defaults checked" else warn "Could not update OpenClaw Codex harness defaults" fi } prepare_openclaw_runtime_for_portal() { if $SKIP_OPENCLAW; then info "Skipping OpenClaw runtime compatibility checks (--skip-openclaw)" return 0 fi if ! prepare_openclaw_upgrade_state; then fail "OpenClaw upgrade stopped before the first new gateway restart because legacy state could not be proven safe. No data was deleted and openclaw doctor --fix was not run." fi if ! verify_openclaw_core_package_pin; then fail "OpenClaw core is not the tested ${PIN_OPENCLAW_CORE_PACKAGE_VERSION} package / ${PIN_OPENCLAW_RUNTIME_VERSION} runtime pair. The updater will restore the preserved previous runtime when applicable." fi # Publish the replacement tool/config under the same durable rollback # decision as the core and native bundle patches. Native AskUserQuestion is # still available throughout this first boot and live base attestation. if ! install_bridgesllm_ask_user_plugin; then fail "OpenClaw is healthy, but the Portal ask-question answer channel could not be installed and verified. The previous plugin/config were restored when possible." fi if ! ensure_openclaw_gateway_boots_cleanly; then fail "OpenClaw gateway did not boot cleanly with the replacement ask-user bridge. Check: journalctl -u openclaw-gateway -n 100 --no-pager" fi if ! ensure_openclaw_gateway_matches_cli; then fail "OpenClaw gateway version/RPC readiness could not be verified. The updater will restore the previous OpenClaw runtime when applicable. Check: systemctl status openclaw-gateway" fi if ! verify_bridgesllm_ask_user_tested_pair /root/.openclaw bootstrap; then fail "The replacement ask-user tool or one of its live settlement methods failed base semantic attestation before native Claude question routing could be changed." fi OPENCLAW_ASK_USER_BASE_ATTESTED=true auto_apply_openclaw_compatibility_hotfix if ! ensure_openclaw_gateway_boots_cleanly \ || ! ensure_openclaw_gateway_matches_cli \ || ! verify_bridgesllm_ask_user_tested_pair; then fail "OpenClaw did not retain the complete ask-user bridge after compatibility activation. The tested-pair rollback remains armed." fi OPENCLAW_ASK_USER_LIVE_ATTESTED=true if $OPENCLAW_PACKAGE_UPDATED || [[ -n "${OPENCLAW_UPGRADE_STATE_MANIFEST:-}" ]]; then local expected_gateway_version expected_gateway_version="$(openclaw_cli_version)" $OPENCLAW_PACKAGE_UPDATED && expected_gateway_version="${PIN_OPENCLAW_RUNTIME_VERSION}" if ! verify_openclaw_gateway_stable "${expected_gateway_version}" 18; then fail "OpenClaw gateway did not remain ready and restart-free through the post-upgrade stability window. The updater will restore the previous runtime." fi fi ensure_openclaw_sandbox_image run_openclaw_state_repair_notice repair_openclaw_portal_model_config bridge_openclaw_codex_cli_auth # Runtime attestation below intentionally rejects a disabled or inert Codex # provider. Converge the required harness state before inspecting it. configure_openclaw_codex_harness_defaults if ! ensure_openclaw_codex_plugin_compatible; then fail "OpenClaw core is healthy, but the tested Codex plugin ${PIN_OPENCLAW_CODEX_PLUGIN_VERSION} could not be installed and loaded. The previous plugin was restored when possible." fi if ! apply_openclaw_codex_plugin_pending_input_hotfix; then fail "OpenClaw Codex is installed, but its active provider bundle could not be prepared for exact ask-question delivery. The previous plugin was restored when possible." fi if ! commit_openclaw_tested_pair; then fail "OpenClaw compatibility verification did not produce the tested core ${PIN_OPENCLAW_CORE_PACKAGE_VERSION} / Codex plugin ${PIN_OPENCLAW_CODEX_PLUGIN_VERSION} / Portal answer-channel runtime." fi # Only this boundary, after the required Portal bridge is live, disarms the # core/plugin/hotfix rollback paths and retires the owned byte backup. } configure_backup_timers() { step_header "Configuring backup automation" CURRENT_STEP="backup automation" $DRY_RUN && { ok "[dry-run] Would configure backup timers"; return; } local backup_script="${PORTAL_DIR}/backup-full.sh" if [[ ! -f "${backup_script}" ]]; then warn "Backup script missing at ${backup_script}; timers not enabled" return 0 fi chmod 750 "${backup_script}" 2>/dev/null || true local backup_state_dir="${PORTAL_DIR}/backend/.data/backups" local backup_config_file="${backup_state_dir}/backup-base-path" mkdir -p "${backup_state_dir}" # `.data` holds credential-lifecycle state as well as backup state, and # `mkdir -p` creates parents under the installer umask. Left at 0755 it fails # the credential ledger's own permission contract, which is how provider # OAuth setup came to fail on every install. Narrow the parent explicitly. chmod 700 "${PORTAL_DIR}/backend/.data" chmod 700 "${backup_state_dir}" if [[ ! -s "${backup_config_file}" ]]; then local configured_backup_base="" local portal_database_url="" portal_database_url="$(read_env_value "${PORTAL_DIR}/backend/.env.production" DATABASE_URL 2>/dev/null || true)" if [[ -n "${portal_database_url}" ]] && command -v node >/dev/null 2>&1 \ && [[ -d "${PORTAL_DIR}/backend/node_modules/@prisma/client" ]]; then configured_backup_base="$( cd "${PORTAL_DIR}/backend" && \ DATABASE_URL="${portal_database_url}" node -e ' const { prisma } = require("./dist/config/database"); prisma.systemSetting.findUnique({ where: { key: "system.backupPath" } }) .then((row) => { if (row?.value) process.stdout.write(row.value); }) .finally(() => prisma.$disconnect()); ' 2>/dev/null || true )" fi printf '%s\n' "${configured_backup_base:-/root/backups}" > "${backup_config_file}" fi chmod 600 "${backup_config_file}" BACKUP_CONFIG_FILE="${backup_config_file}" BACKUP_STATE_DIR="${backup_state_dir}" \ /bin/bash "${backup_script}" --list >> "$LOG_FILE" 2>&1 \ || fail "Configured backup storage path is invalid or insecure." cat > /etc/systemd/system/bridgesllm-backup@.service << BACKUPSERVICE [Unit] Description=BridgesLLM Portal %i backup After=docker.service bridgesllm-product.service Wants=docker.service [Service] Type=oneshot User=root Environment=PORTAL_ROOT=${PORTAL_DIR} Environment=INSTALL_ROOT=${INSTALL_ROOT} Environment=BACKUP_CONFIG_FILE=${backup_config_file} Environment=BACKUP_STATE_DIR=${backup_state_dir} ExecStart=/bin/bash ${backup_script} %i ExecStopPost=/bin/bash ${backup_script} --recover-quiescence Nice=10 IOSchedulingClass=best-effort IOSchedulingPriority=7 UMask=0077 TimeoutStartSec=6h TimeoutStopSec=10m StandardOutput=journal StandardError=journal BACKUPSERVICE cat > /etc/systemd/system/bridgesllm-backup-daily.timer << 'BACKUPDAILY' [Unit] Description=Daily BridgesLLM Portal backup [Timer] OnCalendar=*-*-* 02:00:00 Persistent=true Unit=bridgesllm-backup@daily.service [Install] WantedBy=timers.target BACKUPDAILY cat > /etc/systemd/system/bridgesllm-backup-comprehensive.timer << 'BACKUPWEEKLY' [Unit] Description=Weekly comprehensive BridgesLLM Portal backup [Timer] OnCalendar=Sun *-*-* 03:00:00 Persistent=true Unit=bridgesllm-backup@comprehensive.service [Install] WantedBy=timers.target BACKUPWEEKLY cat > /etc/systemd/system/bridgesllm-backup-monthly.timer << 'BACKUPMONTHLY' [Unit] Description=Monthly BridgesLLM Portal backup [Timer] OnCalendar=*-*-01 04:00:00 Persistent=true Unit=bridgesllm-backup@monthly.service [Install] WantedBy=timers.target BACKUPMONTHLY systemctl daemon-reload systemctl enable --now bridgesllm-backup-daily.timer bridgesllm-backup-comprehensive.timer bridgesllm-backup-monthly.timer >> "$LOG_FILE" 2>&1 \ || fail "Backup timers could not be enabled." local cron_tmp cron_tmp="$(mktemp)" if crontab -l > "${cron_tmp}" 2>/dev/null; then if grep -q 'backup-full\.sh' "${cron_tmp}"; then grep -v 'backup-full\.sh' "${cron_tmp}" | crontab - ok "Removed stale backup cron entries" fi fi rm -f "${cron_tmp}" ok "Backup timers configured" } ensure_agent_zero_project_model_bridge() { local lifecycle="${PORTAL_DIR}/installer/agent-zero-project-model-bridge.sh" local lifecycle_command="install" [[ -f "$lifecycle" && ! -L "$lifecycle" ]] \ || fail "Agent Zero Project model bridge lifecycle is missing from the verified release." [[ "$(stat -c '%u' "$lifecycle")" == "0" ]] \ || fail "Agent Zero Project model bridge lifecycle is not root-owned." (( (8#$(stat -c '%a' "$lifecycle") & 0022) == 0 )) \ || fail "Agent Zero Project model bridge lifecycle is writable by an unsafe principal." if command -v docker >/dev/null 2>&1 \ && docker container inspect bridgesllm-agent-zero >/dev/null 2>&1 \ && [[ "$(docker inspect --format '{{.State.Running}}' bridgesllm-agent-zero 2>/dev/null || true)" == "true" ]]; then lifecycle_command="reconcile" info "Reconciling Agent Zero Project model bridge with the managed OAuth upstream..." else info "Installing fail-closed Agent Zero Project model bridge..." fi bash "$lifecycle" "$lifecycle_command" >> "$LOG_FILE" 2>&1 \ || fail "Agent Zero Project model bridge could not be installed and verified." if [[ "$lifecycle_command" == "reconcile" ]]; then ok "Agent Zero Project model bridge is active and upstream-authenticated" else ok "Agent Zero Project model bridge is active in fail-closed mode (Agent Zero not running)" fi } start_portal() { step_header "Starting portal" CURRENT_STEP="startup" local env_file="${PORTAL_DIR}/backend/.env.production" ensure_project_egress_token_secret "${env_file}" ensure_docker_address_pools provision_project_runtimes "${env_file}" # Start OpenClaw gateway first (portal connects to it) if systemctl is-enabled openclaw-gateway &>/dev/null 2>&1; then prepare_openclaw_runtime_for_portal sleep 3 # Let gateway fully initialize and write its config # CRITICAL: Sync the gateway token into .env.production # OpenClaw may regenerate its token on first start. Read the ACTUAL token # from openclaw.json and ensure .env.production matches. local oc_config_path="${HOME}/.openclaw/openclaw.json" if [[ -f "${oc_config_path}" ]]; then local live_token live_token="$(python3 -c " import json try: d = json.load(open('${oc_config_path}')) print(d.get('gateway',{}).get('auth',{}).get('token','')) except: pass " 2>/dev/null || true)" if [[ -n "${live_token}" ]] && [[ -f "${env_file}" ]]; then local env_token env_token="$(read_env_value "${env_file}" "OPENCLAW_GATEWAY_TOKEN" || true)" if [[ "${live_token}" != "${env_token}" ]]; then info "Syncing gateway token (openclaw.json → .env.production)" sed -i "s/^OPENCLAW_GATEWAY_TOKEN=.*/OPENCLAW_GATEWAY_TOKEN=${live_token}/" "${env_file}" fi fi fi fi systemctl start bridgesllm-product ensure_agent_zero_project_model_bridge if ${RETAINED_RECONNECT_MODE}; then # Runtime intent is applied only after signed files, dependencies, # configuration, migrations, service units, and Project runtime policy # have converged. Starting retained containers from build_portal() was too # early: they could race unfinished migrations and stale bridge policy. restore_retained_runtime_intent # A resumed Agent Zero container changes the bridge from fail-closed to an # authenticated upstream. Reconcile once more after applying the intent. ensure_agent_zero_project_model_bridge fi # Auto-approve portal's device pairing with gateway (loopback should auto-approve # but some OpenClaw versions require explicit approval for operator-scoped devices) if command -v openclaw &>/dev/null; then sleep 3 # Wait for portal to attempt first connect openclaw devices approve --latest >> "$LOG_FILE" 2>&1 || true fi if ! verify_portal_service_health "bridgesllm-product" "http://127.0.0.1:4001/health" 60; then warn "Portal failed health verification — check: journalctl -u bridgesllm-product -n 50" fi if ! verify_portal_update_readiness "${VERSION}" "${PORTAL_UPDATE_PROBE_TOKEN}" 900; then fail "Portal failed exact-version authenticated readiness verification." fi if ! verify_app_content_caddy_ready; then fail "Isolated app-content TLS host did not become ready. Verify APP_CONTENT_DOMAIN DNS and Caddy ACME logs." fi if use_tailnet_profile && ! verify_tailnet_tls_ready; then fail "The tailnet origin https://${TAILNET_DNS_NAME}/ did not answer over HTTPS, so the installer will not print a credential-bearing setup link. Check 'tailscale serve status' and rerun this installer." fi if [[ -n "${DOMAIN}" ]] && ! use_local_profile && ! verify_portal_tls_ready; then fail "Portal TLS is not ready, so the installer will not print or enable a credential-bearing setup link. Verify DOMAIN DNS and Caddy ACME logs, then rerun the installer with --reinstall." fi if ! commit_portal_deploy_provenance; then fail "Portal is healthy, but verified release provenance could not be committed atomically." fi ok "Portal release provenance committed" } # ═══════════════════════════════════════════════════════════════ # Step 9: Done # ═══════════════════════════════════════════════════════════════ print_success() { CURRENT_STEP_NUM=$((CURRENT_STEP_NUM + 1)) local url url="$(portal_setup_url)" local elapsed elapsed="$(elapsed_since_start)" echo "" echo -e " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" if [[ -n "$elapsed" ]]; then echo -e " ${GREEN}${BOLD} Installation complete!${NC} ${DIM}(${elapsed})${NC}" else echo -e " ${GREEN}${BOLD} Installation complete!${NC}" fi echo "" echo -e " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" if [[ -z "${SETUP_TOKEN}" ]]; then echo -e " ${WHITE} Open the Portal:${NC}" echo "" echo -e " ${BOLD}${CYAN} ${url}${NC}" elif use_local_profile; then echo -e " ${WHITE} Open this local-only URL to finish setup:${NC}" echo "" echo -e " ${BOLD}${CYAN} ${url}${NC}" echo "" echo -e " ${DIM} The bootstrap fragment is exchanged once on loopback and removed from the address bar.${NC}" elif use_tailnet_profile; then echo -e " ${WHITE} Tailnet HTTPS was verified. From ANY device on your Tailscale network,${NC}" echo -e " ${WHITE} open this URL to finish setup:${NC}" echo "" echo -e " ${BOLD}${CYAN} ${url}${NC}" echo "" echo -e " ${DIM} Your portal is private: it is reachable only from devices signed into your tailnet.${NC}" echo -e " ${DIM} No public ports are open. Mail and hosted app content need a public domain and are${NC}" echo -e " ${DIM} disabled in this mode — rerun the installer with --domain later to enable them.${NC}" echo -e " ${YELLOW} Note: tailnet origin mode is EXPERIMENTAL and still under field validation.${NC}" elif [[ -n "${DOMAIN}" ]]; then echo -e " ${WHITE} TLS was verified. Open this HTTPS URL to finish setup:${NC}" echo "" echo -e " ${BOLD}${CYAN} ${url}${NC}" echo "" echo -e " ${DIM} The bootstrap fragment is exchanged once over HTTPS and removed from the address bar.${NC}" else local ssh_target ssh_target="$(portal_setup_ssh_user)@${PUBLIC_IP}" echo -e " ${WHITE} No TLS domain is ready, so setup is loopback-only.${NC}" echo -e " ${WHITE} On your computer, open a second terminal and keep this tunnel running:${NC}" echo "" echo -e " ${BOLD}${CYAN} ssh -N -L 4001:127.0.0.1:4001 ${ssh_target}${NC}" echo "" echo -e " ${WHITE} Then open this loopback URL in your browser:${NC}" echo "" echo -e " ${BOLD}${CYAN} ${url}${NC}" echo "" echo -e " ${DIM} Public HTTP returns 403. The SSH tunnel carries setup credentials encrypted.${NC}" fi if [[ -n "${SETUP_TOKEN}" ]]; then echo -e " ${YELLOW} The one-time bootstrap expires no later than 24 hours after minting and cannot be replayed after exchange.${NC}" fi echo "" # What was installed summary echo -e " ${DIM}What was installed:${NC}" local node_ver="" pg_ver="" caddy_ver="" docker_ver="" ollama_ver="" openclaw_ver="" clawhub_ver="" node_ver="$(node -v 2>/dev/null || echo '?')" pg_ver="$(psql --version 2>/dev/null | grep -oP '\d+' | head -1 || echo '?')" caddy_ver="$(caddy version 2>/dev/null | head -1 | cut -d' ' -f1 || echo '?')" docker_ver="$(docker --version 2>/dev/null | grep -oP '\d+\.\d+' | head -1 || echo '?')" ollama_ver="$(ollama_client_version || true)" ollama_ver="${ollama_ver:--}" openclaw_ver="$(openclaw --version 2>/dev/null | head -1 | grep -oP '\d{4}\.\d+\.\d+(-\d+)?' || echo '-')" clawhub_ver="$(clawhub --cli-version 2>/dev/null | head -1 | grep -oP '\d+\.\d+\.\d+' || echo '-')" echo -e " ${DIM}${BULLET}${NC} Node.js ${node_ver} ${DIM}${BULLET}${NC} PostgreSQL ${pg_ver} ${DIM}${BULLET}${NC} Caddy ${caddy_ver}" echo -e " ${DIM}${BULLET}${NC} Docker ${docker_ver} ${DIM}${BULLET}${NC} Ollama ${ollama_ver} ${DIM}${BULLET}${NC} OpenClaw ${openclaw_ver} ${DIM}${BULLET}${NC} ClawHub ${clawhub_ver}" echo "" if use_local_profile; then echo -e " ${DIM}This beta path is for local Windows / WSL testing and is still experimental / untested.${NC}" echo -e " ${DIM}In the wizard, skip domain + HTTPS for now and use localhost access.${NC}" echo -e " ${DIM}Public hosting, custom domains, and external share links remain VPS features for now.${NC}" elif [[ -z "$DOMAIN" ]]; then echo -e " ${DIM}You can prove a domain and hand off to HTTPS in the wizard, or finish entirely through the tunnel.${NC}" fi echo -e " ${DIM}Log: ${LOG_FILE}${NC}" echo "" telemetry_event "install_complete" } # ═══════════════════════════════════════════════════════════════ # Update & Uninstall # ═══════════════════════════════════════════════════════════════ attest_existing_portal_for_update() { local portal_root="${1:-${PORTAL_DIR}}" python3 - "${portal_root}" <<'PY2' import json import os import re import stat import sys root = os.path.abspath(sys.argv[1]) if root != os.path.normpath(sys.argv[1]): raise SystemExit(1) def safe_directory(path: str) -> None: info = os.lstat(path) if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022): raise SystemExit(1) def safe_file(relative: str, maximum: int) -> str: path = os.path.join(root, relative) info = os.lstat(path) if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 or not 0 < info.st_size <= maximum): raise SystemExit(1) raw = open(path, "rb").read() if b"\x00" in raw: raise SystemExit(1) return raw.decode("utf-8") safe_directory(root) for relative in ("backend", "backend/dist", "frontend", "installer"): safe_directory(os.path.join(root, relative)) backend_package = json.loads(safe_file("backend/package.json", 1024 * 1024)) frontend_package = json.loads(safe_file("frontend/package.json", 1024 * 1024)) compiled = safe_file("backend/dist/version.js", 1024 * 1024) safe_file("backend/dist/server.js", 64 * 1024 * 1024) installed = safe_file("installer/install.sh", 2 * 1024 * 1024) safe_file("backend/.env.production", 1024 * 1024) safe_file("frontend/.env", 1024 * 1024) compiled_match = re.search(r"PORTAL_VERSION\s*=\s*['\"]([^'\"]+)['\"]", compiled) installer_match = re.search(r"readonly\s+VERSION\s*=\s*['\"]([^'\"]+)['\"]", installed) versions = [ backend_package.get("version"), frontend_package.get("version"), compiled_match.group(1) if compiled_match else None, installer_match.group(1) if installer_match else None, ] if (not all(isinstance(value, str) for value in versions) or len(set(versions)) != 1 or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", versions[0])): raise SystemExit(1) print(versions[0]) PY2 } create_update_backup_dir() { local backup_root="${1:-${INSTALL_ROOT}/backups}" [[ "${backup_root}" == "${INSTALL_ROOT}/backups" ]] || return 1 install -d -m 0700 -- "${backup_root}" || return 1 [[ -d "${backup_root}" && ! -L "${backup_root}" \ && "$(stat -c '%u:%g:%a' "${backup_root}")" == "0:0:700" ]] || return 1 local backup_dir backup_dir="$(mktemp -d "${backup_root}/update-transaction.XXXXXX")" || return 1 chmod 0700 "${backup_dir}" || return 1 printf '%s\n' "${backup_dir}" } service_unit_cgroup_is_empty() { local service_name="${1:-bridgesllm-product}" local control_group="${2:-}" local cgroup_root="/sys/fs/cgroup" if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && -n "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" ]]; then cgroup_root="${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT}/sys/fs/cgroup" fi [[ -n "${control_group}" && "${control_group}" != "/" \ && "${control_group}" == /* && "${control_group}" != *".."* ]] || return 1 python3 - "${cgroup_root}" "${control_group}" "${service_name}" <<'PY2' import os import pathlib import re import sys root = pathlib.Path(sys.argv[1]).resolve() control_group = sys.argv[2] service = sys.argv[3] if not re.fullmatch(r"/[A-Za-z0-9_.@:/-]+", control_group): raise SystemExit(1) if not control_group.rstrip("/").endswith(f"/{service}.service"): raise SystemExit(1) candidate = (root / control_group.lstrip("/")).resolve() if candidate != root and root not in candidate.parents: raise SystemExit(1) if not candidate.is_dir(): raise SystemExit(1) for procs in candidate.rglob("cgroup.procs"): if any(line.strip() for line in procs.read_text(encoding="ascii").splitlines()): raise SystemExit(1) PY2 } portal_service_port_is_free() { local port="${1:-4001}" local proc_net_root="/proc/net" if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" == "1" \ && -n "${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT:-}" ]]; then proc_net_root="${BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT}/proc/net" fi [[ "${port}" =~ ^[0-9]+$ && "${port}" -ge 1 && "${port}" -le 65535 ]] || return 1 python3 - "${port}" "${proc_net_root}" <<'PY2' import pathlib import sys port = int(sys.argv[1]) proc_net = pathlib.Path(sys.argv[2]) needle = f"{port:04X}" for table in (proc_net / "tcp", proc_net / "tcp6"): try: rows = table.read_text(encoding="ascii").splitlines()[1:] except OSError: raise SystemExit(1) for row in rows: fields = row.split() if len(fields) < 4: raise SystemExit(1) local = fields[1] state = fields[3] try: local_port = local.rsplit(":", 1)[1].upper() except IndexError: raise SystemExit(1) if state == "0A" and local_port == needle: raise SystemExit(1) PY2 } current_boot_id() { local boot_id_path boot_id boot_id_path="$( update_transaction_state_path /proc/sys/kernel/random/boot_id )" || return 1 boot_id="$(tr -d '[:space:]' < "${boot_id_path}" 2>/dev/null)" \ || return 1 [[ "${boot_id}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] \ || return 1 printf '%s\n' "${boot_id}" } process_start_time_of_pid() { # Field 22 of /proc//stat, parsed after the last ')' so a comm value # containing spaces or parentheses can never shift the field index. local pid="$1" stat_path stat_line rest start [[ "${pid}" =~ ^[1-9][0-9]*$ ]] || return 1 stat_path="$(update_transaction_state_path "/proc/${pid}/stat")" || return 1 stat_line="$(cat "${stat_path}" 2>/dev/null)" || return 1 rest="${stat_line##*)}" local -a fields read -r -a fields <<<"${rest}" start="${fields[19]:-}" [[ "${start}" =~ ^[1-9][0-9]*$ ]] || return 1 printf '%s\n' "${start}" } portal_service_is_quiesced() { local service_name="${1:-bridgesllm-product}" local state load_state active_state sub_state main_pid control_pid control_group state="$(systemctl show "${service_name}" \ -p LoadState -p ActiveState -p SubState -p MainPID -p ControlPID -p ControlGroup 2>/dev/null)" \ || return 1 load_state="$(sed -n 's/^LoadState=//p' <<<"${state}")" active_state="$(sed -n 's/^ActiveState=//p' <<<"${state}")" sub_state="$(sed -n 's/^SubState=//p' <<<"${state}")" main_pid="$(sed -n 's/^MainPID=//p' <<<"${state}")" control_pid="$(sed -n 's/^ControlPID=//p' <<<"${state}")" control_group="$(sed -n 's/^ControlGroup=//p' <<<"${state}")" [[ "${load_state}" == "loaded" \ && "${active_state}" =~ ^(inactive|failed)$ \ && "${sub_state}" =~ ^(dead|failed)$ \ && "${main_pid}" == "0" \ && "${control_pid}" == "0" ]] || return 1 if [[ -n "${control_group}" ]]; then service_unit_cgroup_is_empty "${service_name}" "${control_group}" || return 1 fi } capture_portal_service_baseline() { local service_name="${1:-bridgesllm-product}" local state load_state active_state main_pid enabled_state state="$(systemctl show "${service_name}" -p LoadState -p ActiveState -p MainPID 2>/dev/null)" \ || return 1 load_state="$(sed -n 's/^LoadState=//p' <<<"${state}")" active_state="$(sed -n 's/^ActiveState=//p' <<<"${state}")" main_pid="$(sed -n 's/^MainPID=//p' <<<"${state}")" [[ "${load_state}" == "loaded" ]] || return 1 # Process identity is (boot id, pid, start time): a numeric PID alone is # ambiguous across reboots and PID-space reuse, and the replacement proof # after cutover/rollback must never be satisfied or defeated by a recycled # number. UPDATE_TRANSACTION_BASELINE_BOOT_ID="$(current_boot_id)" || return 1 case "${active_state}" in active) [[ "${main_pid}" =~ ^[1-9][0-9]*$ ]] || return 1 UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE=true UPDATE_TRANSACTION_BASELINE_PID="${main_pid}" UPDATE_TRANSACTION_BASELINE_START_TIME="$( process_start_time_of_pid "${main_pid}" )" || return 1 ;; inactive|failed) [[ "${main_pid}" == "0" ]] || return 1 UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE=false UPDATE_TRANSACTION_BASELINE_PID="0" UPDATE_TRANSACTION_BASELINE_START_TIME="0" ;; *) return 1 ;; esac enabled_state="$(systemctl is-enabled "${service_name}" 2>/dev/null || true)" case "${enabled_state}" in enabled) UPDATE_TRANSACTION_PORTAL_WAS_ENABLED=true ;; disabled) UPDATE_TRANSACTION_PORTAL_WAS_ENABLED=false ;; *) return 1 ;; esac } quiesce_portal_service() { local service_name="${1:-bridgesllm-product}" local graceful_timeout="${2:-45}" kill_timeout="${3:-15}" port="${4:-4001}" local waited=0 stop_status=0 [[ "${graceful_timeout}" =~ ^[0-9]+$ && "${kill_timeout}" =~ ^[0-9]+$ \ && "${port}" =~ ^[0-9]+$ ]] || return 1 if portal_service_is_quiesced "${service_name}" && portal_service_port_is_free "${port}"; then return 0 fi systemctl stop --no-block "${service_name}" >> "${LOG_FILE}" 2>&1 || stop_status=$? while (( waited < graceful_timeout )); do if portal_service_is_quiesced "${service_name}" && portal_service_port_is_free "${port}"; then return 0 fi sleep 1 waited=$((waited + 1)) done warn "Portal did not quiesce gracefully (systemctl stop exit ${stop_status}); escalating once to SIGKILL." systemctl kill --kill-who=all --signal=SIGKILL "${service_name}" >> "${LOG_FILE}" 2>&1 \ || return 1 waited=0 while (( waited < kill_timeout )); do if portal_service_is_quiesced "${service_name}" && portal_service_port_is_free "${port}"; then return 0 fi sleep 1 waited=$((waited + 1)) done return 1 } reload_active_root_user_systemd_manager() { local output_path="${1:-${LOG_FILE}}" local load_state active_state load_state="$( systemctl show --property=LoadState --value user@0.service 2>/dev/null )" || return 1 active_state="$( systemctl show --property=ActiveState --value user@0.service 2>/dev/null )" || return 1 case "${load_state}:${active_state}" in loaded:active) [[ -d /run/user/0 && ! -L /run/user/0 \ && -S /run/user/0/bus && ! -L /run/user/0/bus \ && -S /run/user/0/systemd/private \ && ! -L /run/user/0/systemd/private \ && "$(stat -c '%u:%g' /run/user/0 /run/user/0/bus /run/user/0/systemd/private \ | sort -u)" == '0:0' ]] \ || return 1 ( export XDG_RUNTIME_DIR=/run/user/0 export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus systemctl --user daemon-reload >> "${output_path}" 2>&1 ) || return 1 ;; loaded:inactive|not-found:inactive) ;; *) return 1 ;; esac } install_exact_openclaw_gateway_authorization_fence_dropin() { local canonical_dropin_dir="$1" local canonical_dropin_path="$2" local expected="$3" local dropin_dir dropin_path dropin_dir="$( update_transaction_state_path \ "${canonical_dropin_dir}" )" || return 1 dropin_path="$( update_transaction_state_path \ "${canonical_dropin_path}" )" || return 1 python3 - "${dropin_dir}" "${dropin_path}" "${expected}" <<'PY2' \ || return 1 import ctypes import errno import os import stat import sys import tempfile directory, destination, expected_text = sys.argv[1:] expected = expected_text.encode("utf-8") if ( os.geteuid() != 0 or not os.path.isabs(directory) or os.path.normpath(directory) != directory or not os.path.isabs(destination) or os.path.normpath(destination) != destination or os.path.dirname(destination) != directory or os.path.basename(destination) != "20-bridgesllm-authorization-fence.conf" ): raise SystemExit(1) def attest_directory(path, mode=None): details = os.lstat(path) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != 0 or details.st_gid != 0 or details.st_mode & 0o022 or (mode is not None and stat.S_IMODE(details.st_mode) != mode) ): raise SystemExit(1) parent = os.path.dirname(directory) current = os.path.sep attest_directory(current) for component in parent.strip(os.path.sep).split(os.path.sep): if not component: continue current = os.path.join(current, component) try: attest_directory(current) except FileNotFoundError: current_parent = os.path.dirname(current) os.mkdir(current, 0o755) os.chown(current, 0, 0) os.chmod(current, 0o755) current_parent_fd = os.open( current_parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(current_parent_fd) finally: os.close(current_parent_fd) attest_directory(current, 0o755) try: attest_directory(directory, 0o755) except FileNotFoundError: os.mkdir(directory, 0o755) os.chown(directory, 0, 0) os.chmod(directory, 0o755) parent_fd = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(parent_fd) finally: os.close(parent_fd) attest_directory(directory, 0o755) directory_fd = os.open( directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) destination_name = os.path.basename(destination) def attest_destination(): details = os.stat( destination_name, dir_fd=directory_fd, follow_symlinks=False, ) if ( not stat.S_ISREG(details.st_mode) or details.st_uid != 0 or details.st_gid != 0 or stat.S_IMODE(details.st_mode) != 0o644 or details.st_nlink != 1 or details.st_size != len(expected) ): raise SystemExit(1) descriptor = os.open( destination_name, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=directory_fd, ) try: opened = os.fstat(descriptor) if ( (opened.st_dev, opened.st_ino, opened.st_mode, opened.st_nlink) != (details.st_dev, details.st_ino, details.st_mode, details.st_nlink) or opened.st_uid != 0 or opened.st_gid != 0 or opened.st_size != len(expected) ): raise SystemExit(1) content = bytearray() while len(content) < len(expected) + 1: block = os.read(descriptor, len(expected) + 1 - len(content)) if not block: break content.extend(block) if bytes(content) != expected: raise SystemExit(1) finally: os.close(descriptor) temporary = "" try: try: attest_destination() except FileNotFoundError: descriptor, temporary = tempfile.mkstemp( prefix=".20-bridgesllm-authorization-fence.", dir=directory, ) try: os.fchown(descriptor, 0, 0) os.fchmod(descriptor, 0o644) view = memoryview(expected) while view: written = os.write(descriptor, view) view = view[written:] os.fsync(descriptor) finally: os.close(descriptor) libc = ctypes.CDLL(None, use_errno=True) renameat2 = getattr(libc, "renameat2", None) if renameat2 is None: raise SystemExit(1) renameat2.argtypes = [ ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint, ] renameat2.restype = ctypes.c_int temporary_name = os.path.basename(temporary) result = renameat2( directory_fd, os.fsencode(temporary_name), directory_fd, os.fsencode(destination_name), 1, # RENAME_NOREPLACE ) if result != 0: error = ctypes.get_errno() if error != errno.EEXIST: raise OSError(error, os.strerror(error)) attest_destination() else: temporary = "" os.fsync(directory_fd) attest_destination() finally: if temporary: try: os.unlink(temporary) os.fsync(directory_fd) except FileNotFoundError: pass os.close(directory_fd) PY2 } install_openclaw_gateway_authorization_fence_dropin() { local expected_system='[Unit] ConditionPathExists=!/var/lib/bridgesllm/openclaw-gateway-authorization-fence.v1 [Service] KillMode=control-group ' local expected_root_user='[Unit] ConditionPathExists=!/var/lib/bridgesllm/openclaw-gateway-authorization-fence.v1 [Service] KillMode=control-group ExecCondition=/usr/bin/false ' install_exact_openclaw_gateway_authorization_fence_dropin \ "${OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_DROPIN_DIR}" \ "${OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_DROPIN}" \ "${expected_system}" \ || return 1 install_exact_openclaw_gateway_authorization_fence_dropin \ "${OPENCLAW_GATEWAY_ROOT_USER_AUTHORIZATION_FENCE_DROPIN_DIR}" \ "${OPENCLAW_GATEWAY_ROOT_USER_AUTHORIZATION_FENCE_DROPIN}" \ "${expected_root_user}" \ || return 1 systemctl daemon-reload >> "${LOG_FILE}" 2>&1 || return 1 reload_active_root_user_systemd_manager || return 1 } remove_openclaw_gateway_authorization_fence_for_clean_uninstall() { local marker_path dropin_dir dropin_path root_user_dropin_dir root_user_dropin_path marker_path="$( update_transaction_state_path \ "${OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_MARKER}" )" || return 1 dropin_dir="$( update_transaction_state_path \ "${OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_DROPIN_DIR}" )" || return 1 dropin_path="$( update_transaction_state_path \ "${OPENCLAW_GATEWAY_AUTHORIZATION_FENCE_DROPIN}" )" || return 1 root_user_dropin_dir="$( update_transaction_state_path \ "${OPENCLAW_GATEWAY_ROOT_USER_AUTHORIZATION_FENCE_DROPIN_DIR}" )" || return 1 root_user_dropin_path="$( update_transaction_state_path \ "${OPENCLAW_GATEWAY_ROOT_USER_AUTHORIZATION_FENCE_DROPIN}" )" || return 1 local expected_dropin='[Unit] ConditionPathExists=!/var/lib/bridgesllm/openclaw-gateway-authorization-fence.v1 [Service] KillMode=control-group ' local expected_root_user_dropin='[Unit] ConditionPathExists=!/var/lib/bridgesllm/openclaw-gateway-authorization-fence.v1 [Service] KillMode=control-group ExecCondition=/usr/bin/false ' local expected_marker='{"schema":"bridgesllm.openclaw-gateway-authorization-fence.v1","unit":"openclaw-gateway.service"} ' python3 - \ "${marker_path}" "${dropin_dir}" "${dropin_path}" \ "${root_user_dropin_dir}" "${root_user_dropin_path}" \ "${expected_marker}" "${expected_dropin}" \ "${expected_root_user_dropin}" <<'PY2' \ || return 1 import os import stat import sys ( marker, dropin_directory, dropin, root_user_dropin_directory, root_user_dropin, marker_text, dropin_text, root_user_dropin_text, ) = sys.argv[1:] expected_marker = marker_text.encode("utf-8") expected_dropin = dropin_text.encode("utf-8") expected_root_user_dropin = root_user_dropin_text.encode("utf-8") if ( os.geteuid() != 0 or not all(os.path.isabs(path) and os.path.normpath(path) == path for path in ( marker, dropin_directory, dropin, root_user_dropin_directory, root_user_dropin, )) or os.path.dirname(dropin) != dropin_directory or os.path.dirname(root_user_dropin) != root_user_dropin_directory or os.path.basename(marker) != "openclaw-gateway-authorization-fence.v1" or os.path.basename(dropin) != "20-bridgesllm-authorization-fence.conf" or os.path.basename(root_user_dropin) != "20-bridgesllm-authorization-fence.conf" ): raise SystemExit(1) def open_attested_parent(path): parent = os.path.dirname(path) for candidate in ( os.path.sep, *(os.path.join( os.path.sep, *parent.strip(os.path.sep).split(os.path.sep)[:index], ) for index in range( 1, len(parent.strip(os.path.sep).split(os.path.sep)) + 1, )), ): try: details = os.lstat(candidate) except FileNotFoundError: return None if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != 0 or details.st_gid != 0 or details.st_mode & 0o022 ): raise SystemExit(1) return os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) def handle_exact(path, expected, required_mode, remove): parent_fd = open_attested_parent(path) if parent_fd is None: return name = os.path.basename(path) try: try: details = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) except FileNotFoundError: return if ( not stat.S_ISREG(details.st_mode) or details.st_uid != 0 or details.st_gid != 0 or stat.S_IMODE(details.st_mode) != required_mode or details.st_nlink != 1 or details.st_size != len(expected) ): raise SystemExit(1) descriptor = os.open( name, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=parent_fd, ) try: opened = os.fstat(descriptor) if ( (opened.st_dev, opened.st_ino, opened.st_mode, opened.st_nlink) != (details.st_dev, details.st_ino, details.st_mode, details.st_nlink) or opened.st_uid != 0 or opened.st_gid != 0 or opened.st_size != len(expected) ): raise SystemExit(1) content = bytearray() while len(content) < len(expected) + 1: block = os.read(descriptor, len(expected) + 1 - len(content)) if not block: break content.extend(block) if bytes(content) != expected: raise SystemExit(1) finally: os.close(descriptor) if not remove: return rebound = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) if ( rebound.st_dev, rebound.st_ino, rebound.st_mode, rebound.st_nlink, ) != ( details.st_dev, details.st_ino, details.st_mode, details.st_nlink, ): raise SystemExit(1) os.unlink(name, dir_fd=parent_fd) os.fsync(parent_fd) finally: os.close(parent_fd) # Prove the complete managed set before deleting any member, then unlink both # drop-ins first. Until the marker is removed, both managers' currently loaded # conditions remain start inhibitors even though the file entries are gone. handle_exact(marker, expected_marker, 0o600, False) handle_exact(dropin, expected_dropin, 0o644, False) handle_exact( root_user_dropin, expected_root_user_dropin, 0o644, False, ) handle_exact( root_user_dropin, expected_root_user_dropin, 0o644, True, ) handle_exact(dropin, expected_dropin, 0o644, True) handle_exact(marker, expected_marker, 0o600, True) PY2 # Clean-slate teardown has already removed INSTALL_ROOT (and its log # directory) before terminal authority is released. systemctl daemon-reload >/dev/null 2>&1 || return 1 reload_active_root_user_systemd_manager /dev/null || return 1 } install_portal_update_boot_fence() { local active_journal dropin_dir dropin_path active_journal="$(update_transaction_state_path "${UPDATE_ACTIVE_JOURNAL}")" \ || return 1 dropin_dir="$( update_transaction_state_path "${UPDATE_BOOT_FENCE_DROPIN_DIR}" )" || return 1 dropin_path="$( update_transaction_state_path "${UPDATE_BOOT_FENCE_DROPIN}" )" || return 1 local expected="[Unit] ConditionPathExists=!${active_journal} " python3 - "${dropin_dir}" "${dropin_path}" "${expected}" <<'PY2' import os import stat import sys import tempfile directory, destination, expected = sys.argv[1:] parent = os.path.dirname(directory) for path in (parent,): info = os.lstat(path) if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022): raise SystemExit(1) try: info = os.lstat(directory) except FileNotFoundError: os.mkdir(directory, 0o755) info = os.lstat(directory) if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022): raise SystemExit(1) os.chmod(directory, 0o755) try: current = os.lstat(destination) except FileNotFoundError: current = None if current is not None and (not stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) or current.st_uid != 0 or current.st_gid != 0 or current.st_nlink != 1 or current.st_mode & 0o022 or current.st_size > 4096): raise SystemExit(1) if current is not None: with open(destination, "r", encoding="utf-8") as handle: if handle.read() == expected: raise SystemExit(0) raise SystemExit(1) fd, temporary = tempfile.mkstemp(prefix=".20-update-fence.", dir=directory) try: os.fchmod(fd, 0o644) with os.fdopen(fd, "w", encoding="utf-8") as handle: handle.write(expected) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, destination) temporary = "" directory_fd = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY2 systemctl daemon-reload >> "${LOG_FILE}" 2>&1 || return 1 [[ -f "${dropin_path}" && ! -L "${dropin_path}" ]] || return 1 cmp -s "${dropin_path}" <(printf '%s' "${expected}") || return 1 } fence_portal_service_boot() { install_portal_update_boot_fence } restore_portal_service_boot_state() { local service_name="${1:-bridgesllm-product}" if ${UPDATE_TRANSACTION_PORTAL_WAS_ENABLED:-false}; then systemctl enable "${service_name}" >> "${LOG_FILE}" 2>&1 || return 1 [[ "$(systemctl is-enabled "${service_name}" 2>/dev/null || true)" == "enabled" ]] else systemctl disable "${service_name}" >> "${LOG_FILE}" 2>&1 || return 1 [[ "$(systemctl is-enabled "${service_name}" 2>/dev/null || true)" == "disabled" ]] fi } update_candidate_unit_name() { local transaction_id="${1:-${UPDATE_TRANSACTION_ID}}" [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 printf 'bridgesllm-update-candidate-%s\n' "${transaction_id}" } start_update_candidate() { local transaction_id="${1:-${UPDATE_TRANSACTION_ID}}" local candidate_port="${2:-${UPDATE_CANDIDATE_PORT}}" local env_file="${PORTAL_DIR}/backend/.env.production" local unit_name unit_name="$(update_candidate_unit_name "${transaction_id}")" || return 1 [[ -f "${env_file}" && ! -L "${env_file}" ]] || return 1 portal_service_port_is_free "${candidate_port}" || return 1 systemd-run \ --unit="${unit_name}" \ --description="BridgesLLM Portal private update candidate ${transaction_id}" \ --working-directory="${PORTAL_DIR}/backend" \ --property=Type=exec \ --property=User=root \ --property=Group=root \ --property=Restart=no \ --property=KillMode=control-group \ --property=RuntimeMaxSec=1200 \ --property=TimeoutStopSec=30 \ --property="EnvironmentFile=${env_file}" \ /usr/bin/env \ "HOST=127.0.0.1" \ "PORT=${candidate_port}" \ "PORTAL_UPDATE_VALIDATION_MODE=1" \ /usr/bin/node dist/server.js >> "${LOG_FILE}" 2>&1 || return 1 UPDATE_TRANSACTION_CANDIDATE_UNIT="${unit_name}" local waited=0 while (( waited < 30 )); do systemctl is-active --quiet "${unit_name}" && return 0 if systemctl is-failed --quiet "${unit_name}"; then journalctl -u "${unit_name}" -n 80 --no-pager >> "${LOG_FILE}" 2>&1 || true return 1 fi sleep 1 waited=$((waited + 1)) done return 1 } portal_runtime_supports_update_validation_contract() { local portal_root="${1:-${PORTAL_DIR}}" local marker="${portal_root}/installer/update-validation-protocol-v1" local server="${portal_root}/backend/dist/server.js" python3 - "${portal_root}" "${marker}" "${server}" <<'PY2' import os import stat import sys root, marker, server = sys.argv[1:] expected = b"BRIDGESLLM_UPDATE_VALIDATION_CONTRACT_V1\n" if not os.path.isabs(root) or root != os.path.normpath(root): raise SystemExit(1) for path, maximum in ((marker, 128), (server, 64 * 1024 * 1024)): try: details = os.lstat(path) except OSError: # Runtimes built before the validation contract simply do not # support it; that is a truthful negative, not a crash. raise SystemExit(1) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != 0 or details.st_nlink != 1 or details.st_mode & 0o022 or not 0 < details.st_size <= maximum ): raise SystemExit(1) if open(marker, "rb").read() != expected: raise SystemExit(1) if expected.rstrip() not in open(server, "rb").read(): raise SystemExit(1) PY2 } verify_update_database_schema_exact() { # Migration checksums alone do not prove that Prisma-managed constraints, # indexes, defaults, foreign keys, or enums still exist. Compare the live # database to the signed datamodel with Prisma's read-only diff engine. The # database URL travels over stdin into a minimal child environment and is # never placed in a process argument. local db_url="$1" local backend_dir="${2:-${PORTAL_DIR}/backend}" local schema="${backend_dir}/prisma/schema.prisma" local prisma_cli="${backend_dir}/node_modules/prisma/build/index.js" [[ -n "${db_url}" ]] || return 1 pg_url_component "${db_url}" host >/dev/null \ && pg_url_component "${db_url}" port >/dev/null \ && pg_url_component "${db_url}" database >/dev/null \ && pg_url_component "${db_url}" user >/dev/null \ || return 1 python3 - "${backend_dir}" "${schema}" "${prisma_cli}" <<'PY2' \ >> "${LOG_FILE}" 2>&1 || return 1 import os import stat import sys backend, schema, prisma_cli = sys.argv[1:] for path, kind, maximum in ( (backend, "directory", 0), (schema, "file", 1024 * 1024), (prisma_cli, "file", 16 * 1024 * 1024), ): details = os.lstat(path) valid_kind = ( stat.S_ISDIR(details.st_mode) if kind == "directory" else stat.S_ISREG(details.st_mode) and 0 < details.st_size <= maximum ) if ( not valid_kind or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_mode & 0o022 ): raise SystemExit(1) PY2 local runner_source runner_source="$( cat <<'PY2' import os import subprocess import sys import threading backend, schema, prisma_cli = sys.argv[1:] secret = os.environ.get("DATABASE_URL", "").encode() if ( not secret or len(secret) > 128000 or any(byte < 32 or byte == 127 for byte in secret) ): raise SystemExit(1) command = [ "/usr/bin/node", prisma_cli, "migrate", "diff", "--exit-code", "--from-schema-datasource", schema, "--to-schema-datamodel", schema, ] process = subprocess.Popen( command, cwd=backend, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) maximum_output = 1024 * 1024 captured = bytearray() output_overflow = False def drain_output(): global output_overflow while True: chunk = process.stdout.read(65536) if not chunk: return remaining = maximum_output - len(captured) if remaining > 0: captured.extend(chunk[:remaining]) if len(chunk) > remaining: output_overflow = True reader = threading.Thread(target=drain_output, daemon=True) reader.start() try: result = process.wait(timeout=120) except subprocess.TimeoutExpired: print("Prisma schema drift verification timed out", file=sys.stderr) raise SystemExit(1) reader.join(timeout=5) if reader.is_alive(): print("Prisma schema drift output did not close", file=sys.stderr) raise SystemExit(1) sanitized = bytes(captured).replace(secret, b"[redacted DATABASE_URL]") if sanitized: sys.stderr.buffer.write(sanitized) if not sanitized.endswith(b"\n"): sys.stderr.buffer.write(b"\n") if output_overflow: print("Prisma schema drift output exceeded its bound", file=sys.stderr) raise SystemExit(1) if result != 0: print( f"Prisma schema drift verification failed with exit {result}", file=sys.stderr, ) raise SystemExit(1) PY2 )" || return 1 run_attested_database_operation \ "${db_url}" prisma-schema-diff \ python3 -c "${runner_source}" "${backend_dir}" "${schema}" "${prisma_cli}" \ >> "${LOG_FILE}" 2>&1 } stop_update_candidate() { local transaction_id="${1:-${UPDATE_TRANSACTION_ID}}" local candidate_port="${2:-${UPDATE_CANDIDATE_PORT}}" local unit_name state unit_name="$(update_candidate_unit_name "${transaction_id}")" || return 1 state="$(systemctl show "${unit_name}" -p LoadState --value 2>/dev/null || true)" if [[ "${state}" == "loaded" ]]; then systemctl stop --no-block "${unit_name}" >> "${LOG_FILE}" 2>&1 || true local waited=0 while (( waited < 30 )); do # systemd garbage-collects transient units the moment they stop, so # "no longer loaded" is the expected terminal state, not a failure. state="$(systemctl show "${unit_name}" -p LoadState --value 2>/dev/null || true)" if [[ "${state}" != "loaded" ]] \ && portal_service_port_is_free "${candidate_port}"; then UPDATE_TRANSACTION_CANDIDATE_UNIT="" return 0 fi if [[ "${state}" == "loaded" ]] \ && portal_service_is_quiesced "${unit_name}" \ && portal_service_port_is_free "${candidate_port}"; then systemctl reset-failed "${unit_name}" >> "${LOG_FILE}" 2>&1 || true UPDATE_TRANSACTION_CANDIDATE_UNIT="" return 0 fi sleep 1 waited=$((waited + 1)) done state="$(systemctl show "${unit_name}" -p LoadState --value 2>/dev/null || true)" if [[ "${state}" == "loaded" ]]; then systemctl kill --kill-who=all --signal=SIGKILL "${unit_name}" >> "${LOG_FILE}" 2>&1 \ || return 1 sleep 1 state="$(systemctl show "${unit_name}" -p LoadState --value 2>/dev/null || true)" if [[ "${state}" == "loaded" ]]; then portal_service_is_quiesced "${unit_name}" || return 1 fi fi fi portal_service_port_is_free "${candidate_port}" || return 1 UPDATE_TRANSACTION_CANDIDATE_UNIT="" } verify_update_candidate() { local transaction_id="${1:-${UPDATE_TRANSACTION_ID}}" local expected_version="${2:-${VERSION}}" local probe_token="${3:-${PORTAL_UPDATE_PROBE_TOKEN}}" local candidate_port="${4:-${UPDATE_CANDIDATE_PORT}}" local unit_name db_url unit_name="$(update_candidate_unit_name "${transaction_id}")" || return 1 portal_runtime_supports_update_validation_contract "${PORTAL_DIR}" \ || return 1 db_url="$(read_env_value \ "${PORTAL_DIR}/backend/.env.production" DATABASE_URL)" || return 1 verify_portal_service_health \ "${unit_name}" "http://127.0.0.1:${candidate_port}/health" 60 \ && verify_portal_update_readiness \ "${expected_version}" "${probe_token}" 900 "${unit_name}" \ "http://127.0.0.1:${candidate_port}/health/update-ready" candidate \ && verify_update_database_schema_exact "${db_url}" "${PORTAL_DIR}/backend" } verify_canonical_portal_routes_for_transaction() { local expected_version="$1" use_local_profile && return 0 if use_tailnet_profile; then verify_tailnet_tls_ready return fi [[ -n "${DOMAIN}" ]] || return 1 verify_portal_tls_ready || return 1 [[ "${expected_version}" == 3.* ]] || verify_app_content_caddy_ready } start_canonical_portal_for_transaction() { if ${UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE:-false}; then systemctl start bridgesllm-product >> "${LOG_FILE}" 2>&1 else systemctl stop bridgesllm-product >> "${LOG_FILE}" 2>&1 || true portal_service_is_quiesced "bridgesllm-product" \ && portal_service_port_is_free 4001 fi } verify_canonical_portal_for_transaction() { local expected_version="$1" probe_token="$2" local allow_legacy_validation="${3:-false}" if ${UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE:-false}; then verify_portal_service_health \ "bridgesllm-product" "http://127.0.0.1:4001/health" 60 || return 1 if [[ -n "${probe_token}" ]]; then local readiness_mode="canonical" if ! portal_runtime_supports_update_validation_contract "${PORTAL_DIR}"; then [[ "${allow_legacy_validation}" == "true" \ && "${expected_version}" == 4.* ]] || return 1 readiness_mode="legacy-canonical" fi verify_portal_update_readiness \ "${expected_version}" "${probe_token}" 900 "bridgesllm-product" \ "http://127.0.0.1:4001/health/update-ready" "${readiness_mode}" || return 1 else [[ "${expected_version}" == 3.* ]] || return 1 [[ "$(attest_existing_portal_for_update "${PORTAL_DIR}")" \ == "${expected_version}" ]] || return 1 fi verify_canonical_portal_routes_for_transaction "${expected_version}" || return 1 # Replacement identity is (boot id, pid, start time): a recycled PID on # the same boot must not defeat the proof, and after a reboot every # process is provably new regardless of its number. local candidate_pid candidate_start current_boot candidate_pid="$(systemctl show bridgesllm-product -p MainPID --value 2>/dev/null || true)" [[ "${candidate_pid}" =~ ^[1-9][0-9]*$ ]] || return 1 candidate_start="$(process_start_time_of_pid "${candidate_pid}")" || return 1 current_boot="$(current_boot_id)" || return 1 if [[ "${current_boot}" == "${UPDATE_TRANSACTION_BASELINE_BOOT_ID:-}" ]]; then [[ "${candidate_pid}" != "${UPDATE_TRANSACTION_BASELINE_PID:-0}" \ || "${candidate_start}" != "${UPDATE_TRANSACTION_BASELINE_START_TIME:-0}" ]] \ || return 1 fi else portal_service_is_quiesced "bridgesllm-product" \ && portal_service_port_is_free 4001 fi } restore_and_verify_canonical_portal_state() { local expected_version="$1" probe_token="$2" restore_portal_service_boot_state "bridgesllm-product" || return 1 start_canonical_portal_for_transaction \ && verify_canonical_portal_for_transaction \ "${expected_version}" "${probe_token}" true } canonical_portal_is_serving_version() { # Deliberately narrow: proves only that the canonical service is running and # answering as the expected version. Used to decide whether a failed # verification may safely tear the Portal back down. local expected_version="$1" [[ -n "${expected_version}" ]] || return 1 systemctl is-active --quiet bridgesllm-product || return 1 [[ "$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' \ http://127.0.0.1:4001/health 2>/dev/null || true)" == "200" ]] || return 1 [[ "$(attest_existing_portal_for_update "${PORTAL_DIR}" 2>/dev/null || true)" \ == "${expected_version}" ]] } update_transaction_helper_path() { update_transaction_state_path "${UPDATE_STATE_HELPER}" } run_update_transaction_state_helper() { local helper test_root="" helper="$(update_transaction_helper_path)" || return 1 [[ -f "${helper}" && ! -L "${helper}" ]] || return 1 test_root="$(update_transaction_test_root 2>/dev/null || true)" if [[ -n "${test_root}" ]]; then python3 "${helper}" --test-root "${test_root}" "$@" else python3 "${helper}" "$@" fi } install_update_transaction_state_helper() { local staged_portal="$1" local source="${staged_portal}/installer/update-transaction-state.py" local destination state_root caddy_source caddy_destination destination="$(update_transaction_helper_path)" || return 1 state_root="$(dirname "${destination}")" python3 - "${source}" "${state_root}" "${destination}" <<'PY2' import os import stat import sys import tempfile source, state_root, destination = sys.argv[1:] expected_uid = os.geteuid() source_info = os.lstat(source) if ( not stat.S_ISREG(source_info.st_mode) or stat.S_ISLNK(source_info.st_mode) or source_info.st_uid != expected_uid or source_info.st_nlink != 1 or not 1 <= source_info.st_size <= 256 * 1024 ): raise SystemExit(1) payload = open(source, "rb").read() if b"\x00" in payload or not payload.startswith(b"#!/usr/bin/env python3\n"): raise SystemExit(1) parent = os.path.dirname(state_root) parent_info = os.lstat(parent) if ( not stat.S_ISDIR(parent_info.st_mode) or stat.S_ISLNK(parent_info.st_mode) or parent_info.st_uid != expected_uid or parent_info.st_mode & 0o022 ): raise SystemExit(1) try: os.mkdir(state_root, 0o700) except FileExistsError: pass state_info = os.lstat(state_root) if ( not stat.S_ISDIR(state_info.st_mode) or stat.S_ISLNK(state_info.st_mode) or state_info.st_uid != expected_uid or state_info.st_gid != os.getegid() or state_info.st_mode & 0o077 ): raise SystemExit(1) os.chmod(state_root, 0o700) for receipt in ("active-update.json", "cutover-update.json"): try: os.lstat(os.path.join(state_root, receipt)) except FileNotFoundError: continue raise SystemExit(1) try: current = os.lstat(destination) except FileNotFoundError: current = None if current is not None and ( not stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) or current.st_uid != expected_uid or current.st_nlink != 1 or current.st_mode & 0o077 ): raise SystemExit(1) descriptor, temporary = tempfile.mkstemp( prefix=".update-transaction-state.", dir=state_root ) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, destination) temporary = "" directory_fd = os.open( state_root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY2 python3 "${destination}" --help >/dev/null 2>&1 || return 1 caddy_source="${staged_portal}/installer/caddy-managed-config.py" caddy_destination="$( update_transaction_state_path "${UPDATE_CADDY_RECOVERY_HELPER}" )" || return 1 python3 - "${caddy_source}" "${caddy_destination}" <<'PY2' import os import stat import sys import tempfile source, destination = sys.argv[1:] expected_uid = os.geteuid() details = os.lstat(source) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != expected_uid or details.st_nlink != 1 or not 1 <= details.st_size <= 256 * 1024 ): raise SystemExit(1) payload = open(source, "rb").read() if b"\x00" in payload or not payload.startswith(b"#!/usr/bin/env python3\n"): raise SystemExit(1) parent = os.path.dirname(destination) parent_details = os.lstat(parent) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_uid != expected_uid or parent_details.st_mode & 0o077 ): raise SystemExit(1) try: current = os.lstat(destination) except FileNotFoundError: current = None if current is not None and ( not stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) or current.st_uid != expected_uid or current.st_nlink != 1 or current.st_mode & 0o077 ): raise SystemExit(1) descriptor, temporary = tempfile.mkstemp(prefix=".caddy-recovery.", dir=parent) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, destination) temporary = "" directory_fd = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY2 python3 "${caddy_destination}" --help >/dev/null 2>&1 || return 1 } read_update_transaction_field() { local target="$1" field="$2" run_update_transaction_state_helper read \ --target "${target}" --field "${field}" } update_artifact_manifest() { local action="$1" artifact_path="$2" manifest_path="$3" local allow_absent="${4:-false}" [[ "${action}" == "create" || "${action}" == "verify" ]] || return 1 [[ "${allow_absent}" == "true" || "${allow_absent}" == "false" ]] || return 1 python3 - "${action}" "${artifact_path}" "${manifest_path}" "${allow_absent}" <<'PY2' import hashlib import hmac import json import os import re import stat import sys import tempfile action, artifact_path, manifest_path, allow_absent_raw = sys.argv[1:] allow_absent = allow_absent_raw == "true" expected_uid = os.geteuid() expected_gid = os.getegid() max_entries = 350_000 max_bytes = 20 * 1024 * 1024 * 1024 max_manifest = 128 * 1024 * 1024 def fail() -> None: raise SystemExit(1) def validate_text(value: str) -> None: if any(ord(char) < 32 or ord(char) == 127 for char in value): fail() def xattr_identity(path: str) -> list[list[str]]: values: list[list[str]] = [] try: names = sorted(os.listxattr(path, follow_symlinks=False)) for name in names: validate_text(name) value = os.getxattr(path, name, follow_symlinks=False) values.append([name, hashlib.sha256(value).hexdigest()]) except OSError: fail() return values def regular_digest(path: str, before: os.stat_result) -> str: flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) try: descriptor = os.open(path, flags) except OSError: fail() digest = hashlib.sha256() try: opened = os.fstat(descriptor) if ( not stat.S_ISREG(opened.st_mode) or opened.st_dev != before.st_dev or opened.st_ino != before.st_ino or opened.st_nlink != 1 ): fail() while True: chunk = os.read(descriptor, 1024 * 1024) if not chunk: break digest.update(chunk) after = os.fstat(descriptor) if ( after.st_dev != opened.st_dev or after.st_ino != opened.st_ino or after.st_size != opened.st_size or after.st_mtime_ns != opened.st_mtime_ns or after.st_ctime_ns != opened.st_ctime_ns ): fail() finally: os.close(descriptor) return digest.hexdigest() def entry(path: str, relative: str, root: str) -> tuple[dict[str, object], int]: validate_text(relative) try: details = os.lstat(path) except OSError: fail() if ( details.st_uid != expected_uid or details.st_gid != expected_gid or (not stat.S_ISLNK(details.st_mode) and details.st_mode & 0o022) ): fail() common: dict[str, object] = { "path": relative, "mode": stat.S_IMODE(details.st_mode), "uid": details.st_uid, "gid": details.st_gid, "mtime_ns": details.st_mtime_ns, "xattrs": xattr_identity(path), } if stat.S_ISDIR(details.st_mode): common["type"] = "directory" common["size"] = 0 return common, 0 if stat.S_ISREG(details.st_mode): if details.st_nlink != 1: fail() common["type"] = "file" common["size"] = details.st_size common["sha256"] = regular_digest(path, details) return common, details.st_size if stat.S_ISLNK(details.st_mode): try: target = os.readlink(path) except OSError: fail() validate_text(target) if os.path.isabs(target): fail() resolved = os.path.normpath(os.path.join(os.path.dirname(path), target)) if os.path.commonpath((root, resolved)) != root: fail() common["type"] = "symlink" common["size"] = len(os.fsencode(target)) common["target"] = target return common, len(os.fsencode(target)) fail() def scan() -> dict[str, object]: try: root_details = os.lstat(artifact_path) except FileNotFoundError: if not allow_absent: fail() return { "schema": "bridgesllm-update-artifact-v1", "state": "absent", "entries": [], "total_bytes": 0, } except OSError: fail() if stat.S_ISLNK(root_details.st_mode): fail() root = os.path.normpath(os.path.abspath(artifact_path)) rows: list[dict[str, object]] = [] total_bytes = 0 if stat.S_ISDIR(root_details.st_mode): root_identity = (root_details.st_dev, root_details.st_ino) for current, directories, files in os.walk(root, topdown=True, followlinks=False): directories.sort() files.sort() relative_current = os.path.relpath(current, root) row, consumed = entry(current, relative_current, root) rows.append(row) total_bytes += consumed for name in [*directories, *files]: path = os.path.join(current, name) if os.path.islink(path): relative = os.path.relpath(path, root) row, consumed = entry(path, relative, root) rows.append(row) total_bytes += consumed directories[:] = [ name for name in directories if not os.path.islink(os.path.join(current, name)) ] for name in files: path = os.path.join(current, name) if os.path.islink(path): continue relative = os.path.relpath(path, root) row, consumed = entry(path, relative, root) rows.append(row) total_bytes += consumed if len(rows) > max_entries or total_bytes > max_bytes: fail() final_root = os.lstat(root) if (final_root.st_dev, final_root.st_ino) != root_identity: fail() else: row, consumed = entry(root, ".", root) rows.append(row) total_bytes += consumed rows.sort(key=lambda item: os.fsencode(str(item["path"]))) record: dict[str, object] = { "schema": "bridgesllm-update-artifact-v1", "state": "present", "entries": rows, "total_bytes": total_bytes, } return record def canonical(record: dict[str, object]) -> bytes: return ( json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n" ).encode("ascii") record = scan() identity = hashlib.sha256(canonical(record)).hexdigest() record["integrity_sha256"] = identity payload = canonical(record) if len(payload) > max_manifest: fail() parent = os.path.dirname(manifest_path) if action == "create": try: parent_details = os.lstat(parent) except FileNotFoundError: os.mkdir(parent, 0o700) parent_details = os.lstat(parent) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_uid != expected_uid or parent_details.st_gid != expected_gid or parent_details.st_mode & 0o077 ): fail() descriptor, temporary = tempfile.mkstemp(prefix=".artifact.", dir=parent) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, manifest_path) temporary = "" directory_fd = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass else: try: details = os.lstat(manifest_path) except OSError: fail() if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != expected_uid or details.st_gid != expected_gid or details.st_nlink != 1 or stat.S_IMODE(details.st_mode) != 0o600 or not 1 <= details.st_size <= max_manifest ): fail() try: stored = open(manifest_path, "rb").read() except OSError: fail() if not hmac.compare_digest(stored, payload): fail() PY2 } update_transaction_manifest_path() { local target="$1" name="$2" [[ "${name}" =~ ^[a-z][a-z0-9-]{0,63}$ ]] || return 1 local transaction_dir transaction_dir="$(read_update_transaction_field "${target}" transaction_dir)" \ || return 1 printf '%s/manifests/%s.json\n' "${transaction_dir}" "${name}" } capture_update_running_app_intent() { local db_url="$1" local backup_dir transaction_id payload manifest raw connection_url backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 transaction_id="$( read_update_transaction_field active transaction_id )" || return 1 payload="${backup_dir}/running-app-intent.json" manifest="$( update_transaction_manifest_path active running-app-intent )" || return 1 [[ ! -e "${payload}" && ! -L "${payload}" \ && ! -e "${manifest}" && ! -L "${manifest}" ]] || return 1 connection_url="$(libpq_database_url "${db_url}")" || return 1 raw="$(mktemp "${backup_dir}/.running-app-intent.raw.XXXXXX")" \ || return 1 chmod 0600 "${raw}" || { rm -f -- "${raw}"; return 1; } assert_update_database_topology_unchanged "${db_url}" app-intent-capture \ || { rm -f -- "${raw}"; return 1; } if ! run_with_update_pgpass "${db_url}" \ psql --dbname="${connection_url}" --no-psqlrc \ -v ON_ERROR_STOP=1 -qAtc ' SET search_path TO pg_catalog; SELECT COALESCE(json_agg(json_build_object( '\''id'\'', app."id", '\''userId'\'', app."userId", '\''name'\'', app."name", '\''zipPath'\'', app."zipPath", '\''isActive'\'', app."isActive", '\''deployType'\'', app."deployType", '\''port'\'', app."port", '\''processStatus'\'', app."processStatus" ) ORDER BY app."id"), '\''[]'\''::json)::text FROM public."App" AS app WHERE app."deployType" = '\''fullstack'\'' AND app."processStatus" IN ('\''running'\'', '\''starting'\''); ' > "${raw}" 2>> "${LOG_FILE}"; then rm -f -- "${raw}" return 1 fi assert_update_database_topology_unchanged \ "${db_url}" app-intent-capture-complete \ || { rm -f -- "${raw}"; return 1; } if ! python3 - "${raw}" "${payload}" "${transaction_id}" <<'PY2' import json import os import pathlib import stat import sys import tempfile source = pathlib.Path(sys.argv[1]) destination = pathlib.Path(sys.argv[2]) transaction_id = sys.argv[3] try: rows = json.loads(source.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError): raise SystemExit(1) if not isinstance(rows, list) or len(rows) > 100_000: raise SystemExit(1) expected_keys = { "id", "userId", "name", "zipPath", "isActive", "deployType", "port", "processStatus", } seen = set() for row in rows: if not isinstance(row, dict) or set(row) != expected_keys: raise SystemExit(1) if ( not all( isinstance(row[key], str) and 0 < len(row[key].encode("utf-8")) <= 4096 and "\x00" not in row[key] for key in ("id", "userId", "name", "zipPath") ) or row["id"] in seen or row["deployType"] != "fullstack" or row["processStatus"] not in {"running", "starting"} or not isinstance(row["isActive"], bool) or not isinstance(row["port"], int) or isinstance(row["port"], bool) or not 1 <= row["port"] <= 65535 or not os.path.isabs(row["zipPath"]) or os.path.normpath(row["zipPath"]) != row["zipPath"] ): raise SystemExit(1) seen.add(row["id"]) document = { "schema": "bridgesllm-update-running-app-intent-v1", "transactionId": transaction_id, "apps": rows, } encoded = (json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n").encode() descriptor = os.open( destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) try: os.write(descriptor, encoded) os.fchmod(descriptor, 0o600) os.fsync(descriptor) finally: os.close(descriptor) PY2 then rm -f -- "${raw}" "${payload}" return 1 fi rm -f -- "${raw}" fsync_update_payload "${payload}" || return 1 update_artifact_manifest create "${payload}" "${manifest}" false \ && update_artifact_manifest verify "${payload}" "${manifest}" false } reconcile_update_running_app_intent() { local target="${1:-active}" local db_url="$2" local backup_dir transaction_id payload manifest sql connection_url backup_dir="$(read_update_transaction_field "${target}" backup_dir)" \ || return 1 transaction_id="$( read_update_transaction_field "${target}" transaction_id )" || return 1 payload="${backup_dir}/running-app-intent.json" manifest="$( update_transaction_manifest_path "${target}" running-app-intent )" || return 1 update_artifact_manifest verify "${payload}" "${manifest}" false \ || return 1 sql="$(mktemp "${backup_dir}/.running-app-intent.sql.XXXXXX")" \ || return 1 chmod 0600 "${sql}" || { rm -f -- "${sql}"; return 1; } if ! python3 - "${payload}" "${sql}" "${transaction_id}" <<'PY2' import json import os import pathlib import sys source = pathlib.Path(sys.argv[1]) destination = pathlib.Path(sys.argv[2]) transaction_id = sys.argv[3] document = json.loads(source.read_text(encoding="utf-8")) if ( not isinstance(document, dict) or set(document) != {"schema", "transactionId", "apps"} or document.get("schema") != "bridgesllm-update-running-app-intent-v1" or document.get("transactionId") != transaction_id or not isinstance(document.get("apps"), list) ): raise SystemExit(1) encoded = json.dumps(document, sort_keys=True, separators=(",", ":")) literal = encoded.replace("'", "''") statement = f'''BEGIN; SET LOCAL search_path TO pg_catalog; DO $bridgesllm_running_app_intent$ DECLARE item jsonb; current_row record; desired_status text; BEGIN FOR item IN SELECT value FROM jsonb_array_elements('{literal}'::jsonb -> 'apps') LOOP SELECT app."userId", app."name", app."zipPath", app."isActive", app."deployType", app."port", app."processStatus" INTO current_row FROM public."App" AS app WHERE app."id" = item ->> 'id' FOR UPDATE; IF NOT FOUND THEN RAISE EXCEPTION 'captured running App disappeared during update'; END IF; IF current_row."userId" IS DISTINCT FROM item ->> 'userId' OR current_row."name" IS DISTINCT FROM item ->> 'name' OR current_row."zipPath" IS DISTINCT FROM item ->> 'zipPath' OR current_row."isActive" IS DISTINCT FROM (item ->> 'isActive')::boolean OR current_row."deployType" IS DISTINCT FROM item ->> 'deployType' OR current_row."port" IS DISTINCT FROM (item ->> 'port')::integer THEN RAISE EXCEPTION 'captured running App identity changed during update'; END IF; desired_status := item ->> 'processStatus'; IF desired_status NOT IN ('running', 'starting') THEN RAISE EXCEPTION 'captured running App intent is invalid'; END IF; IF current_row."processStatus" = 'stopped' THEN UPDATE public."App" SET "processStatus" = desired_status, "updatedAt" = CURRENT_TIMESTAMP WHERE "id" = item ->> 'id'; ELSIF current_row."processStatus" NOT IN ('running', 'starting') THEN RAISE EXCEPTION 'captured running App entered a non-recoverable state'; END IF; END LOOP; END $bridgesllm_running_app_intent$; COMMIT; ''' descriptor = os.open( destination, os.O_WRONLY | os.O_TRUNC | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.write(descriptor, statement.encode("utf-8")) os.fchmod(descriptor, 0o600) os.fsync(descriptor) finally: os.close(descriptor) PY2 then rm -f -- "${sql}" return 1 fi connection_url="$(libpq_database_url "${db_url}")" \ || { rm -f -- "${sql}"; return 1; } if ! run_attested_database_operation \ "${db_url}" app-intent-reconcile \ psql --dbname="${connection_url}" --no-psqlrc \ -v ON_ERROR_STOP=1 --file="${sql}" \ >> "${LOG_FILE}" 2>&1; then rm -f -- "${sql}" return 1 fi rm -f -- "${sql}" update_artifact_manifest verify "${payload}" "${manifest}" false } snapshot_update_backend_env_alias() { local source="$1" snapshot="$2" python3 - "${source}" "${snapshot}" <<'PY2' import base64 import json import os import stat import sys source, snapshot = sys.argv[1:] kind_path = os.path.join(snapshot, "backend.env.alias.json") file_path = os.path.join(snapshot, "backend.env.alias.file") def xattrs(path: str) -> list[list[str]]: return [ [ name, base64.b64encode( os.getxattr(path, name, follow_symlinks=False) ).decode("ascii"), ] for name in sorted(os.listxattr(path, follow_symlinks=False)) ] try: details = os.lstat(source) except FileNotFoundError: record = { "schema": "bridgesllm-backend-env-alias-v1", "kind": "absent", } else: if stat.S_ISREG(details.st_mode): if details.st_nlink != 1: raise SystemExit(1) source_fd = os.open( source, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) destination_fd = os.open( file_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) try: opened = os.fstat(source_fd) if ( not stat.S_ISREG(opened.st_mode) or opened.st_dev != details.st_dev or opened.st_ino != details.st_ino or opened.st_nlink != 1 ): raise SystemExit(1) os.fchown(destination_fd, os.geteuid(), os.getegid()) os.fchmod(destination_fd, 0o600) while True: chunk = os.read(source_fd, 1024 * 1024) if not chunk: break view = memoryview(chunk) while view: consumed = os.write(destination_fd, view) if consumed <= 0: raise OSError("snapshot write made no progress") view = view[consumed:] os.fsync(destination_fd) after = os.fstat(source_fd) if ( after.st_dev != opened.st_dev or after.st_ino != opened.st_ino or after.st_size != opened.st_size or after.st_mtime_ns != opened.st_mtime_ns or after.st_ctime_ns != opened.st_ctime_ns ): raise SystemExit(1) finally: os.close(source_fd) os.close(destination_fd) record = { "schema": "bridgesllm-backend-env-alias-v1", "kind": "file", "mode": stat.S_IMODE(opened.st_mode), "uid": opened.st_uid, "gid": opened.st_gid, "atimeNs": opened.st_atime_ns, "mtimeNs": opened.st_mtime_ns, "size": opened.st_size, "xattrs": xattrs(source), } elif stat.S_ISLNK(details.st_mode): target = os.fsencode(os.readlink(source)) if len(target) > 4096 or b"\0" in target: raise SystemExit(1) record = { "schema": "bridgesllm-backend-env-alias-v1", "kind": "symlink", "targetBase64": base64.b64encode(target).decode("ascii"), "uid": details.st_uid, "gid": details.st_gid, "atimeNs": details.st_atime_ns, "mtimeNs": details.st_mtime_ns, "xattrs": xattrs(source), } else: raise SystemExit(1) payload = ( json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n" ).encode("ascii") descriptor = os.open( kind_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) try: os.fchmod(descriptor, 0o600) view = memoryview(payload) while view: consumed = os.write(descriptor, view) if consumed <= 0: raise OSError("alias snapshot write made no progress") view = view[consumed:] os.fsync(descriptor) finally: os.close(descriptor) directory_fd = os.open( snapshot, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY2 } restore_update_backend_env_alias() { local snapshot="$1" destination="$2" python3 - "${snapshot}" "${destination}" <<'PY2' import base64 import hashlib import json import os import secrets import stat import sys snapshot, destination = sys.argv[1:] kind_path = os.path.join(snapshot, "backend.env.alias.json") file_path = os.path.join(snapshot, "backend.env.alias.file") try: marker = os.lstat(kind_path) record = json.loads(open(kind_path, "rb").read()) except (OSError, ValueError): raise SystemExit(1) if ( not stat.S_ISREG(marker.st_mode) or stat.S_ISLNK(marker.st_mode) or marker.st_uid != os.geteuid() or marker.st_gid != os.getegid() or marker.st_nlink != 1 or stat.S_IMODE(marker.st_mode) != 0o600 or not isinstance(record, dict) or record.get("schema") != "bridgesllm-backend-env-alias-v1" or record.get("kind") not in {"absent", "file", "symlink"} ): raise SystemExit(1) parent = os.path.dirname(destination) parent_details = os.lstat(parent) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_uid != os.geteuid() or parent_details.st_gid != os.getegid() or parent_details.st_mode & 0o022 ): raise SystemExit(1) def fsync_destination_parent() -> None: descriptor = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) # The promotion helper can be killed after publishing its private temporary # symlink but before rename. Recovery owns and removes only that exact bounded # residue before comparing/restoring the original alias state. temporary_prefix = f".{os.path.basename(destination)}.update-link." removed_residue = False for candidate_name in os.listdir(parent): if not candidate_name.startswith(temporary_prefix): continue candidate = os.path.join(parent, candidate_name) candidate_details = os.lstat(candidate) if ( not stat.S_ISLNK(candidate_details.st_mode) or candidate_details.st_uid != os.geteuid() or candidate_details.st_gid != os.getegid() or os.readlink(candidate) != ".env.production" ): raise SystemExit(1) os.unlink(candidate) removed_residue = True if removed_residue: fsync_destination_parent() def xattr_identity(path: str) -> tuple[tuple[bytes, bytes], ...]: return tuple( ( os.fsencode(name), os.getxattr(path, name, follow_symlinks=False), ) for name in sorted(os.listxattr(path, follow_symlinks=False)) ) def file_identity(path: str) -> tuple[object, ...]: details = os.lstat(path) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_nlink != 1 ): raise OSError("not an exact regular file") digest = hashlib.sha256() descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: opened = os.fstat(descriptor) if ( opened.st_dev != details.st_dev or opened.st_ino != details.st_ino or opened.st_nlink != 1 ): raise OSError("file changed while opening") while True: chunk = os.read(descriptor, 1024 * 1024) if not chunk: break digest.update(chunk) finally: os.close(descriptor) return ( stat.S_IMODE(details.st_mode), details.st_uid, details.st_gid, details.st_size, details.st_mtime_ns, digest.digest(), xattr_identity(path), ) def file_matches_record(path: str) -> bool: try: expected_xattrs = tuple( ( os.fsencode(name), base64.b64decode(value, validate=True), ) for name, value in record["xattrs"] ) expected = ( int(record["mode"]), int(record["uid"]), int(record["gid"]), int(record["size"]), int(record["mtimeNs"]), hashlib.sha256(open(file_path, "rb").read()).digest(), expected_xattrs, ) except (KeyError, OSError, TypeError, ValueError): raise SystemExit(1) try: return file_identity(path) == expected except OSError: return False def original_matches() -> bool: kind = record["kind"] try: current = os.lstat(destination) except FileNotFoundError: return kind == "absent" if kind == "file": return file_matches_record(destination) if kind == "symlink": if not stat.S_ISLNK(current.st_mode): return False try: expected_target = base64.b64decode( record["targetBase64"], validate=True ) expected_xattrs = tuple( ( os.fsencode(name), base64.b64decode(value, validate=True), ) for name, value in record["xattrs"] ) except (KeyError, TypeError, ValueError): raise SystemExit(1) return ( os.fsencode(os.readlink(destination)) == expected_target and current.st_uid == record.get("uid") and current.st_gid == record.get("gid") and current.st_mtime_ns == record.get("mtimeNs") and xattr_identity(destination) == expected_xattrs ) return False if original_matches(): raise SystemExit(0) try: current = os.lstat(destination) except FileNotFoundError: current = None promoted = ( current is not None and stat.S_ISLNK(current.st_mode) and current.st_uid == os.geteuid() and current.st_gid == os.getegid() and os.fsencode(os.readlink(destination)) == b".env.production" ) if not promoted: raise SystemExit(1) kind = record["kind"] temporary = "" if kind == "absent": os.unlink(destination) elif kind == "file": source_details = os.lstat(file_path) if ( not stat.S_ISREG(source_details.st_mode) or stat.S_ISLNK(source_details.st_mode) or source_details.st_nlink != 1 ): raise SystemExit(1) try: desired_mode = int(record["mode"]) desired_uid = int(record["uid"]) desired_gid = int(record["gid"]) desired_atime_ns = int(record["atimeNs"]) desired_mtime_ns = int(record["mtimeNs"]) desired_size = int(record["size"]) desired_xattrs = [ (name, base64.b64decode(value, validate=True)) for name, value in record["xattrs"] ] except (KeyError, TypeError, ValueError): raise SystemExit(1) if ( not 0 <= desired_mode <= 0o7777 or desired_uid < 0 or desired_gid < 0 or desired_atime_ns < 0 or desired_mtime_ns < 0 or desired_size != source_details.st_size ): raise SystemExit(1) for _ in range(16): temporary = os.path.join( parent, f".update-env-file.{secrets.token_hex(12)}" ) try: descriptor = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, ) break except FileExistsError: temporary = "" else: raise SystemExit(1) try: os.fchown(descriptor, desired_uid, desired_gid) os.fchmod(descriptor, desired_mode) with open(file_path, "rb") as source: while True: chunk = source.read(1024 * 1024) if not chunk: break view = memoryview(chunk) while view: consumed = os.write(descriptor, view) if consumed <= 0: raise OSError("restore write made no progress") view = view[consumed:] for name, value in desired_xattrs: os.setxattr( temporary, name, value, follow_symlinks=False, ) os.utime( temporary, ns=(desired_atime_ns, desired_mtime_ns), follow_symlinks=False, ) os.fsync(descriptor) os.close(descriptor) descriptor = -1 os.replace(temporary, destination) temporary = "" finally: if descriptor >= 0: os.close(descriptor) if temporary: try: os.unlink(temporary) except FileNotFoundError: pass else: try: target = base64.b64decode(record["targetBase64"], validate=True) expected_xattrs = [ (name, base64.b64decode(value, validate=True)) for name, value in record["xattrs"] ] uid = int(record["uid"]) gid = int(record["gid"]) atime_ns = int(record["atimeNs"]) mtime_ns = int(record["mtimeNs"]) except (KeyError, TypeError, ValueError): raise SystemExit(1) if len(target) > 4096 or b"\0" in target: raise SystemExit(1) for _ in range(16): temporary = os.path.join( parent, f".update-env-link.{secrets.token_hex(12)}" ) try: os.symlink(os.fsdecode(target), temporary) break except FileExistsError: temporary = "" else: raise SystemExit(1) try: os.lchown(temporary, uid, gid) for name, value in expected_xattrs: os.setxattr(temporary, name, value, follow_symlinks=False) os.utime( temporary, ns=(atime_ns, mtime_ns), follow_symlinks=False, ) os.replace(temporary, destination) temporary = "" finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass fsync_destination_parent() if not original_matches(): raise SystemExit(1) PY2 } snapshot_update_runtime_and_environment() { local backup_dir runtime_snapshot environment_snapshot local runtime_manifest environment_manifest backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 runtime_snapshot="${backup_dir}/portal-runtime" environment_snapshot="${backup_dir}/environment" runtime_manifest="$(update_transaction_manifest_path active runtime)" || return 1 environment_manifest="$(update_transaction_manifest_path active environment)" || return 1 install -d -m 0700 -- "${runtime_snapshot}" "${environment_snapshot}" || return 1 rsync -aHAX --numeric-ids --delete \ --exclude='node_modules' \ --exclude='.git' \ --exclude='.env' \ --exclude='.env.production' \ --exclude='/projects' \ --exclude='/apps' \ --exclude='/assets' \ --exclude='/upload-temp' \ --exclude='/.data' \ --exclude='/backend/.data' \ "${PORTAL_DIR}/" "${runtime_snapshot}/" >> "${LOG_FILE}" 2>&1 || return 1 # Public 3.26.x tarballs shipped runtime files with group/other-writable # modes (0777 vnc_auto.html) and tarball-numeric owners (xfce4-config). # The snapshot manifest contract rightly refuses such entries. Normalize # ONLY the snapshot copies: the live tree stays untouched until cutover, # and a rollback restores exact content with sane root-owned permissions. local snapshot_entry while IFS= read -r -d '' snapshot_entry; do printf 'normalized legacy snapshot permissions: %s\n' "${snapshot_entry}" \ >> "${LOG_FILE}" chown -h root:root -- "${snapshot_entry}" || return 1 [[ -L "${snapshot_entry}" ]] \ || chmod g-w,o-w -- "${snapshot_entry}" || return 1 done < <( find "${runtime_snapshot}" \ \( \( -perm /022 -a ! -type l \) -o ! -user root -o ! -group root \) \ -print0 2>> "${LOG_FILE}" ) || return 1 cp -a -- "${PORTAL_DIR}/backend/.env.production" \ "${environment_snapshot}/backend.env.production" || return 1 cp -a -- "${PORTAL_DIR}/frontend/.env" \ "${environment_snapshot}/frontend.env" || return 1 snapshot_update_backend_env_alias \ "${PORTAL_DIR}/backend/.env" "${environment_snapshot}" || return 1 sync -f "${backup_dir}" || return 1 update_artifact_manifest create "${runtime_snapshot}" "${runtime_manifest}" false \ && update_artifact_manifest verify "${runtime_snapshot}" "${runtime_manifest}" false \ && update_artifact_manifest create "${environment_snapshot}" "${environment_manifest}" false \ && update_artifact_manifest verify "${environment_snapshot}" "${environment_manifest}" false } snapshot_update_dependencies() { local backup_dir snapshot manifest preexisted backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 snapshot="${backup_dir}/backend-node-modules" manifest="$(update_transaction_manifest_path active dependencies)" || return 1 preexisted="$(read_update_transaction_field active node_modules_preexisted)" || return 1 if [[ "${preexisted}" == "true" ]]; then install -d -m 0700 -- "${snapshot}" || return 1 rsync -aHAX --numeric-ids --delete \ "${PORTAL_DIR}/backend/node_modules/" "${snapshot}/" \ >> "${LOG_FILE}" 2>&1 || return 1 sync -f "${snapshot}" || return 1 else [[ ! -e "${snapshot}" && ! -L "${snapshot}" ]] || return 1 fi update_artifact_manifest create "${snapshot}" "${manifest}" true \ && update_artifact_manifest verify "${snapshot}" "${manifest}" true } snapshot_update_caddy() { local backup_dir snapshot manifest required backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 snapshot="${backup_dir}/Caddyfile" manifest="$(update_transaction_manifest_path active caddy)" || return 1 required="$(read_update_transaction_field active caddy_snapshot_required)" || return 1 if [[ "${required}" == "true" ]]; then [[ -f /etc/caddy/Caddyfile && ! -L /etc/caddy/Caddyfile ]] || return 1 cp -a -- /etc/caddy/Caddyfile "${snapshot}" || return 1 sync -f "${backup_dir}" || return 1 else [[ ! -e "${snapshot}" && ! -L "${snapshot}" ]] || return 1 fi update_artifact_manifest create "${snapshot}" "${manifest}" true \ && update_artifact_manifest verify "${snapshot}" "${manifest}" true } snapshot_update_database() { local db_url="$1" local backup_dir manifest contract_manifest backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 manifest="$(update_transaction_manifest_path active database)" || return 1 contract_manifest="$( update_transaction_manifest_path active database-contract )" || return 1 backup_database_for_update "${db_url}" "${backup_dir}" || return 1 snapshot_update_database_contract "${backup_dir}" || return 1 update_artifact_manifest create \ "${backup_dir}/database-before-update.dump" "${manifest}" false \ && update_artifact_manifest verify \ "${backup_dir}/database-before-update.dump" "${manifest}" false \ && update_artifact_manifest create \ "${backup_dir}/database-contract.variant" "${contract_manifest}" false \ && update_artifact_manifest verify \ "${backup_dir}/database-contract.variant" "${contract_manifest}" false } snapshot_update_provenance() { local backup_dir manifest snapshot backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 manifest="$(update_transaction_manifest_path active provenance)" || return 1 capture_portal_deploy_stamp_for_update "${backup_dir}" || return 1 snapshot="${backup_dir}/last-portal-deploy.previous" if [[ -f "${snapshot}" && ! -L "${snapshot}" ]]; then fsync_update_payload "${snapshot}" || return 1 fi update_artifact_manifest create "${snapshot}" "${manifest}" true \ && update_artifact_manifest verify "${snapshot}" "${manifest}" true } update_regular_files_equal() { local first="$1" second="$2" python3 - "${first}" "${second}" <<'PY2' import hashlib import os import stat import sys def identity(path: str) -> tuple[object, ...]: details = os.lstat(path) if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_nlink != 1 ): raise SystemExit(1) digest = hashlib.sha256() with open(path, "rb") as handle: while True: chunk = handle.read(1024 * 1024) if not chunk: break digest.update(chunk) xattrs = tuple( (name, os.getxattr(path, name, follow_symlinks=False)) for name in sorted(os.listxattr(path, follow_symlinks=False)) ) return ( stat.S_IMODE(details.st_mode), details.st_uid, details.st_gid, details.st_size, details.st_mtime_ns, digest.digest(), xattrs, ) try: if identity(sys.argv[1]) != identity(sys.argv[2]): raise SystemExit(1) except OSError: raise SystemExit(1) PY2 } restore_update_runtime_snapshot() { local backup_dir snapshot manifest changes backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 snapshot="${backup_dir}/portal-runtime" manifest="$(update_transaction_manifest_path active runtime)" || return 1 update_artifact_manifest verify "${snapshot}" "${manifest}" false || return 1 rsync -aHAX --numeric-ids --delete \ --exclude='node_modules' \ --exclude='.git' \ --exclude='.env' \ --exclude='.env.production' \ --exclude='/projects' \ --exclude='/apps' \ --exclude='/assets' \ --exclude='/upload-temp' \ --exclude='/.data' \ --exclude='/backend/.data' \ "${snapshot}/" "${PORTAL_DIR}/" >> "${LOG_FILE}" 2>&1 || return 1 sync -f "${PORTAL_DIR}" || return 1 changes="$(rsync -aHAXnc --numeric-ids --delete --itemize-changes \ --exclude='node_modules' \ --exclude='.git' \ --exclude='.env' \ --exclude='.env.production' \ --exclude='/projects' \ --exclude='/apps' \ --exclude='/assets' \ --exclude='/upload-temp' \ --exclude='/.data' \ --exclude='/backend/.data' \ "${snapshot}/" "${PORTAL_DIR}/" 2>> "${LOG_FILE}")" || return 1 [[ -z "${changes}" ]] } atomic_restore_update_file() { local source="$1" destination="$2" python3 - "${source}" "${destination}" <<'PY2' import os import stat import sys import tempfile source, destination = sys.argv[1:] source_details = os.lstat(source) if ( not stat.S_ISREG(source_details.st_mode) or stat.S_ISLNK(source_details.st_mode) or source_details.st_nlink != 1 ): raise SystemExit(1) parent = os.path.dirname(destination) parent_details = os.lstat(parent) if not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode): raise SystemExit(1) try: current = os.lstat(destination) except FileNotFoundError: current = None if current is not None and ( not stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) or current.st_nlink != 1 ): raise SystemExit(1) payload = open(source, "rb").read() xattrs = tuple( (name, os.getxattr(source, name, follow_symlinks=False)) for name in sorted(os.listxattr(source, follow_symlinks=False)) ) descriptor, temporary = tempfile.mkstemp(prefix=".update-restore.", dir=parent) try: os.fchown(descriptor, source_details.st_uid, source_details.st_gid) os.fchmod(descriptor, stat.S_IMODE(source_details.st_mode)) with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) for name, value in xattrs: os.setxattr(temporary, name, value, follow_symlinks=False) os.utime( temporary, ns=(source_details.st_atime_ns, source_details.st_mtime_ns), follow_symlinks=False, ) os.replace(temporary, destination) temporary = "" directory_fd = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY2 } restore_update_environment_snapshot() { local backup_dir snapshot updated manifest updated_manifest backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 snapshot="${backup_dir}/environment" updated="${backup_dir}/environment.updated" manifest="$(update_transaction_manifest_path active environment)" || return 1 updated_manifest="$( update_transaction_manifest_path active environment-updated )" || return 1 update_artifact_manifest verify "${snapshot}" "${manifest}" false || return 1 update_artifact_manifest verify "${updated}" "${updated_manifest}" false || return 1 # Environment promotion replaces the backend and frontend files separately, # so SIGKILL between them leaves a legitimate mixed state. Converge each # file independently: already-original is done, exactly-updated rolls back, # anything else is foreign content and fails closed. local relative current for relative in backend.env.production frontend.env; do case "${relative}" in backend.env.production) current="${PORTAL_DIR}/backend/.env.production" ;; frontend.env) current="${PORTAL_DIR}/frontend/.env" ;; esac if update_regular_files_equal "${snapshot}/${relative}" "${current}"; then continue fi update_regular_files_equal "${updated}/${relative}" "${current}" || return 1 atomic_restore_update_file "${snapshot}/${relative}" "${current}" || return 1 done restore_update_backend_env_alias \ "${snapshot}" "${PORTAL_DIR}/backend/.env" || return 1 sync -f "${PORTAL_DIR}/backend" || return 1 sync -f "${PORTAL_DIR}/frontend" || return 1 update_regular_files_equal \ "${snapshot}/backend.env.production" "${PORTAL_DIR}/backend/.env.production" \ && update_regular_files_equal \ "${snapshot}/frontend.env" "${PORTAL_DIR}/frontend/.env" \ && restore_update_backend_env_alias \ "${snapshot}" "${PORTAL_DIR}/backend/.env" } restore_update_dependencies_snapshot() { local backup_dir snapshot manifest preexisted backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 snapshot="${backup_dir}/backend-node-modules" manifest="$(update_transaction_manifest_path active dependencies)" || return 1 preexisted="$(read_update_transaction_field active node_modules_preexisted)" || return 1 update_artifact_manifest verify "${snapshot}" "${manifest}" true || return 1 if [[ -e "${PORTAL_DIR}/backend/node_modules" \ || -L "${PORTAL_DIR}/backend/node_modules" ]]; then [[ -d "${PORTAL_DIR}/backend/node_modules" \ && ! -L "${PORTAL_DIR}/backend/node_modules" ]] || return 1 rm -rf -- "${PORTAL_DIR}/backend/node_modules" fi if [[ "${preexisted}" == "true" ]]; then install -d -m 0700 -- "${PORTAL_DIR}/backend/node_modules" || return 1 rsync -aHAX --numeric-ids --delete \ "${snapshot}/" "${PORTAL_DIR}/backend/node_modules/" \ >> "${LOG_FILE}" 2>&1 || return 1 sync -f "${PORTAL_DIR}/backend/node_modules" || return 1 update_artifact_manifest verify \ "${PORTAL_DIR}/backend/node_modules" "${manifest}" false else update_artifact_manifest verify \ "${PORTAL_DIR}/backend/node_modules" "${manifest}" true fi } restore_update_caddy_snapshot() { local backup_dir snapshot expected_current manifest required helper backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 snapshot="${backup_dir}/Caddyfile" expected_current="${backup_dir}/Caddyfile.updated" manifest="$(update_transaction_manifest_path active caddy)" || return 1 required="$(read_update_transaction_field active caddy_snapshot_required)" || return 1 update_artifact_manifest verify "${snapshot}" "${manifest}" true || return 1 [[ "${required}" == "true" ]] || return 0 if [[ ! -e "${expected_current}" && ! -L "${expected_current}" ]]; then # The managed helper durably writes the exact would-be installed snapshot # before replacing Caddy. If it does not exist, replacement could not have # begun; accept only an exact match to the original snapshot. update_artifact_manifest verify /etc/caddy/Caddyfile "${manifest}" false return fi [[ -f "${expected_current}" && ! -L "${expected_current}" ]] || return 1 helper="$(update_transaction_state_path "${UPDATE_CADDY_RECOVERY_HELPER}")" \ || return 1 [[ -f "${helper}" && ! -L "${helper}" ]] || return 1 python3 "${helper}" restore \ --caddy-path /etc/caddy/Caddyfile \ --snapshot-path "${snapshot}" \ --expected-current-path "${expected_current}" \ >> "${LOG_FILE}" 2>&1 || return 1 update_artifact_manifest verify /etc/caddy/Caddyfile "${manifest}" false } restore_update_provenance_snapshot() { local backup_dir manifest existed backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 manifest="$(update_transaction_manifest_path active provenance)" || return 1 existed="$(read_update_transaction_field active deploy_stamp_preexisted)" || return 1 update_artifact_manifest verify \ "${backup_dir}/last-portal-deploy.previous" "${manifest}" true || return 1 UPDATE_RECOVERY_DEPLOY_STAMP_CAPTURED=true UPDATE_RECOVERY_DEPLOY_STAMP_EXISTED="${existed}" UPDATE_RECOVERY_DEPLOY_STAMP_BACKUP="${backup_dir}/last-portal-deploy.previous" restore_portal_deploy_stamp_after_failed_update || return 1 update_artifact_manifest verify "${PORTAL_DEPLOY_STAMP}" "${manifest}" true } prepare_updated_environment_candidate() { local backup_dir source candidate manifest env_file stage_dir transaction_id local portal_app_sources_root backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 stage_dir="$(read_update_transaction_field active stage_dir)" || return 1 transaction_id="$(read_update_transaction_field active transaction_id)" \ || return 1 source="${backup_dir}/environment" candidate="${backup_dir}/environment.updated" manifest="$(update_transaction_manifest_path active environment-updated)" \ || return 1 [[ -d "${source}" && ! -L "${source}" ]] || return 1 rm -rf -- "${candidate}" install -d -m 0700 -- "${candidate}" || return 1 rsync -aHAX --numeric-ids --delete "${source}/" "${candidate}/" \ >> "${LOG_FILE}" 2>&1 || return 1 env_file="${candidate}/backend.env.production" [[ -f "${env_file}" && ! -L "${env_file}" ]] || return 1 # A transient validation process receives this flag through systemd only. # Persisting it would turn the canonical Portal into a health-only server. remove_env_assignment_atomic "${env_file}" "PORTAL_UPDATE_VALIDATION_MODE" \ || return 1 sed -i 's|^\(MAIL_DOMAIN=[^[:space:]]*\)DOMAIN=.*|\1|' "${env_file}" \ || return 1 grep -q '^HOST=' "${env_file}" \ && sed -i 's|^HOST=.*|HOST=127.0.0.1|' "${env_file}" \ || printf '\nHOST=127.0.0.1\n' >> "${env_file}" grep -q '^INSTALL_ROOT=' "${env_file}" \ || printf 'INSTALL_ROOT=%s\n' "${INSTALL_ROOT}" >> "${env_file}" grep -q '^APPS_ROOT=' "${env_file}" \ || printf 'APPS_ROOT=%s/apps\n' "${INSTALL_ROOT}" >> "${env_file}" grep -q '^UPLOAD_DIR=' "${env_file}" \ || printf 'UPLOAD_DIR=%s/uploads\n' "${INSTALL_ROOT}" >> "${env_file}" grep -q '^INSTALL_PROFILE=' "${env_file}" \ || printf 'INSTALL_PROFILE=%s\n' "${INSTALL_PROFILE}" >> "${env_file}" grep -q '^PORTAL_URL=' "${env_file}" \ || printf 'PORTAL_URL=%s\n' "$(portal_primary_origin)" >> "${env_file}" portal_app_sources_root="$( select_portal_app_sources_root \ "${env_file}" "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" )" || return 1 ensure_portal_app_sources_root "${portal_app_sources_root}" || return 1 set_env_value_atomic \ "${env_file}" "PORTAL_APPS_ROOT" "${portal_app_sources_root}" \ || return 1 set_env_value_atomic \ "${env_file}" "PORTAL_APP_ZIPS_ROOT" \ "${PORTAL_DIR}/upload-temp/app-zips" || return 1 set_env_value_atomic "${env_file}" "APP_CONTENT_DOMAIN" "${APP_CONTENT_DOMAIN}" \ || return 1 set_env_value_atomic "${env_file}" "APP_CONTENT_ORIGIN" "${APP_CONTENT_ORIGIN}" \ || return 1 set_env_value_atomic "${env_file}" "APP_CONTENT_DNS_MODE" "${APP_CONTENT_DNS_MODE}" \ || return 1 if [[ -n "${TELEMETRY_INSTALL_ID}" ]]; then set_env_value_atomic "${env_file}" "TELEMETRY_INSTALL_ID" "${TELEMETRY_INSTALL_ID}" \ || return 1 fi if [[ -n "${DOMAIN}" ]]; then set_env_value_atomic "${env_file}" "DOMAIN" "${DOMAIN}" || return 1 fi grep -q '^JWT_REFRESH_SECRET=' "${env_file}" \ || printf 'JWT_REFRESH_SECRET=%s\n' "$(rand_hex 32)" >> "${env_file}" set_env_value_atomic \ "${env_file}" "PORTAL_UPDATE_PROBE_TOKEN" "${PORTAL_UPDATE_PROBE_TOKEN}" \ || return 1 local project_egress_secret="" project_egress_secret="$( read_env_value "${env_file}" "PROJECT_EGRESS_TOKEN_SECRET" || true )" if [[ -z "${project_egress_secret}" ]]; then project_egress_secret="$(rand_hex 32)" fi valid_project_egress_token_secret "${project_egress_secret}" || return 1 set_env_value_atomic \ "${env_file}" "PROJECT_EGRESS_TOKEN_SECRET" "${project_egress_secret}" \ || return 1 apply_prepared_update_project_runtime_environment \ "${env_file}" "${stage_dir}" "${transaction_id}" || return 1 chmod --reference="${source}/backend.env.production" "${env_file}" || return 1 chown --reference="${source}/backend.env.production" "${env_file}" || return 1 sync -f "${candidate}" || return 1 update_artifact_manifest create "${candidate}" "${manifest}" false \ && update_artifact_manifest verify "${candidate}" "${manifest}" false } atomic_install_update_symlink() { local target="$1" destination="$2" python3 - "${target}" "${destination}" <<'PY2' import os import secrets import stat import sys target, destination = sys.argv[1:] if ( not target or os.path.isabs(target) or "/" in target or "\0" in target or not os.path.isabs(destination) or os.path.normpath(destination) != destination or destination == os.path.sep ): raise SystemExit(1) parent = os.path.dirname(destination) name = os.path.basename(destination) parent_details = os.lstat(parent) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_uid != os.geteuid() or parent_details.st_gid != os.getegid() or parent_details.st_mode & 0o022 ): raise SystemExit(1) try: current = os.lstat(destination) except FileNotFoundError: current = None if current is not None and not ( stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) ): raise SystemExit(1) def fsync_parent(): descriptor = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) prefix = f".{name}.update-link." # A prior hard kill before rename can leave only this bounded temporary # symlink. The global installer operation lock excludes a concurrent owner. for candidate_name in os.listdir(parent): if not candidate_name.startswith(prefix): continue candidate = os.path.join(parent, candidate_name) details = os.lstat(candidate) if ( not stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or os.readlink(candidate) != target ): raise SystemExit(1) os.unlink(candidate) fsync_parent() temporary = os.path.join( parent, f"{prefix}{os.getpid()}.{secrets.token_hex(8)}" ) source_only = ( os.environ.get("BRIDGESLLM_INSTALLER_SOURCE_ONLY") == "1" and bool(os.environ.get("BRIDGESLLM_UPDATE_TRANSACTION_TEST_ROOT")) ) crash_at = ( os.environ.get("BRIDGESLLM_UPDATE_TRANSACTION_TEST_ENV_LINK_CRASH_AT", "") if source_only else "" ) if crash_at not in {"", "before-replace", "after-replace"}: raise SystemExit(1) try: os.symlink(target, temporary) created = os.lstat(temporary) if ( not stat.S_ISLNK(created.st_mode) or created.st_uid != os.geteuid() or created.st_gid != os.getegid() or os.readlink(temporary) != target ): raise SystemExit(1) fsync_parent() if crash_at == "before-replace": os._exit(95) os.replace(temporary, destination) temporary = "" installed = os.lstat(destination) if ( not stat.S_ISLNK(installed.st_mode) or installed.st_uid != os.geteuid() or installed.st_gid != os.getegid() or os.readlink(destination) != target ): raise SystemExit(1) if crash_at == "after-replace": os._exit(96) fsync_parent() finally: if temporary: try: os.unlink(temporary) fsync_parent() except FileNotFoundError: pass PY2 } install_updated_environment_candidate() { local backup_dir candidate manifest backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 candidate="${backup_dir}/environment.updated" manifest="$(update_transaction_manifest_path active environment-updated)" \ || return 1 update_artifact_manifest verify "${candidate}" "${manifest}" false || return 1 atomic_restore_update_file \ "${candidate}/backend.env.production" \ "${PORTAL_DIR}/backend/.env.production" || return 1 atomic_restore_update_file \ "${candidate}/frontend.env" \ "${PORTAL_DIR}/frontend/.env" || return 1 atomic_install_update_symlink \ .env.production "${PORTAL_DIR}/backend/.env" || return 1 local validation_assignment_status=0 env_file_has_assignment \ "${PORTAL_DIR}/backend/.env.production" \ "PORTAL_UPDATE_VALIDATION_MODE" || validation_assignment_status=$? [[ "${validation_assignment_status}" -eq 1 ]] || return 1 update_regular_files_equal \ "${candidate}/backend.env.production" \ "${PORTAL_DIR}/backend/.env.production" \ && update_regular_files_equal \ "${candidate}/frontend.env" "${PORTAL_DIR}/frontend/.env" } overlay_verified_update_runtime() { local staged_portal="$1" changes [[ -f "${staged_portal}/frontend/dist/index.html" \ && -f "${staged_portal}/backend/dist/server.js" \ && -f "${staged_portal}/backend/dist/services/projectEgressPolicy.js" \ && -f "${staged_portal}/backend/dist/services/projectEgressProxy.js" ]] \ || return 1 rsync -aHAX --numeric-ids --delete \ --exclude='node_modules' \ --exclude='.git' \ --exclude='.env' \ --exclude='.env.production' \ --exclude='/projects' \ --exclude='/apps' \ --exclude='/assets' \ --exclude='/upload-temp' \ --exclude='/.data' \ --exclude='/backend/.data' \ "${staged_portal}/" "${PORTAL_DIR}/" >> "${LOG_FILE}" 2>&1 || return 1 sync -f "${PORTAL_DIR}" || return 1 changes="$(rsync -aHAXnc --numeric-ids --delete --itemize-changes \ --exclude='node_modules' \ --exclude='.git' \ --exclude='.env' \ --exclude='.env.production' \ --exclude='/projects' \ --exclude='/apps' \ --exclude='/assets' \ --exclude='/upload-temp' \ --exclude='/.data' \ --exclude='/backend/.data' \ "${staged_portal}/" "${PORTAL_DIR}/" 2>> "${LOG_FILE}")" || return 1 [[ -z "${changes}" ]] } install_prepared_update_dependencies() { local staged_portal="$1" local source="${staged_portal}/backend/node_modules" [[ -d "${source}" && ! -L "${source}" ]] || return 1 if [[ -e "${PORTAL_DIR}/backend/node_modules" \ || -L "${PORTAL_DIR}/backend/node_modules" ]]; then [[ -d "${PORTAL_DIR}/backend/node_modules" \ && ! -L "${PORTAL_DIR}/backend/node_modules" ]] || return 1 rm -rf -- "${PORTAL_DIR}/backend/node_modules" fi install -d -m 0700 -- "${PORTAL_DIR}/backend/node_modules" || return 1 rsync -aHAX --numeric-ids --delete \ "${source}/" "${PORTAL_DIR}/backend/node_modules/" \ >> "${LOG_FILE}" 2>&1 || return 1 sync -f "${PORTAL_DIR}/backend/node_modules" || return 1 ( cd "${PORTAL_DIR}/backend" node -e "for (const name of ['@prisma/client','@prisma/adapter-pg','pg','bcrypt','sharp','node-pty']) require(name)" ) >> "${LOG_FILE}" 2>&1 || return 1 verify_prisma_client_runtime \ "${PORTAL_DIR}/backend" installed-prisma-check \ >> "${LOG_FILE}" 2>&1 } prepare_update_transaction_layout() { local transaction_id="$1" stage_dir="$2" [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 local state_root transaction_root backup_root expected_stage_root state_root="$(update_transaction_state_path "${UPDATE_STATE_ROOT}")" || return 1 transaction_root="${state_root}/transactions" backup_root="$(update_transaction_state_path "${UPDATE_BACKUP_ROOT}")" || return 1 expected_stage_root="$(update_transaction_state_path "${UPDATE_STAGE_ROOT}")" \ || return 1 python3 - "${transaction_id}" "${state_root}" "${transaction_root}" \ "${backup_root}" "${expected_stage_root}" "${stage_dir}" <<'PY2' import os import stat import sys transaction_id, state_root, transaction_root, backup_root, stage_root, stage_dir = sys.argv[1:] expected_uid = os.geteuid() expected_gid = os.getegid() expected = ( os.path.join(transaction_root, transaction_id), os.path.join(backup_root, f"update-{transaction_id}"), os.path.join(stage_root, f"update-{transaction_id}"), ) if stage_dir != expected[2]: raise SystemExit(1) def secure_directory(path: str, *, mode=None) -> None: details = os.lstat(path) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != expected_uid or details.st_gid != expected_gid or details.st_mode & 0o077 ): raise SystemExit(1) if mode is not None and stat.S_IMODE(details.st_mode) != mode: raise SystemExit(1) for root in (state_root, stage_root): secure_directory(root, mode=0o700) for root in (transaction_root, backup_root): parent = os.path.dirname(root) try: parent_details = os.lstat(parent) except FileNotFoundError: grandparent = os.path.dirname(parent) grandparent_details = os.lstat(grandparent) if ( not stat.S_ISDIR(grandparent_details.st_mode) or stat.S_ISLNK(grandparent_details.st_mode) or grandparent_details.st_uid != expected_uid or grandparent_details.st_mode & 0o022 ): raise SystemExit(1) os.mkdir(parent, 0o700) parent_details = os.lstat(parent) if ( not stat.S_ISDIR(parent_details.st_mode) or stat.S_ISLNK(parent_details.st_mode) or parent_details.st_uid != expected_uid or parent_details.st_mode & 0o022 ): raise SystemExit(1) try: os.mkdir(root, 0o700) except FileExistsError: pass os.chmod(root, 0o700) secure_directory(root, mode=0o700) for leaf in expected[:2]: try: os.mkdir(leaf, 0o700) except FileExistsError: raise SystemExit(1) secure_directory(leaf, mode=0o700) secure_directory(expected[2], mode=0o700) for directory in (*expected, transaction_root, backup_root, stage_root, state_root): descriptor = os.open( directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) PY2 } prepare_update_transaction() { local previous_version="$1" staged_portal="$2" local transaction_id="${3:-${UPDATE_TRANSACTION_ID}}" local -a database_topology_pin=() [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 mapfile -t database_topology_pin < <( read_update_disk_reserve_database_topology "${transaction_id}" ) || return 1 [[ "${#database_topology_pin[@]}" -eq 3 ]] || return 1 install_update_transaction_state_helper "${staged_portal}" || return 1 prepare_update_transaction_layout \ "${transaction_id}" "$(dirname "${staged_portal}")" || return 1 local node_modules_preexisted=false deploy_stamp_preexisted=false local caddy_snapshot_required=false repair_reinstall=false local openclaw_package_preexisted=false openclaw_package_version="" local openclaw_runtime_version="" openclaw_state_preexisted=false local openclaw_gateway_was_active=false openclaw_gateway_was_enabled=false local openclaw_codex_plugin_preexisted=false openclaw_codex_plugin_version="" [[ -d "${PORTAL_DIR}/backend/node_modules" \ && ! -L "${PORTAL_DIR}/backend/node_modules" ]] \ && node_modules_preexisted=true [[ -f "${PORTAL_DEPLOY_STAMP}" && ! -L "${PORTAL_DEPLOY_STAMP}" ]] \ && deploy_stamp_preexisted=true if ! use_local_profile && ! use_tailnet_profile; then caddy_snapshot_required=true fi ${REPAIR_REINSTALL:-false} && repair_reinstall=true if command -v openclaw >/dev/null 2>&1; then openclaw_package_preexisted=true openclaw_package_version="$(openclaw_core_package_version || true)" openclaw_runtime_version="$(openclaw_cli_version || true)" [[ -n "${openclaw_package_version}" && -n "${openclaw_runtime_version}" ]] \ || return 1 local plugin_details="" if plugin_details="$(openclaw_codex_plugin_details 2>/dev/null)"; then openclaw_codex_plugin_preexisted=true openclaw_codex_plugin_version="$(head -1 <<<"${plugin_details}")" [[ -n "${openclaw_codex_plugin_version}" ]] || return 1 else local plugin_status=$? [[ "${plugin_status}" -eq 2 ]] || return 1 fi fi openclaw_installation_state_present "${HOME}/.openclaw" \ && openclaw_state_preexisted=true systemctl is-active --quiet openclaw-gateway && openclaw_gateway_was_active=true systemctl is-enabled openclaw-gateway >/dev/null 2>&1 \ && openclaw_gateway_was_enabled=true local -a command=( create --transaction-id "${transaction_id}" --previous-version "${previous_version}" --target-version "${VERSION}" --release-artifact-sha256 "${VERIFIED_RELEASE_ARTIFACT_SHA256}" --release-manifest-sha256 "${VERIFIED_RELEASE_MANIFEST_SHA256}" --database-system-identifier "${database_topology_pin[0]}" --database-topology-sha256 "${database_topology_pin[1]}" --previous-main-pid "${UPDATE_TRANSACTION_BASELINE_PID}" --baseline-boot-id "${UPDATE_TRANSACTION_BASELINE_BOOT_ID}" --previous-main-start-time "${UPDATE_TRANSACTION_BASELINE_START_TIME}" --portal-was-active "${UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE}" --portal-was-enabled "${UPDATE_TRANSACTION_PORTAL_WAS_ENABLED}" --deploy-stamp-preexisted "${deploy_stamp_preexisted}" --caddy-snapshot-required "${caddy_snapshot_required}" --node-modules-preexisted "${node_modules_preexisted}" --repair-reinstall "${repair_reinstall}" --openclaw-package-preexisted "${openclaw_package_preexisted}" --openclaw-state-preexisted "${openclaw_state_preexisted}" --openclaw-gateway-was-active "${openclaw_gateway_was_active}" --openclaw-gateway-was-enabled "${openclaw_gateway_was_enabled}" --openclaw-codex-plugin-preexisted "${openclaw_codex_plugin_preexisted}" ) [[ -z "${openclaw_package_version}" ]] \ || command+=(--openclaw-package-version "${openclaw_package_version}") [[ -z "${openclaw_runtime_version}" ]] \ || command+=(--openclaw-runtime-version "${openclaw_runtime_version}") [[ -z "${openclaw_codex_plugin_version}" ]] \ || command+=(--openclaw-codex-plugin-version "${openclaw_codex_plugin_version}") # Install both permanent ConditionPathExists boundaries while no receipt # exists. The OpenClaw fence marker is managed only by the Portal's # authorization-transition coordinator; update/repair must never clear it. install_openclaw_gateway_authorization_fence_dropin || return 1 # Publishing the active update receipt then becomes the Portal boot-fence # decision. install_portal_update_boot_fence || return 1 run_update_transaction_state_helper "${command[@]}" >/dev/null || return 1 # From receipt publication onward, every failure path must use the durable # recovery engine and must preserve the stage/backup directories it names. UPDATE_RECOVERY_ARMED=true UPDATE_TRANSACTION_ID="${transaction_id}" UPDATE_TRANSACTION_GENERATION="$( read_update_transaction_field active generation )" || return 1 UPDATE_RECOVERY_BACKUP_DIR="$( read_update_transaction_field active backup_dir )" || return 1 UPDATE_RELEASE_STAGE_DIR="$( read_update_transaction_field active stage_dir )" || return 1 UPDATE_TRANSACTION_PREVIOUS_VERSION="${previous_version}" UPDATE_TRANSACTION_TARGET_VERSION="${VERSION}" } dashboard_update_progress_for_transaction_phase() { local phase="$1" case "${phase}" in boot_blocked) dashboard_update_progress running 42 portal-transaction \ "Opening protected update transaction" \ "Step 7 of 13 · Portal boot is fenced before any live runtime mutation." ;; portal_quiesced) dashboard_update_progress running 46 portal-quiesced \ "Portal stopped safely" \ "Step 7 of 13 · The existing Portal is quiesced and the rollback boundary is active." ;; runtime_snapshot_complete) dashboard_update_progress running 50 rollback-snapshots \ "Taking recovery snapshots" \ "Step 8 of 13 · Runtime and environment recovery copies are sealed." ;; database_snapshot_complete) dashboard_update_progress running 52 rollback-snapshots \ "Taking recovery snapshots" \ "Step 8 of 13 · The database recovery snapshot is verified." ;; node_modules_moved) dashboard_update_progress running 55 rollback-snapshots \ "Recovery snapshots complete" \ "Step 8 of 13 · Runtime, database, routing, and dependency recovery points are sealed." ;; runtime_overlaid) dashboard_update_progress running 62 runtime-install \ "Installing signed Portal runtime" \ "Step 9 of 13 · Signed runtime files were promoted; environment and database work follows." ;; dependencies_updated) dashboard_update_progress running 66 runtime-install \ "Installing signed Portal runtime" \ "Step 9 of 13 · Preverified backend dependencies were promoted." ;; database_migrated) dashboard_update_progress running 70 database-migration \ "Portal runtime and database installed" \ "Step 9 of 13 · Database migrations completed inside the rollback transaction." ;; candidate_started) dashboard_update_progress running 76 candidate-verification \ "Starting private candidate" \ "Step 10 of 13 · The signed Portal is running on a private validation port." ;; candidate_verified) dashboard_update_progress running 82 candidate-verification \ "Private candidate verified" \ "Step 10 of 13 · Exact version, schema, and mutation-free readiness checks passed." ;; provenance_committed) dashboard_update_progress running 86 cutover-preparation \ "Preparing verified cutover" \ "Step 11 of 13 · Signed release provenance is committed." ;; canonical_started) dashboard_update_progress running 91 portal-restarting \ "Updated Portal is restarting" \ "Step 12 of 13 · Reconnecting automatically while the canonical service starts." ;; canonical_verified) dashboard_update_progress running 93 portal-restarting \ "Updated Portal verified" \ "Step 12 of 13 · The canonical service passed exact-version and readiness checks." ;; boot_state_restored) dashboard_update_progress running 94 portal-restarting \ "Restoring Portal service state" \ "Step 12 of 13 · The prior enablement policy is restored before commit." ;; committed) DASHBOARD_UPDATE_PORTAL_COMMITTED=true dashboard_update_progress running 95 portal-committed \ "Updated Portal is online" \ "Step 12 of 13 · The signed Portal is durably committed; transaction cleanup and host integration remain." ;; recovery_pending|recovery_quiesce_pending) dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Preparing automatic recovery" \ "The updater is preserving the recovery boundary and stopping partial runtime state." ;; recovery_quiesced) dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Portal stopped for recovery" \ "The previous runtime, database, routing, and dependencies can now be restored safely." ;; recovery_database_restored) dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Database recovery verified" \ "The pre-update database snapshot has been restored and attested." ;; recovery_runtime_restored) dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Previous Portal files restored" \ "The pre-update runtime and environment are back in place." ;; recovery_dependencies_restored) dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Previous dependencies restored" \ "The exact pre-update backend dependencies are back in place." ;; recovery_service_restored|recovery_verify_pending) dashboard_update_progress recovering \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" recovery \ "Verifying restored Portal" \ "The previous Portal is restarting for exact-version and health verification." ;; recovered) dashboard_update_progress rolled_back \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" rolled-back \ "Previous Portal restored" \ "The previous Portal was restored, restarted, and verified after the update stopped." ;; esac } advance_update_transaction_phase() { local target="$1" expected_phase="$2" next_phase="$3" local generation transaction_id transaction_id="$(read_update_transaction_field "${target}" transaction_id)" \ || return 1 generation="$(read_update_transaction_field "${target}" generation)" \ || return 1 run_update_transaction_state_helper update \ --target "${target}" \ --transaction-id "${transaction_id}" \ --expected-generation "${generation}" \ --expected-phase "${expected_phase}" \ --phase "${next_phase}" >/dev/null || return 1 UPDATE_TRANSACTION_ID="${transaction_id}" UPDATE_TRANSACTION_GENERATION="$((generation + 1))" dashboard_update_progress_for_transaction_phase "${next_phase}" } cutover_update_transaction() { local expected_phase="${1:-cutover_pending}" local generation transaction_id transaction_id="$(read_update_transaction_field active transaction_id)" || return 1 generation="$(read_update_transaction_field active generation)" || return 1 run_update_transaction_state_helper cutover \ --transaction-id "${transaction_id}" \ --expected-generation "${generation}" \ --expected-phase "${expected_phase}" >/dev/null || return 1 UPDATE_TRANSACTION_ID="${transaction_id}" UPDATE_TRANSACTION_GENERATION="${generation}" } finish_update_transaction() { local target="$1" terminal_phase="$2" local generation transaction_id transaction_id="$(read_update_transaction_field "${target}" transaction_id)" \ || return 1 generation="$(read_update_transaction_field "${target}" generation)" || return 1 run_update_transaction_state_helper remove \ --target "${target}" \ --transaction-id "${transaction_id}" \ --expected-generation "${generation}" \ --expected-phase "${terminal_phase}" >/dev/null || return 1 UPDATE_RECOVERY_ARMED=false UPDATE_TRANSACTION_ID="" UPDATE_TRANSACTION_GENERATION="" UPDATE_TRANSACTION_PREVIOUS_VERSION="" UPDATE_TRANSACTION_TARGET_VERSION="" } arm_update_transaction_traps() { trap 'handle_update_transaction_err $?' ERR trap 'handle_sigint' SIGINT trap 'handle_sigterm' TERM trap 'handle_sighup' HUP } update_forward_phase_rank() { case "$1" in prepared) echo 0 ;; boot_block_pending) echo 1 ;; boot_blocked) echo 2 ;; portal_quiesce_pending) echo 3 ;; portal_quiesced) echo 4 ;; runtime_snapshot_pending) echo 5 ;; runtime_snapshot_complete) echo 6 ;; database_snapshot_pending) echo 7 ;; database_snapshot_complete) echo 8 ;; caddy_snapshot_pending) echo 9 ;; caddy_snapshot_complete) echo 10 ;; openclaw_snapshot_pending) echo 11 ;; openclaw_snapshot_complete) echo 12 ;; node_modules_move_pending) echo 13 ;; node_modules_moved) echo 14 ;; runtime_overlay_pending) echo 15 ;; runtime_overlaid) echo 16 ;; environment_update_pending) echo 17 ;; environment_updated) echo 18 ;; caddy_update_pending) echo 19 ;; caddy_updated) echo 20 ;; dependencies_update_pending) echo 21 ;; dependencies_updated) echo 22 ;; database_migration_pending) echo 23 ;; database_migrated) echo 24 ;; openclaw_update_pending) echo 25 ;; openclaw_updated) echo 26 ;; candidate_start_pending) echo 27 ;; candidate_started) echo 28 ;; candidate_verification_pending) echo 29 ;; candidate_verified) echo 30 ;; provenance_commit_pending) echo 31 ;; provenance_committed) echo 32 ;; cutover_pending) echo 33 ;; canonical_start_pending) echo 34 ;; canonical_started) echo 35 ;; canonical_verification_pending) echo 36 ;; canonical_verified) echo 37 ;; boot_state_restore_pending) echo 38 ;; boot_state_restored) echo 39 ;; committed) echo 40 ;; *) return 1 ;; esac } update_recovery_reached_phase() { local threshold="$1" recovery_from from_rank threshold_rank recovery_from="$(read_update_transaction_field active recovery_from_phase)" \ || return 1 [[ -n "${recovery_from}" ]] || return 1 from_rank="$(update_forward_phase_rank "${recovery_from}")" || return 1 threshold_rank="$(update_forward_phase_rank "${threshold}")" || return 1 (( from_rank >= threshold_rank )) } current_update_receipt_target() { local active_journal cutover_journal target="" active_journal="$(update_transaction_state_path "${UPDATE_ACTIVE_JOURNAL}")" \ || return 1 cutover_journal="$(update_transaction_state_path "${UPDATE_CUTOVER_JOURNAL}")" \ || return 1 if [[ -e "${active_journal}" || -L "${active_journal}" ]]; then [[ ! -e "${cutover_journal}" && ! -L "${cutover_journal}" ]] || return 1 target=active elif [[ -e "${cutover_journal}" || -L "${cutover_journal}" ]]; then target=cutover else return 1 fi # `read` validates the entire receipt schema, metadata, generation, bounded # paths, artifact identities, and phase contract before returning any field. read_update_transaction_field "${target}" transaction_id >/dev/null \ || return 1 printf '%s\n' "${target}" } load_update_origin_from_environment() { local env_file="$1" local profile origin domain tailnet_name app_domain app_origin app_mode [[ -f "${env_file}" && ! -L "${env_file}" ]] || return 1 profile="$(read_env_value "${env_file}" "INSTALL_PROFILE" || true)" origin="$(read_env_value "${env_file}" "ORIGIN_MODE" || true)" domain="$(read_env_value "${env_file}" "DOMAIN" || true)" tailnet_name="$(read_env_value "${env_file}" "TAILNET_DNS_NAME" || true)" app_domain="$(read_env_value "${env_file}" "APP_CONTENT_DOMAIN" || true)" app_origin="$(read_env_value "${env_file}" "APP_CONTENT_ORIGIN" || true)" app_mode="$(read_env_value "${env_file}" "APP_CONTENT_DNS_MODE" || true)" case "${profile}" in ""|server) profile="server" ;; local) ;; *) return 1 ;; esac case "${origin}" in "") ;; tailnet) ;; *) return 1 ;; esac [[ ! ( "${profile}" == "local" && "${origin}" == "tailnet" ) ]] || return 1 if [[ "${profile}" == "local" || "${origin}" == "tailnet" ]]; then [[ -z "${domain}" ]] || return 1 else domain="$( python3 - "${domain}" <<'PY2' import re import sys value = sys.argv[1].strip().lower().rstrip(".") labels = value.split(".") valid = ( 1 <= len(value) <= 253 and len(labels) >= 2 and all( 1 <= len(label) <= 63 and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", label) for label in labels ) ) if not valid: raise SystemExit(1) print(value) PY2 )" || return 1 fi if [[ "${origin}" == "tailnet" ]]; then tailnet_name="$( python3 - "${tailnet_name}" <<'PY2' import re import sys value = sys.argv[1].strip().lower().rstrip(".") labels = value.split(".") if ( not value.endswith(".ts.net") or len(value) > 253 or not all( 1 <= len(label) <= 63 and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", label) for label in labels ) ): raise SystemExit(1) print(value) PY2 )" || return 1 else tailnet_name="" fi if [[ "${profile}" == "server" && -z "${origin}" ]]; then if [[ -z "${app_domain}" ]]; then app_domain="$(app_content_domain_from_origin "${app_origin}" || true)" fi if [[ -n "${app_domain}" ]]; then app_domain="$( python3 - "${app_domain}" <<'PY2' import re import sys value = sys.argv[1].strip().lower().rstrip(".") labels = value.split(".") if ( len(value) > 253 or len(labels) < 2 or not all( 1 <= len(label) <= 63 and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", label) for label in labels ) ): raise SystemExit(1) print(value) PY2 )" || return 1 fi case "${app_mode}" in ""|custom|sslip) ;; *) return 1 ;; esac else app_domain="" fi INSTALL_PROFILE="${profile}" ORIGIN_MODE="${origin}" DOMAIN="${domain}" TAILNET_DNS_NAME="${tailnet_name}" APP_CONTENT_DOMAIN="${app_domain}" APP_CONTENT_DNS_MODE="${app_mode}" } load_update_transaction_context() { local target="$1" UPDATE_TRANSACTION_ID="$( read_update_transaction_field "${target}" transaction_id )" || return 1 UPDATE_TRANSACTION_GENERATION="$( read_update_transaction_field "${target}" generation )" || return 1 UPDATE_TRANSACTION_PREVIOUS_VERSION="$( read_update_transaction_field "${target}" previous_version )" || return 1 UPDATE_TRANSACTION_TARGET_VERSION="$( read_update_transaction_field "${target}" target_version )" || return 1 UPDATE_TRANSACTION_BASELINE_PID="$( read_update_transaction_field "${target}" previous_main_pid )" || return 1 UPDATE_TRANSACTION_BASELINE_BOOT_ID="$( read_update_transaction_field "${target}" baseline_boot_id )" || return 1 UPDATE_TRANSACTION_BASELINE_START_TIME="$( read_update_transaction_field "${target}" previous_main_start_time )" || return 1 UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE="$( read_update_transaction_field "${target}" portal_was_active )" || return 1 UPDATE_TRANSACTION_PORTAL_WAS_ENABLED="$( read_update_transaction_field "${target}" portal_was_enabled )" || return 1 UPDATE_RECOVERY_BACKUP_DIR="$( read_update_transaction_field "${target}" backup_dir )" || return 1 UPDATE_RELEASE_STAGE_DIR="$( read_update_transaction_field "${target}" stage_dir )" || return 1 [[ "${UPDATE_TRANSACTION_ID}" =~ ^[a-f0-9]{32}$ \ && "${UPDATE_TRANSACTION_GENERATION}" =~ ^[1-9][0-9]*$ \ && "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+ \ && "${UPDATE_TRANSACTION_TARGET_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+ \ && "${UPDATE_TRANSACTION_BASELINE_PID}" =~ ^[0-9]+$ \ && "${UPDATE_TRANSACTION_PORTAL_WAS_ACTIVE}" =~ ^(true|false)$ \ && "${UPDATE_TRANSACTION_PORTAL_WAS_ENABLED}" =~ ^(true|false)$ ]] || return 1 UPDATE_RECOVERY_ARMED=true } reopen_cutover_update_transaction() { local phase generation transaction_id transaction_id="$(read_update_transaction_field cutover transaction_id)" \ || return 1 generation="$(read_update_transaction_field cutover generation)" || return 1 phase="$(read_update_transaction_field cutover phase)" || return 1 run_update_transaction_state_helper reopen \ --transaction-id "${transaction_id}" \ --expected-generation "${generation}" \ --expected-phase "${phase}" >/dev/null || return 1 UPDATE_TRANSACTION_ID="${transaction_id}" UPDATE_TRANSACTION_GENERATION="$((generation + 1))" } remove_update_transaction_artifacts() { local mode="$1" transaction_id="$2" backup_dir="$3" stage_dir="$4" transaction_dir="$5" local test_root="" test_root="$(update_transaction_test_root 2>/dev/null || true)" python3 - "${mode}" "${transaction_id}" "${backup_dir}" "${stage_dir}" \ "${transaction_dir}" "${test_root}" <<'PY2' import os import shutil import stat import sys mode, transaction_id, backup_dir, stage_dir, transaction_dir, test_root = sys.argv[1:] prefix = test_root if test_root else os.path.sep expected = ( os.path.join(prefix, "opt/bridgesllm/backups/update-transactions", f"update-{transaction_id}"), os.path.join(prefix, "opt/bridgesllm/update-staging", f"update-{transaction_id}"), os.path.join(prefix, "var/lib/bridgesllm-installer/transactions", transaction_id), ) provided = (backup_dir, stage_dir, transaction_dir) if provided != expected: raise SystemExit(1) # The secret-bearing payload (backup snapshots, staged release) must be # removable while the terminal receipt still exists, so a crash mid-cleanup # always leaves an authoritative receipt pointing at idempotent work. The # journal directory holds only manifests and helpers and is removed last. if mode == "payload": selected = provided[:2] elif mode == "journal": selected = provided[2:] else: raise SystemExit(1) expected_uid = os.geteuid() for path in selected: try: details = os.lstat(path) except FileNotFoundError: continue if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != expected_uid or details.st_mode & 0o077 ): raise SystemExit(1) for path in selected: try: shutil.rmtree(path) except FileNotFoundError: pass parent = os.path.dirname(path) descriptor = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) PY2 } complete_update_transaction() { local target="$1" terminal_phase="$2" local transaction_id backup_dir stage_dir transaction_dir cleanup_policy transaction_id="$(read_update_transaction_field "${target}" transaction_id)" \ || return 1 backup_dir="$(read_update_transaction_field "${target}" backup_dir)" || return 1 stage_dir="$(read_update_transaction_field "${target}" stage_dir)" || return 1 transaction_dir="$(read_update_transaction_field "${target}" transaction_dir)" \ || return 1 # Once canonical verification reaches a terminal receipt, the preallocated # emergency blocks are no longer needed. Release them while the receipt # still exists so a crash leaves an idempotent, authoritative cleanup path. # A crash after payload removal but before receipt removal intentionally # leaves no staged runtime-policy evidence. Dual payload absence is itself # the durable replay boundary: the removal below can run only after runtime # tags were cleaned successfully. Do not require Docker again on that exact # terminal replay; the authoritative receipt and journal still remain. if [[ -e "${backup_dir}" || -L "${backup_dir}" \ || -e "${stage_dir}" || -L "${stage_dir}" ]]; then cleanup_policy="$( update_project_runtime_cleanup_policy_for_target "${target}" \ 2>/dev/null || true )" cleanup_prepared_update_project_runtime_tags \ "${transaction_id}" "${cleanup_policy}" || return 1 fi release_update_disk_reserves || return 1 # Order is load-bearing: secret-bearing payload first (receipt retains # recovery authority over a crash here), then the receipt, then the # secret-free journal directory. remove_update_transaction_artifacts payload \ "${transaction_id}" "${backup_dir}" "${stage_dir}" "${transaction_dir}" \ || return 1 finish_update_transaction "${target}" "${terminal_phase}" || return 1 remove_update_transaction_artifacts journal \ "${transaction_id}" "${backup_dir}" "${stage_dir}" "${transaction_dir}" \ || return 1 if [[ "${terminal_phase}" == "recovered" ]]; then dashboard_update_progress rolled_back \ "${DASHBOARD_UPDATE_PROGRESS_PERCENT}" rolled-back \ "Previous Portal restored" \ "The previous Portal was restored, restarted, and verified after the update stopped." elif [[ "${terminal_phase}" == "committed" ]]; then DASHBOARD_UPDATE_PORTAL_COMMITTED=true dashboard_update_progress running 95 portal-committed \ "Updated Portal is online" \ "Step 12 of 13 · The signed Portal is committed; final host integration is still running." fi } complete_committed_update_transaction() { # `committed` is the first phase from which rollback to the prior Portal is # impossible. Promote the already pinned immutable IDs now, never while an # old environment might still be restored. Replays are idempotent because # the same receipt and image IDs remain authoritative until cleanup finishes. promote_prepared_update_project_runtime_images cutover \ && complete_update_transaction cutover committed } refence_cutover_recovery() { reopen_cutover_update_transaction || return 1 install_portal_update_boot_fence || return 1 quiesce_portal_service "bridgesllm-product" 20 10 4001 } finish_forward_cutover_transaction() { load_update_transaction_context cutover || return 1 load_update_origin_from_environment \ "${PORTAL_DIR}/backend/.env.production" || return 1 local phase probe_token validation_assignment_status while true; do phase="$(read_update_transaction_field cutover phase)" || return 1 case "${phase}" in cutover_pending) stop_update_candidate "${UPDATE_TRANSACTION_ID}" "${UPDATE_CANDIDATE_PORT}" \ || { refence_cutover_recovery || true; return 1; } advance_update_transaction_phase \ cutover cutover_pending canonical_start_pending || return 1 ;; canonical_start_pending) if ! start_canonical_portal_for_transaction; then refence_cutover_recovery || true return 1 fi advance_update_transaction_phase \ cutover canonical_start_pending canonical_started || return 1 ;; canonical_started) advance_update_transaction_phase \ cutover canonical_started canonical_verification_pending || return 1 ;; canonical_verification_pending) probe_token="$(read_env_value \ "${PORTAL_DIR}/backend/.env.production" \ "PORTAL_UPDATE_PROBE_TOKEN" || true)" validation_assignment_status=0 env_file_has_assignment \ "${PORTAL_DIR}/backend/.env.production" \ "PORTAL_UPDATE_VALIDATION_MODE" \ || validation_assignment_status=$? if [[ -z "${probe_token}" ]] \ || [[ "${validation_assignment_status}" -ne 1 ]] \ || ! verify_canonical_portal_for_transaction \ "${UPDATE_TRANSACTION_TARGET_VERSION}" "${probe_token}"; then refence_cutover_recovery || true return 1 fi advance_update_transaction_phase \ cutover canonical_verification_pending canonical_verified || return 1 ;; canonical_verified) advance_update_transaction_phase \ cutover canonical_verified boot_state_restore_pending || return 1 ;; boot_state_restore_pending) if ! restore_portal_service_boot_state "bridgesllm-product"; then refence_cutover_recovery || true return 1 fi advance_update_transaction_phase \ cutover boot_state_restore_pending boot_state_restored || return 1 ;; boot_state_restored) advance_update_transaction_phase \ cutover boot_state_restored committed || return 1 ;; committed) complete_committed_update_transaction return ;; *) return 1 ;; esac done } recover_active_update_transaction() { load_update_transaction_context active || return 1 local phase recovery_from probe_token db_url backup_dir manifest contract_variant while true; do phase="$(read_update_transaction_field active phase 2>/dev/null || true)" if [[ -z "${phase}" ]]; then # Recovery cutover phases deliberately move the receipt out of the boot # fence. Continue from that exact cutover receipt. phase="$(read_update_transaction_field cutover phase)" || return 1 case "${phase}" in recovery_cutover_pending|recovery_service_restore_pending|recovery_service_restored|recovery_verify_pending|recovered) ;; *) return 1 ;; esac fi case "${phase}" in recovery_pending) advance_update_transaction_phase \ active recovery_pending recovery_quiesce_pending || return 1 ;; recovery_quiesce_pending) stop_update_candidate "${UPDATE_TRANSACTION_ID}" "${UPDATE_CANDIDATE_PORT}" \ || return 1 quiesce_portal_service "bridgesllm-product" 45 15 4001 || return 1 advance_update_transaction_phase \ active recovery_quiesce_pending recovery_quiesced || return 1 ;; recovery_quiesced) if update_recovery_reached_phase portal_quiesce_pending; then local app_intent_env="${PORTAL_DIR}/backend/.env.production" if update_recovery_reached_phase runtime_snapshot_complete; then backup_dir="$(read_update_transaction_field active backup_dir)" \ || return 1 app_intent_env="${backup_dir}/environment/backend.env.production" fi db_url="$(read_env_value \ "${app_intent_env}" DATABASE_URL || true)" [[ -n "${db_url}" ]] || return 1 reconcile_update_running_app_intent active "${db_url}" || return 1 fi advance_update_transaction_phase \ active recovery_quiesced recovery_openclaw_restore_pending || return 1 ;; recovery_openclaw_restore_pending) # Global OpenClaw/tool convergence is intentionally post-commit. These # journal phases are a proven no-op kept for schema compatibility. advance_update_transaction_phase \ active recovery_openclaw_restore_pending recovery_openclaw_restored \ || return 1 ;; recovery_openclaw_restored) advance_update_transaction_phase \ active recovery_openclaw_restored recovery_database_restore_pending \ || return 1 ;; recovery_database_restore_pending) if update_recovery_reached_phase database_migration_pending; then backup_dir="$(read_update_transaction_field active backup_dir)" || return 1 manifest="$(update_transaction_manifest_path active database)" || return 1 update_artifact_manifest verify \ "${backup_dir}/database-before-update.dump" "${manifest}" false \ || return 1 contract_variant="$( read_update_database_contract_snapshot active "${backup_dir}" )" || return 1 db_url="$(read_env_value \ "${backup_dir}/environment/backend.env.production" \ "DATABASE_URL" || true)" [[ -n "${db_url}" ]] || return 1 UPDATE_RECOVERY_DATABASE_URL="${db_url}" UPDATE_RECOVERY_DATABASE_DUMP="${backup_dir}/database-before-update.dump" UPDATE_RECOVERY_DATABASE_CONTRACT_VARIANT="${contract_variant}" restore_database_after_failed_update || return 1 fi advance_update_transaction_phase \ active recovery_database_restore_pending recovery_database_restored \ || return 1 ;; recovery_database_restored) advance_update_transaction_phase \ active recovery_database_restored recovery_caddy_restore_pending \ || return 1 ;; recovery_caddy_restore_pending) if update_recovery_reached_phase caddy_update_pending; then restore_update_caddy_snapshot || return 1 fi advance_update_transaction_phase \ active recovery_caddy_restore_pending recovery_caddy_restored \ || return 1 ;; recovery_caddy_restored) advance_update_transaction_phase \ active recovery_caddy_restored recovery_environment_restore_pending \ || return 1 ;; recovery_environment_restore_pending) if update_recovery_reached_phase environment_update_pending; then restore_update_environment_snapshot || return 1 fi advance_update_transaction_phase \ active recovery_environment_restore_pending recovery_environment_restored \ || return 1 ;; recovery_environment_restored) load_update_origin_from_environment \ "${PORTAL_DIR}/backend/.env.production" || return 1 advance_update_transaction_phase \ active recovery_environment_restored recovery_runtime_restore_pending \ || return 1 ;; recovery_runtime_restore_pending) if update_recovery_reached_phase runtime_overlay_pending; then restore_update_runtime_snapshot || return 1 fi advance_update_transaction_phase \ active recovery_runtime_restore_pending recovery_runtime_restored \ || return 1 ;; recovery_runtime_restored) advance_update_transaction_phase \ active recovery_runtime_restored recovery_dependencies_restore_pending \ || return 1 ;; recovery_dependencies_restore_pending) if update_recovery_reached_phase dependencies_update_pending; then restore_update_dependencies_snapshot || return 1 fi advance_update_transaction_phase \ active recovery_dependencies_restore_pending recovery_dependencies_restored \ || return 1 ;; recovery_dependencies_restored) advance_update_transaction_phase \ active recovery_dependencies_restored recovery_provenance_restore_pending \ || return 1 ;; recovery_provenance_restore_pending) if update_recovery_reached_phase provenance_commit_pending; then restore_update_provenance_snapshot || return 1 fi load_update_origin_from_environment \ "${PORTAL_DIR}/backend/.env.production" || return 1 probe_token="$(read_env_value \ "${PORTAL_DIR}/backend/.env.production" \ "PORTAL_UPDATE_PROBE_TOKEN" || true)" if portal_runtime_supports_update_validation_contract "${PORTAL_DIR}"; then [[ -n "${probe_token}" ]] || return 1 start_update_candidate \ "${UPDATE_TRANSACTION_ID}" "${UPDATE_CANDIDATE_PORT}" || return 1 verify_portal_update_readiness \ "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" "${probe_token}" 900 \ "$(update_candidate_unit_name "${UPDATE_TRANSACTION_ID}")" \ "http://127.0.0.1:${UPDATE_CANDIDATE_PORT}/health/update-ready" candidate \ || return 1 stop_update_candidate \ "${UPDATE_TRANSACTION_ID}" "${UPDATE_CANDIDATE_PORT}" || return 1 else # Public 3.x and the immediately preceding 4.0 runtime predate the # immutable validation-protocol marker. Never boot those runtimes as # a second private Portal: attest the restored bytes now, then start # the canonical service exactly once below. A token-bearing legacy # 4.x endpoint is verified in explicit legacy-canonical mode, where # the updateValidation fields must be absent rather than false. [[ "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" == 3.* \ || "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" == 4.* ]] || return 1 [[ "$(attest_existing_portal_for_update "${PORTAL_DIR}")" \ == "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" ]] || return 1 fi advance_update_transaction_phase \ active recovery_provenance_restore_pending recovery_provenance_restored \ || return 1 ;; recovery_provenance_restored) advance_update_transaction_phase \ active recovery_provenance_restored recovery_cutover_pending || return 1 ;; recovery_cutover_pending|recovery_service_restore_pending|recovery_service_restored|recovery_verify_pending) local target load_update_origin_from_environment \ "${PORTAL_DIR}/backend/.env.production" || return 1 target="$(current_update_receipt_target)" || return 1 if [[ "${target}" == "active" ]]; then cutover_update_transaction "${phase}" || return 1 continue fi case "${phase}" in recovery_cutover_pending) advance_update_transaction_phase \ cutover recovery_cutover_pending recovery_service_restore_pending \ || return 1 ;; recovery_service_restore_pending) probe_token="$(read_env_value \ "${PORTAL_DIR}/backend/.env.production" \ "PORTAL_UPDATE_PROBE_TOKEN" || true)" if ! restore_and_verify_canonical_portal_state \ "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" "${probe_token}"; then # The rollback target may already be restored and serving. # Re-fencing then stops a healthy Portal and leaves the machine # with no Portal at all, which is strictly worse than an # unfinished journal entry. Only fence when nothing is serving. if canonical_portal_is_serving_version \ "${UPDATE_TRANSACTION_PREVIOUS_VERSION}"; then warn "Rollback verification did not fully pass, but Portal ${UPDATE_TRANSACTION_PREVIOUS_VERSION} is running and healthy. Leaving it in service; the recovery journal and rollback artifacts were preserved." return 1 fi refence_cutover_recovery || true return 1 fi advance_update_transaction_phase \ cutover recovery_service_restore_pending recovery_service_restored \ || return 1 ;; recovery_service_restored) advance_update_transaction_phase \ cutover recovery_service_restored recovery_verify_pending || return 1 ;; recovery_verify_pending) probe_token="$(read_env_value \ "${PORTAL_DIR}/backend/.env.production" \ "PORTAL_UPDATE_PROBE_TOKEN" || true)" if ! verify_canonical_portal_for_transaction \ "${UPDATE_TRANSACTION_PREVIOUS_VERSION}" "${probe_token}" true; then refence_cutover_recovery || true return 1 fi advance_update_transaction_phase \ cutover recovery_verify_pending recovered || return 1 ;; esac ;; recovered) complete_update_transaction cutover recovered return ;; *) return 1 ;; esac done } update_recovery_phase_rank() { case "$1" in recovery_pending) echo 0 ;; recovery_quiesce_pending) echo 1 ;; recovery_quiesced) echo 2 ;; recovery_openclaw_restore_pending) echo 3 ;; recovery_openclaw_restored) echo 4 ;; recovery_database_restore_pending) echo 5 ;; recovery_database_restored) echo 6 ;; recovery_caddy_restore_pending) echo 7 ;; recovery_caddy_restored) echo 8 ;; recovery_environment_restore_pending) echo 9 ;; recovery_environment_restored) echo 10 ;; recovery_runtime_restore_pending) echo 11 ;; recovery_runtime_restored) echo 12 ;; recovery_dependencies_restore_pending) echo 13 ;; recovery_dependencies_restored) echo 14 ;; recovery_provenance_restore_pending) echo 15 ;; recovery_provenance_restored) echo 16 ;; recovery_cutover_pending) echo 17 ;; recovery_service_restore_pending) echo 18 ;; recovery_service_restored) echo 19 ;; recovery_verify_pending) echo 20 ;; recovered) echo 21 ;; *) return 1 ;; esac } assert_update_recovery_preconditions_before_emergency_release() { # This function is deliberately read-only. It proves every feasible # authority, path, artifact, and PostgreSQL prerequisite before the final # reserve is consumed. Any failure leaves every emergency inode allocated # for an administrator to correct the non-space problem and retry. local target="$1" phase="$2" actual_target transaction_id local receipt_system_identifier receipt_topology_digest local backup_dir stage_dir transaction_dir state_root backup_root stage_root local test_root="" recovery_from="${phase}" from_rank recovery_rank=-1 local env_file db_url contract_variant manifest local -a pinned=() [[ "${target}" =~ ^(active|cutover)$ && -n "${phase}" ]] || return 1 actual_target="$(current_update_receipt_target)" || return 1 [[ "${actual_target}" == "${target}" ]] || return 1 load_update_transaction_context "${target}" || return 1 transaction_id="${UPDATE_TRANSACTION_ID}" receipt_system_identifier="$( read_update_transaction_field "${target}" database_system_identifier )" || return 1 receipt_topology_digest="$( read_update_transaction_field "${target}" database_topology_sha256 )" || return 1 mapfile -t pinned < <( read_update_disk_reserve_database_topology "${transaction_id}" ) || return 1 [[ "${#pinned[@]}" -eq 3 \ && "${pinned[0]}" == "${receipt_system_identifier}" \ && "${pinned[1]}" == "${receipt_topology_digest}" ]] || return 1 backup_dir="$(read_update_transaction_field "${target}" backup_dir)" \ || return 1 stage_dir="$(read_update_transaction_field "${target}" stage_dir)" \ || return 1 transaction_dir="$( read_update_transaction_field "${target}" transaction_dir )" || return 1 state_root="$(update_transaction_state_path "${UPDATE_STATE_ROOT}")" \ || return 1 backup_root="$(update_transaction_state_path "${UPDATE_BACKUP_ROOT}")" \ || return 1 stage_root="$(update_transaction_state_path "${UPDATE_STAGE_ROOT}")" \ || return 1 test_root="$(update_transaction_test_root 2>/dev/null || true)" python3 - "${transaction_id}" "${backup_dir}" "${stage_dir}" \ "${transaction_dir}" "${state_root}" "${backup_root}" "${stage_root}" \ "${test_root}" <<'PY2' || return 1 import os import stat import sys ( transaction_id, backup_dir, stage_dir, transaction_dir, state_root, backup_root, stage_root, test_root, ) = sys.argv[1:] expected = ( os.path.join(backup_root, f"update-{transaction_id}"), os.path.join(stage_root, f"update-{transaction_id}"), os.path.join(state_root, "transactions", transaction_id), ) if (backup_dir, stage_dir, transaction_dir) != expected: raise SystemExit(1) if test_root: root = os.path.realpath(test_root) if any( os.path.commonpath((root, os.path.realpath(path))) != root for path in (*expected, state_root, backup_root, stage_root) ): raise SystemExit(1) stage_payload = expected[1] for path in (*expected, state_root, backup_root, stage_root): try: details = os.lstat(path) except FileNotFoundError: # The state helper's terminal cleanup removes secret-bearing payload # directories while the receipt still exists, so an absent stage # payload is a legitimate post-cleanup state. Every other directory # in this layout must exist; a present stage must still pass the # full secure-directory contract below. if path == stage_payload: continue raise SystemExit(1) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != os.geteuid() or details.st_gid != os.getegid() or details.st_mode & 0o077 ): raise SystemExit(1) PY2 if [[ "${phase}" == recovery_* && "${phase}" != "recovered" ]]; then recovery_from="$( read_update_transaction_field "${target}" recovery_from_phase )" || return 1 recovery_rank="$(update_recovery_phase_rank "${phase}")" || return 1 fi from_rank="$(update_forward_phase_rank "${recovery_from}")" || return 1 if (( from_rank >= $(update_forward_phase_rank portal_quiesce_pending) )); then manifest="$( update_transaction_manifest_path "${target}" running-app-intent )" || return 1 update_artifact_manifest verify \ "${backup_dir}/running-app-intent.json" "${manifest}" false \ || return 1 fi # Runtime/environment snapshots exist before any mutable Portal layer. Use # the original protected environment as database authority whenever it has # been captured; before that point the canonical environment is unchanged. env_file="${PORTAL_DIR}/backend/.env.production" if (( from_rank >= $(update_forward_phase_rank runtime_snapshot_complete) )); then manifest="$(update_transaction_manifest_path "${target}" environment)" \ || return 1 update_artifact_manifest verify \ "${backup_dir}/environment" "${manifest}" false || return 1 env_file="${backup_dir}/environment/backend.env.production" fi assert_env_file_no_duplicate_keys "${env_file}" || return 1 load_update_origin_from_environment "${env_file}" || return 1 db_url="$(read_env_value "${env_file}" "DATABASE_URL" || true)" [[ -n "${db_url}" ]] || return 1 assert_update_database_topology_unchanged \ "${db_url}" recovery-emergency-release || return 1 if (( from_rank >= $(update_forward_phase_rank database_migration_pending) \ && (recovery_rank < 0 \ || recovery_rank <= $(update_recovery_phase_rank recovery_database_restore_pending)) )); then manifest="$(update_transaction_manifest_path "${target}" database)" \ || return 1 update_artifact_manifest verify \ "${backup_dir}/database-before-update.dump" "${manifest}" false \ || return 1 contract_variant="$( read_update_database_contract_snapshot "${target}" "${backup_dir}" )" || return 1 [[ "${contract_variant}" =~ ^(owner-null|pg-database-owner-default)$ ]] \ || return 1 fi if (( from_rank >= $(update_forward_phase_rank caddy_update_pending) \ && (recovery_rank < 0 \ || recovery_rank <= $(update_recovery_phase_rank recovery_caddy_restore_pending)) )); then manifest="$(update_transaction_manifest_path "${target}" caddy)" \ || return 1 update_artifact_manifest verify \ "${backup_dir}/Caddyfile" "${manifest}" true || return 1 fi if (( from_rank >= $(update_forward_phase_rank environment_update_pending) \ && (recovery_rank < 0 \ || recovery_rank <= $(update_recovery_phase_rank recovery_environment_restore_pending)) )); then manifest="$(update_transaction_manifest_path "${target}" environment-updated)" \ || return 1 update_artifact_manifest verify \ "${backup_dir}/environment.updated" "${manifest}" false || return 1 fi if (( from_rank >= $(update_forward_phase_rank runtime_overlay_pending) \ && (recovery_rank < 0 \ || recovery_rank <= $(update_recovery_phase_rank recovery_runtime_restore_pending)) )); then manifest="$(update_transaction_manifest_path "${target}" runtime)" \ || return 1 update_artifact_manifest verify \ "${backup_dir}/portal-runtime" "${manifest}" false || return 1 fi if (( from_rank >= $(update_forward_phase_rank dependencies_update_pending) \ && (recovery_rank < 0 \ || recovery_rank <= $(update_recovery_phase_rank recovery_dependencies_restore_pending)) )); then manifest="$(update_transaction_manifest_path "${target}" dependencies)" \ || return 1 update_artifact_manifest verify \ "${backup_dir}/backend-node-modules" "${manifest}" true || return 1 fi if (( from_rank >= $(update_forward_phase_rank provenance_commit_pending) \ && (recovery_rank < 0 \ || recovery_rank <= $(update_recovery_phase_rank recovery_provenance_restore_pending)) )); then manifest="$(update_transaction_manifest_path "${target}" provenance)" \ || return 1 update_artifact_manifest verify \ "${backup_dir}/last-portal-deploy.previous" "${manifest}" true \ || return 1 fi } recover_pending_update_transaction() { ${UPDATE_RECOVERY_IN_PROGRESS:-false} && return 1 UPDATE_RECOVERY_IN_PROGRESS=true trap - ERR trap '' SIGINT TERM HUP set +e local target phase result=1 target="$(current_update_receipt_target 2>/dev/null || true)" [[ "${target}" =~ ^(active|cutover)$ ]] || return 1 phase="$(read_update_transaction_field "${target}" phase 2>/dev/null || true)" # A transaction created by an older installer can resume through this newer # recovery engine. Install and attest the static OpenClaw start inhibitor # before any rollback or forward-cutover path can boot Portal 4.0 code. install_openclaw_gateway_authorization_fence_dropin || return 1 case "${phase}" in committed|recovered) # Terminal cleanup owns the full reserve release and can resume a # partially journaled cleanup after SIGKILL. ;; *) prepare_update_disk_reserves_for_recovery || return 1 ;; esac # Forward cutover needs no rollback space and must not burn the emergency # tier merely to start or verify a service. Every path that is entering or # resuming rollback first proves all read-only preconditions, then consumes # an explicitly escalated tier immediately before its first durable write. case "${target}:${phase}" in cutover:cutover_pending|cutover:canonical_start_pending|\ cutover:canonical_started|cutover:canonical_verification_pending|\ cutover:canonical_verified|cutover:boot_state_restore_pending|\ cutover:boot_state_restored|cutover:committed|cutover:recovered) ;; *) assert_update_recovery_preconditions_before_emergency_release \ "${target}" "${phase}" || return 1 consume_update_emergency_disk_reserves_for_recovery || return 1 ;; esac if [[ "${target}" == "cutover" ]]; then phase="$(read_update_transaction_field cutover phase 2>/dev/null || true)" case "${phase}" in committed) complete_committed_update_transaction result=$? ;; cutover_pending|canonical_start_pending|canonical_started|canonical_verification_pending|canonical_verified|boot_state_restore_pending|boot_state_restored) finish_forward_cutover_transaction result=$? ;; recovery_cutover_pending|recovery_service_restore_pending|recovery_service_restored|recovery_verify_pending) if reopen_cutover_update_transaction \ && install_portal_update_boot_fence \ && quiesce_portal_service "bridgesllm-product" 45 15 4001; then recover_active_update_transaction result=$? fi ;; recovered) complete_update_transaction cutover recovered result=$? ;; esac elif [[ "${target}" == "active" ]]; then phase="$(read_update_transaction_field active phase 2>/dev/null || true)" if [[ "${phase}" == recovery_* ]]; then install_portal_update_boot_fence \ && recover_active_update_transaction result=$? elif [[ -n "${phase}" && "${phase}" != "committed" ]]; then if advance_update_transaction_phase \ active "${phase}" recovery_pending \ && install_portal_update_boot_fence; then recover_active_update_transaction result=$? fi fi fi if [[ "${result}" -eq 0 ]]; then set -e UPDATE_RECOVERY_IN_PROGRESS=false trap 'handle_err $LINENO' ERR trap handle_sigint SIGINT trap handle_sigterm TERM trap handle_sighup HUP else # A failed recovery is terminal for this process: keep every trap # disabled and the in-progress latch set so no second attempt and no # secondary failure handler can run. The journal deliberately retains # the last pending phase for the next installer invocation to resume. trap '' ERR SIGINT TERM HUP fi return "${result}" } do_update() { banner echo "" if $REPAIR_REINSTALL; then echo -e " ${BOLD}${WHITE}Repairing BridgesLLM Portal installation${NC}" else echo -e " ${BOLD}${WHITE}Updating BridgesLLM Portal${NC}" fi echo "" CURRENT_STEP="update" dashboard_update_progress running 10 host-safety \ "Checking host safety" \ "Step 2 of 13 · Inspecting scheduled Docker cleanup without changing unrelated host jobs." converge_unsafe_docker_prune_automation \ || fail "Unsafe scheduled Docker cleanup remains active. Review the guard details above, disable the unknown job or repair the legacy-file drift, and retry." local previous_portal_version="" previous_portal_version="$(attest_existing_portal_for_update "${PORTAL_DIR}")" \ || fail "The existing Portal runtime or configuration is incomplete, linked, writable by another account, or version-inconsistent. Repair it explicitly before updating." node_version_meets_minimum \ || fail "The running Portal uses a Node.js release outside ${OPENCLAW_NODE_ENGINE_RANGE}. Update the shared Node runtime first (Node 22 must be >=22.22.3 and <23), then retry. The updater refused before its boot fence and will not replace Node without a rollback boundary. See docs/PORTAL_NODE_RUNTIME_REMEDIATION.md." dashboard_update_progress running 16 portal-preflight \ "Validating current Portal and recovery prerequisites" \ "Step 3 of 13 · The installed runtime and shared Node.js boundary are attested." load_existing_telemetry_install_id ensure_telemetry_install_id local env_file="${PORTAL_DIR}/backend/.env.production" # Environment authority must be unambiguous before anything reads it: a key # assigned twice makes installer reads and systemd's effective (last-wins) # value diverge, splitting database/origin/probe authority mid-transaction. assert_env_file_no_duplicate_keys "${env_file}" \ || fail "The existing backend environment file is missing, linked, or assigns a variable more than once; the installer and systemd would disagree about the effective configuration. Repair ${env_file} before updating." assert_env_file_no_duplicate_keys "${PORTAL_DIR}/frontend/.env" \ || fail "The existing frontend environment file is missing, linked, or assigns a variable more than once. Repair ${PORTAL_DIR}/frontend/.env before updating." assert_prisma_runtime_environment_safe "${env_file}" \ || fail "The existing backend or installer process environment overrides the attested database runtime. Remove Prisma engine switches, node-postgres PG* fallbacks, NODE_PG_FORCE_NATIVE, and NODE_TLS_REJECT_UNAUTHORIZED before updating." local validation_assignment_status=0 env_file_has_assignment "${env_file}" "PORTAL_UPDATE_VALIDATION_MODE" \ || validation_assignment_status=$? if [[ "${validation_assignment_status}" -ne 1 ]]; then fail "PORTAL_UPDATE_VALIDATION_MODE is a transient updater flag and must never be stored in .env.production. Remove that line before retrying." fi # Read the exact existing database identity. A missing or malformed value # must never drift into the installer's local default. local existing_db_url="" existing_db_url="$(read_env_value "${env_file}" "DATABASE_URL" || true)" local db_component for db_component in host port database user password; do pg_url_component "${existing_db_url}" "${db_component}" >/dev/null \ || fail "The existing DATABASE_URL is missing or malformed; refusing to touch any database." done pg_url_uses_public_schema "${existing_db_url}" \ || fail "The existing DATABASE_URL selects a non-public or ambiguous Prisma schema. Portal rollback is defined only for the public schema; remove the schema option or set schema=public before updating." pg_url_uses_supported_prisma_adapter_options "${existing_db_url}" \ || fail "The existing DATABASE_URL has an ambiguous or unsupported database-driver option. Remote databases must set sslmode=disable, require, verify-ca, or verify-full. An absolute sslrootcert is required for verify-ca/verify-full; sslcert/client identities/keys, sslaccept, and channel_binding are not supported. Only lowercase application_name, fallback_application_name, options, client_encoding, replication, and documented Prisma pool controls may be supplied; literal plus signs must be percent-encoded. Custom connect_timeout and pool_timeout values must match. Repair the URL before updating." PORTAL_UPDATE_PROBE_TOKEN="$( read_env_value "${env_file}" "PORTAL_UPDATE_PROBE_TOKEN" || true )" if [[ -z "${PORTAL_UPDATE_PROBE_TOKEN}" \ && "${previous_portal_version}" != 3.* ]]; then fail "This Portal 4.x installation is missing its authenticated update probe token. Repair the environment before updating." fi [[ -n "${PORTAL_UPDATE_PROBE_TOKEN}" ]] \ || PORTAL_UPDATE_PROBE_TOKEN="$(rand_hex 32)" DB_PASSWORD="$(pg_url_component "${existing_db_url}" password)" \ || fail "The existing database password could not be parsed safely." # Use one identity for the pre-staging reserves, stage tree, and durable # receipt. Publishing it before admission lets reserve files be named and # recovered deterministically even if the installer is SIGKILLed while the # signed candidate is still being staged. UPDATE_TRANSACTION_ID="$(openssl rand -hex 16)" [[ "${UPDATE_TRANSACTION_ID}" =~ ^[a-f0-9]{32}$ ]] \ || fail "Could not create a bounded update transaction identifier." attest_update_database_ownership "${existing_db_url}" admission \ || fail "The existing database does not follow the exact installer ownership contract (public must be the only user schema; every dumpable public-schema object and ACL must have Portal-owned/default state). Rollback could not restore it exactly, so the update refuses to start. Details: ${LOG_FILE}." local disk_admission_status=0 assert_update_transaction_disk_admission "${PORTAL_DIR}" "${existing_db_url}" \ || disk_admission_status=$? if [[ "${disk_admission_status}" -eq 2 ]]; then fail "This host does not have enough free disk for staging, rollback snapshots, the installer journal, and recovery reserve. Free space and retry. Measured shortfall: $(update_disk_admission_last_detail). Details: ${LOG_FILE}." elif [[ "${disk_admission_status}" -ne 0 ]]; then fail "The update refused to start because this host could not be admitted for a safe, rollback-capable upgrade. This is not a disk-space shortfall. Reason: $(update_disk_admission_last_detail). Details: ${LOG_FILE}." fi ensure_media_toolchain \ || fail "Animated GIF uploads require ffmpeg and ffprobe, but the updater could not repair the FFmpeg package while the existing Portal was still online. Check ${LOG_FILE}, repair apt, and retry." dashboard_update_progress running 22 capacity-preflight \ "Checking database, disk, and recovery capacity" \ "Step 4 of 13 · Database ownership, disk reserves, and required host tools passed admission." local existing_install_profile existing_install_profile="$(read_env_value "${env_file}" "INSTALL_PROFILE" || true)" [[ -n "${existing_install_profile}" ]] && INSTALL_PROFILE="${existing_install_profile}" if [[ -z "${ORIGIN_MODE}" ]]; then ORIGIN_MODE="$(read_env_value "${env_file}" "ORIGIN_MODE" || true)" fi if use_tailnet_profile && [[ -z "${TAILNET_DNS_NAME}" ]]; then TAILNET_DNS_NAME="$(read_env_value "${env_file}" "TAILNET_DNS_NAME" || true)" local live_tailnet_name live_tailnet_name="$(tailnet_dns_name_from_status)" [[ -n "${live_tailnet_name}" ]] && TAILNET_DNS_NAME="${live_tailnet_name}" fi [[ -n "${DOMAIN}" ]] || DOMAIN="$(read_env_value "${env_file}" "DOMAIN" || true)" if [[ -z "${APP_CONTENT_DOMAIN}" ]]; then APP_CONTENT_DOMAIN="$(read_env_value "${env_file}" "APP_CONTENT_DOMAIN" || true)" APP_CONTENT_DNS_MODE="$(read_env_value "${env_file}" "APP_CONTENT_DNS_MODE" || true)" if [[ -z "${APP_CONTENT_DOMAIN}" ]]; then local existing_app_content_origin existing_app_content_origin="$(read_env_value "${env_file}" "APP_CONTENT_ORIGIN" || true)" APP_CONTENT_DOMAIN="$(app_content_domain_from_origin "${existing_app_content_origin}" || true)" fi fi # Stage and prepare the candidate while the old Portal remains online. dashboard_update_progress running 24 signed-release \ "Downloading and verifying signed release" \ "Step 5 of 13 · Fetching the versioned manifest, signature, and Portal artifact." UPDATE_RELEASE_STAGE_DIR="$( new_release_stage_dir "${UPDATE_TRANSACTION_ID}" )" || fail "Could not create the private update staging directory." if ! stage_verified_release "${UPDATE_RELEASE_STAGE_DIR}"; then cleanup_release_stage_dir "${UPDATE_RELEASE_STAGE_DIR}" || true UPDATE_RELEASE_STAGE_DIR="" fail "Update release signature, version, digest, or archive validation failed before downtime." fi dashboard_update_progress running 30 signed-release \ "Signed release verified" \ "Step 5 of 13 · Manifest signature, release version, digest, and archive boundaries passed." local staged_update_dir="${UPDATE_RELEASE_STAGE_DIR}/portal" # Public 3.26.x installs never shipped build-essential; candidate native # module verification needs it. Install while the old Portal is still up. ensure_build_tools prepare_staged_backend_runtime_dependencies "${staged_update_dir}" \ || fail "Candidate runtime dependencies could not be prepared and verified before downtime." prepare_update_project_runtimes \ "${staged_update_dir}" "${env_file}" "${UPDATE_TRANSACTION_ID}" if [[ "${previous_portal_version}" != 3.* ]]; then local continuity_repair_plan="${UPDATE_RELEASE_STAGE_DIR}/portal-continuity-repair-plan.json" run_staged_portal_rebootability_preflight \ "${staged_update_dir}" "${env_file}" "${continuity_repair_plan}" \ || fail "The signed candidate could not classify persisted App/Project continuity safely. No database mutation, transaction, or downtime was started; repair the reported ambiguity and retry. Details: ${LOG_FILE}." run_staged_portal_continuity_repair \ "${staged_update_dir}" "${env_file}" "${continuity_repair_plan}" \ || fail "The signed candidate could not apply its exact serializable App continuity repair while the existing Portal was online. No update transaction or downtime was started; the repair refused a race or ambiguous row. Details: ${LOG_FILE}." run_staged_portal_rebootability_preflight \ "${staged_update_dir}" "${env_file}" \ || fail "The signed candidate could not reattest a clean, rebootable App/Project state after continuity repair. The existing Portal remains online and no update transaction or downtime was started. Details: ${LOG_FILE}." else info "Legacy 3.x source detected; the 4.x App identity preflight will run after continuity enrollment." fi disk_admission_status=0 assert_update_transaction_disk_admission \ "${PORTAL_DIR}" "${existing_db_url}" "${staged_update_dir}" \ || disk_admission_status=$? if [[ "${disk_admission_status}" -eq 2 ]]; then fail "The fully staged candidate and additive Project runtime images do not leave enough disk for runtime promotion, rollback snapshots, the installer journal, and recovery. No receipt or downtime was started; free space and retry. Measured shortfall: $(update_disk_admission_last_detail). Details: ${LOG_FILE}." elif [[ "${disk_admission_status}" -ne 0 ]]; then fail "The staged candidate could not be admitted for promotion, and not because of free disk. No receipt or downtime was started. Reason: $(update_disk_admission_last_detail). Details: ${LOG_FILE}." fi dashboard_update_progress running 38 runtime-preparation \ "Preparing dependencies and project runtimes" \ "Step 6 of 13 · Backend dependencies and additive runtime images are staged and verified." recover_domain_from_caddyfile local existing_public_ip detected_public_ip existing_public_ip="$(read_env_value "${env_file}" "PUBLIC_IP" || true)" detected_public_ip="$(curl -4 -fsSL --max-time 8 https://api.ipify.org 2>/dev/null || true)" PUBLIC_IP="${detected_public_ip:-${existing_public_ip}}" if [[ "${APP_CONTENT_DNS_MODE}" == "sslip" ]]; then APP_CONTENT_DOMAIN="" fi configure_app_content_identity capture_portal_service_baseline "bridgesllm-product" \ || fail "Portal service state is masked, static, unknown, or internally inconsistent; refusing an update without an exact restore policy." prepare_update_transaction \ "${previous_portal_version}" "${staged_update_dir}" "${UPDATE_TRANSACTION_ID}" \ || fail "The durable update transaction could not be created safely." dashboard_update_progress running 40 portal-transaction \ "Opening protected update transaction" \ "Step 7 of 13 · A durable rollback journal now protects the live cutover." arm_update_transaction_traps advance_update_transaction_phase active prepared boot_block_pending \ || fail "Could not record the boot-fence write-ahead phase." fence_portal_service_boot \ || fail "The Portal boot fence could not be installed and verified." advance_update_transaction_phase active boot_block_pending boot_blocked \ || fail "Could not commit the boot-fence phase." capture_update_running_app_intent "${existing_db_url}" \ || fail "The exact running full-stack App intent could not be sealed before Portal shutdown." advance_update_transaction_phase \ active boot_blocked portal_quiesce_pending \ || fail "Could not record the Portal quiescence write-ahead phase." quiesce_portal_service "bridgesllm-product" 45 15 4001 \ || fail "Portal could not be proven stopped; no runtime or database mutation was attempted." reconcile_update_running_app_intent active "${existing_db_url}" \ || fail "Running full-stack App intent changed incompatibly during Portal shutdown; the update was rolled back." advance_update_transaction_phase \ active portal_quiesce_pending portal_quiesced \ || fail "Could not commit the Portal quiescence phase." reattest_quiesced_update_project_egress_generation \ "${UPDATE_RELEASE_STAGE_DIR}" "${UPDATE_TRANSACTION_ID}" \ || fail "Managed Project egress resources changed after candidate preparation. The update was rolled back before runtime or database mutation; retry after Project activity is quiet." advance_update_transaction_phase \ active portal_quiesced runtime_snapshot_pending \ || fail "Could not record the runtime snapshot phase." snapshot_update_runtime_and_environment \ || fail "The exact runtime and environment rollback snapshots could not be created." snapshot_update_provenance \ || fail "The deploy-provenance rollback snapshot could not be created." advance_update_transaction_phase \ active runtime_snapshot_pending runtime_snapshot_complete \ || fail "Could not commit the runtime snapshot phase." advance_update_transaction_phase \ active runtime_snapshot_complete database_snapshot_pending \ || fail "Could not record the database snapshot phase." snapshot_update_database "${existing_db_url}" \ || fail "The PostgreSQL rollback snapshot could not be created and verified." advance_update_transaction_phase \ active database_snapshot_pending database_snapshot_complete \ || fail "Could not commit the database snapshot phase." advance_update_transaction_phase \ active database_snapshot_complete caddy_snapshot_pending \ || fail "Could not record the routing snapshot phase." snapshot_update_caddy \ || fail "The exact Caddy rollback snapshot could not be created." advance_update_transaction_phase \ active caddy_snapshot_pending caddy_snapshot_complete \ || fail "Could not commit the routing snapshot phase." advance_update_transaction_phase \ active caddy_snapshot_complete openclaw_snapshot_pending \ || fail "Could not record the ancillary-state snapshot phase." # Global OpenClaw and host-tool convergence is post-commit. These phases are # a proven no-op because a Portal snapshot cannot restore global packages. advance_update_transaction_phase \ active openclaw_snapshot_pending openclaw_snapshot_complete \ || fail "Could not commit the ancillary-state snapshot phase." advance_update_transaction_phase \ active openclaw_snapshot_complete node_modules_move_pending \ || fail "Could not record the dependency snapshot phase." snapshot_update_dependencies \ || fail "The exact backend dependency rollback snapshot could not be created." advance_update_transaction_phase \ active node_modules_move_pending node_modules_moved \ || fail "Could not commit the dependency snapshot phase." prepare_updated_environment_candidate \ || fail "The updated environment candidate could not be prepared without mutating the live installation." seal_update_project_runtime_promotion_manifest active \ || fail "Prepared Project runtime identities could not be sealed in the non-secret transaction journal." advance_update_transaction_phase \ active node_modules_moved runtime_overlay_pending \ || fail "Could not record the runtime overlay write-ahead phase." overlay_verified_update_runtime "${staged_update_dir}" \ || fail "The signed Portal runtime could not be overlaid exactly." advance_update_transaction_phase \ active runtime_overlay_pending runtime_overlaid \ || fail "Could not commit the runtime overlay phase." advance_update_transaction_phase \ active runtime_overlaid environment_update_pending \ || fail "Could not record the environment update write-ahead phase." install_updated_environment_candidate \ || fail "The verified environment candidate could not be installed atomically." advance_update_transaction_phase \ active environment_update_pending environment_updated \ || fail "Could not commit the environment update phase." advance_update_transaction_phase \ active environment_updated caddy_update_pending \ || fail "Could not record the routing update write-ahead phase." if ! use_local_profile && ! use_tailnet_profile; then write_caddy_config \ || fail "The managed Caddy transaction failed; unrelated sites were preserved." fi advance_update_transaction_phase \ active caddy_update_pending caddy_updated \ || fail "Could not commit the routing update phase." advance_update_transaction_phase \ active caddy_updated dependencies_update_pending \ || fail "Could not record the dependency replacement write-ahead phase." install_prepared_update_dependencies "${staged_update_dir}" \ || fail "The preverified backend dependencies could not be promoted." advance_update_transaction_phase \ active dependencies_update_pending dependencies_updated \ || fail "Could not commit the dependency replacement phase." advance_update_transaction_phase \ active dependencies_updated database_migration_pending \ || fail "Could not record the database migration write-ahead phase." run_migrations_safe "${existing_db_url}" advance_update_transaction_phase \ active database_migration_pending database_migrated \ || fail "Could not commit the database migration phase." advance_update_transaction_phase \ active database_migrated openclaw_update_pending \ || fail "Could not record the post-Portal dependency phase." advance_update_transaction_phase \ active openclaw_update_pending openclaw_updated \ || fail "Could not commit the post-Portal dependency phase." assert_prepared_update_project_runtime_images_available active \ || fail "Prepared Project runtime images disappeared before candidate validation." advance_update_transaction_phase \ active openclaw_updated candidate_start_pending \ || fail "Could not record the private candidate start phase." start_update_candidate "${UPDATE_TRANSACTION_ID}" "${UPDATE_CANDIDATE_PORT}" \ || fail "The signed Portal candidate could not start in its private validation unit." advance_update_transaction_phase \ active candidate_start_pending candidate_started \ || fail "Could not commit the private candidate start phase." advance_update_transaction_phase \ active candidate_started candidate_verification_pending \ || fail "Could not record the private candidate verification phase." verify_update_candidate \ "${UPDATE_TRANSACTION_ID}" "${VERSION}" \ "${PORTAL_UPDATE_PROBE_TOKEN}" "${UPDATE_CANDIDATE_PORT}" \ || fail "The private candidate failed exact version, schema, or mutation-free readiness verification." advance_update_transaction_phase \ active candidate_verification_pending candidate_verified \ || fail "Could not commit the private candidate verification phase." advance_update_transaction_phase \ active candidate_verified provenance_commit_pending \ || fail "Could not record the deploy-provenance write-ahead phase." commit_portal_deploy_provenance \ || fail "Verified candidate provenance could not be committed atomically." advance_update_transaction_phase \ active provenance_commit_pending provenance_committed \ || fail "Could not commit the deploy-provenance phase." advance_update_transaction_phase \ active provenance_committed cutover_pending \ || fail "Could not record the canonical cutover decision." dashboard_update_progress running 88 portal-cutover \ "Switching to the verified Portal" \ "Step 11 of 13 · The private candidate passed; publishing the durable cutover decision now." # The receipt move is the durable commit decision. Before it, recovery rolls # back. After it, recovery finishes the already verified forward cutover. cutover_update_transaction cutover_pending \ || fail "The atomic Portal cutover decision could not be published." finish_forward_cutover_transaction \ || fail "Canonical Portal cutover did not complete; recovery was re-fenced." UPDATE_RELEASE_STAGE_DIR="" trap 'handle_err $LINENO' ERR trap handle_sigint SIGINT trap handle_sigterm TERM trap handle_sighup HUP # Host-wide tools and optional runtimes stay outside the Portal transaction. # The new Portal is already loaded, schema-ready, and provenance-attested. DASHBOARD_UPDATE_PORTAL_COMMITTED=true dashboard_update_progress running 97 postflight \ "Completing host services and cleanup" \ "Step 13 of 13 · Portal is online while host integration and optional runtimes converge." update_dependencies telemetry_event "deps_updated" info "Checking Remote Desktop..." setup_remote_desktop configure_backup_timers ensure_project_egress_token_secret "${env_file}" ensure_docker_address_pools provision_project_runtimes "${env_file}" ensure_agent_zero_project_model_bridge if systemctl is-enabled openclaw-gateway &>/dev/null 2>&1; then prepare_openclaw_runtime_for_portal fi if command -v openclaw &>/dev/null; then openclaw devices approve --latest >> "$LOG_FILE" 2>&1 || true fi # This proof belongs after every post-commit host mutation. The outer # systemd finalizer accepts the 99% checkpoint as its success prerequisite, # so do not publish that checkpoint from a generic, spoofable health body. # Reuse the authenticated update-ready contract, systemd activity/PID proof, # exact version, schema, and canonical-route checks that guarded cutover. verify_canonical_portal_for_transaction \ "${VERSION}" "${PORTAL_UPDATE_PROBE_TOKEN}" \ || fail "Final authenticated Portal readiness verification failed after host integration." dashboard_update_progress running 99 postflight \ "Finalizing update receipt" \ "Step 13 of 13 · Host integration and authenticated exact-version health verification passed." echo "" echo -e " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e " ${GREEN}${BOLD} Update complete!${NC}" telemetry_event "update_complete" echo -e " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" } write_retained_install_receipt() { local portal_dir="${1:-${PORTAL_DIR}}" local marker_path="${2:-${RETAINED_INSTALL_MARKER}}" local manifest_path="${3:-${RETAINED_INSTALL_MANIFEST}}" local env_file="${portal_dir}/backend/.env.production" [[ -f "${env_file}" && ! -L "${env_file}" ]] || return 1 assert_env_file_no_duplicate_keys "${env_file}" || return 1 python3 - "${portal_dir}" "${marker_path}" "${manifest_path}" "${VERSION}" <<'PY' import datetime import hashlib import json import os import pathlib import stat import sys import tempfile portal_raw, marker_raw, manifest_raw, version = sys.argv[1:] portal = pathlib.Path(portal_raw) marker = pathlib.Path(marker_raw) manifest = pathlib.Path(manifest_raw) def bounded_absolute(path: pathlib.Path) -> pathlib.Path: value = str(path) if not value.startswith("/") or os.path.normpath(value) != value or value == "/": raise ValueError("retained path is not a bounded canonical absolute path") return path for item in (portal, marker, manifest): bounded_absolute(item) if marker.parent != manifest.parent or portal.parent != marker.parent: raise ValueError("retained receipt paths do not share the installation boundary") current = pathlib.Path("/") for component in portal.parts[1:]: current /= component info = os.lstat(current) if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022): raise ValueError("retained Portal path crosses an unsafe ownership boundary") if os.path.realpath(portal) != str(portal): raise ValueError("retained Portal path does not resolve to itself") allowed_roots = { ".data", "apps", "assets", "backend", "frontend", "projects", "upload-temp", } allowed_backend = {".data", ".env", ".env.production"} allowed_frontend = {".env"} entries = [] def digest_file(path: pathlib.Path, initial: os.stat_result) -> str: digest = hashlib.sha256() with path.open("rb", buffering=0) as handle: while True: chunk = handle.read(1024 * 1024) if not chunk: break digest.update(chunk) final = os.lstat(path) identity = ( initial.st_dev, initial.st_ino, initial.st_mode, initial.st_uid, initial.st_gid, initial.st_size, initial.st_mtime_ns, ) if identity != ( final.st_dev, final.st_ino, final.st_mode, final.st_uid, final.st_gid, final.st_size, final.st_mtime_ns, ): raise ValueError("retained file changed while it was inventoried") return digest.hexdigest() for child in sorted(portal.iterdir(), key=lambda item: item.name): if child.name not in allowed_roots: raise ValueError(f"unexpected retained top-level entry: {child.name}") for directory, dirnames, filenames in os.walk(portal, topdown=True, followlinks=False): base = pathlib.Path(directory) relative_base = base.relative_to(portal) if relative_base == pathlib.Path("."): dirnames[:] = sorted(name for name in dirnames if name in allowed_roots) elif relative_base == pathlib.Path("backend"): unexpected = set(dirnames) | set(filenames) unexpected -= allowed_backend if unexpected: raise ValueError("unexpected retained backend payload") elif relative_base == pathlib.Path("frontend"): unexpected = set(dirnames) | set(filenames) unexpected -= allowed_frontend if unexpected: raise ValueError("unexpected retained frontend payload") dirnames.sort() filenames.sort() for name in [*dirnames, *filenames]: path = base / name relative = path.relative_to(portal).as_posix() info = os.lstat(path) common = { "path": relative, "mode": stat.S_IMODE(info.st_mode), "uid": info.st_uid, "gid": info.st_gid, } if stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode): entries.append({**common, "type": "directory"}) elif stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode): if info.st_nlink != 1: raise ValueError("retained file has multiple hard links") entries.append({ **common, "type": "file", "size": info.st_size, "sha256": digest_file(path, info), }) elif stat.S_ISLNK(info.st_mode): target = os.readlink(path) resolved = os.path.realpath(path) if (relative != "backend/.env" or target != ".env.production" or resolved != str(portal / "backend" / ".env.production")): raise ValueError("retained tree contains an unsafe symbolic link") entries.append({**common, "type": "symlink", "target": target}) if name in dirnames: dirnames.remove(name) else: raise ValueError("retained tree contains a special inode") env = portal / "backend" / ".env.production" env_info = os.lstat(env) if (not stat.S_ISREG(env_info.st_mode) or stat.S_ISLNK(env_info.st_mode) or env_info.st_uid != 0 or env_info.st_nlink != 1 or env_info.st_mode & 0o022 or env_info.st_size <= 0 or env_info.st_size > 1024 * 1024): raise ValueError("retained environment is unsafe") env_sha256 = digest_file(env, env_info) document = { "schema": "bridgesllm.retained-install-tree.v1", "portalRoot": str(portal), "entries": sorted(entries, key=lambda entry: entry["path"]), } encoded = (json.dumps(document, indent=2, sort_keys=True) + "\n").encode() def publish(path: pathlib.Path, payload: bytes) -> None: path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) parent_info = os.lstat(path.parent) if (not stat.S_ISDIR(parent_info.st_mode) or stat.S_ISLNK(parent_info.st_mode) or parent_info.st_uid != 0 or parent_info.st_mode & 0o022): raise ValueError("retained receipt parent is unsafe") fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) try: os.fchmod(fd, 0o600) with os.fdopen(fd, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) temporary = "" os.chmod(path, 0o600) parent_fd = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(parent_fd) finally: os.close(parent_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass publish(manifest, encoded) manifest_sha256 = hashlib.sha256(encoded).hexdigest() marker_document = { "schema": "bridgesllm.retained-install.v1", "installerVersion": version, "createdAt": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), "installRoot": str(marker.parent), "portalRoot": str(portal), "manifestPath": str(manifest), "manifestSha256": manifest_sha256, "environmentSha256": env_sha256, } publish(marker, (json.dumps(marker_document, indent=2, sort_keys=True) + "\n").encode()) PY } verify_retained_install_receipt() { local portal_dir="${1:-${PORTAL_DIR}}" local marker_path="${2:-${RETAINED_INSTALL_MARKER}}" local manifest_path="${3:-${RETAINED_INSTALL_MANIFEST}}" local allow_runtime_overlay="${4:-false}" [[ "${allow_runtime_overlay}" == "true" || "${allow_runtime_overlay}" == "false" ]] \ || return 1 [[ -f "${marker_path}" && ! -L "${marker_path}" \ && -f "${manifest_path}" && ! -L "${manifest_path}" ]] || return 1 [[ "$(stat -c '%u:%g:%a' "${marker_path}" 2>/dev/null)" == '0:0:600' \ && "$(stat -c '%u:%g:%a' "${manifest_path}" 2>/dev/null)" == '0:0:600' ]] \ || return 1 python3 - "${portal_dir}" "${marker_path}" "${manifest_path}" "${allow_runtime_overlay}" <<'PY' import hashlib import json import os import pathlib import stat import sys portal = pathlib.Path(sys.argv[1]) marker_path = pathlib.Path(sys.argv[2]) manifest_path = pathlib.Path(sys.argv[3]) allow_runtime_overlay = sys.argv[4] == "true" try: marker_raw = marker_path.read_bytes() manifest_raw = manifest_path.read_bytes() marker = json.loads(marker_raw) manifest = json.loads(manifest_raw) except (OSError, ValueError, json.JSONDecodeError): raise SystemExit(1) if ( marker.get("schema") != "bridgesllm.retained-install.v1" or marker.get("installRoot") != str(portal.parent) or marker.get("portalRoot") != str(portal) or marker.get("manifestPath") != str(manifest_path) or marker.get("manifestSha256") != hashlib.sha256(manifest_raw).hexdigest() or manifest.get("schema") != "bridgesllm.retained-install-tree.v1" or manifest.get("portalRoot") != str(portal) or not isinstance(manifest.get("entries"), list) ): raise SystemExit(1) expected = manifest["entries"] if len(expected) > 2_000_000: raise SystemExit(1) observed = [] def retained_path(relative: str) -> bool: parts = pathlib.PurePosixPath(relative).parts if not parts: return False if parts[0] in {".data", "apps", "assets", "projects", "upload-temp"}: return True if parts[0] == "backend": return len(parts) == 1 or (len(parts) >= 2 and parts[1] in {".data", ".env", ".env.production"}) if parts[0] == "frontend": return len(parts) == 1 or (len(parts) == 2 and parts[1] == ".env") return False for directory, dirnames, filenames in os.walk(portal, topdown=True, followlinks=False): base = pathlib.Path(directory) dirnames.sort() filenames.sort() for name in [*dirnames, *filenames]: path = base / name relative = path.relative_to(portal).as_posix() if allow_runtime_overlay and not retained_path(relative): continue info = os.lstat(path) common = { "path": relative, "mode": stat.S_IMODE(info.st_mode), "uid": info.st_uid, "gid": info.st_gid, } if stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode): observed.append({**common, "type": "directory"}) elif stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode): if info.st_nlink != 1: raise SystemExit(1) digest = hashlib.sha256() with path.open("rb", buffering=0) as handle: while True: chunk = handle.read(1024 * 1024) if not chunk: break digest.update(chunk) final = os.lstat(path) if ( info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_size, info.st_mtime_ns, ) != ( final.st_dev, final.st_ino, final.st_mode, final.st_uid, final.st_gid, final.st_size, final.st_mtime_ns, ): raise SystemExit(1) observed.append({ **common, "type": "file", "size": info.st_size, "sha256": digest.hexdigest(), }) elif stat.S_ISLNK(info.st_mode): observed.append({**common, "type": "symlink", "target": os.readlink(path)}) if name in dirnames: dirnames.remove(name) else: raise SystemExit(1) if sorted(observed, key=lambda entry: entry["path"]) != expected: raise SystemExit(1) env = portal / "backend" / ".env.production" try: env_digest = hashlib.sha256(env.read_bytes()).hexdigest() except OSError: raise SystemExit(1) if env_digest != marker.get("environmentSha256"): raise SystemExit(1) PY } detect_retained_install_reconnect() { if [[ ! -e "${RETAINED_INSTALL_MARKER}" && ! -L "${RETAINED_INSTALL_MARKER}" \ && ! -e "${RETAINED_INSTALL_MANIFEST}" && ! -L "${RETAINED_INSTALL_MANIFEST}" ]]; then return 0 fi verify_retained_install_receipt \ || fail "A retained Portal tree exists, but its root-only reconnect receipt or exact tree manifest no longer matches. Refusing to overlay it." RETAINED_RECONNECT_MODE=true FORCE_FRESH=true info "Verified the exact retained Portal data tree; reconnecting it to the signed runtime." } clear_retained_install_receipt() { local allow_runtime_overlay="${1:-false}" verify_retained_install_receipt \ "${PORTAL_DIR}" "${RETAINED_INSTALL_MARKER}" "${RETAINED_INSTALL_MANIFEST}" \ "${allow_runtime_overlay}" || return 1 rm -f -- "${RETAINED_INSTALL_MARKER}" "${RETAINED_INSTALL_MANIFEST}" || return 1 [[ ! -e "${RETAINED_INSTALL_MARKER}" && ! -L "${RETAINED_INSTALL_MARKER}" \ && ! -e "${RETAINED_INSTALL_MANIFEST}" && ! -L "${RETAINED_INSTALL_MANIFEST}" ]] } assert_fresh_install_target_available() { local portal_dir="${1:-${PORTAL_DIR}}" local retained_reconnect="${2:-${RETAINED_RECONNECT_MODE}}" if [[ ! -e "${portal_dir}" && ! -L "${portal_dir}" ]]; then return 0 fi # detect_retained_install_reconnect() is the only production path that sets # this flag, and it does so only after verifying the root-only receipt and # exact retained-tree manifest. That attested tree is the expected input to # build_portal(); rejecting it here would make Keep Data uninstall permanent. if [[ "${retained_reconnect}" == "true" ]]; then return 0 fi fail "A partial or unattested Portal path already exists at ${portal_dir}. Refusing to treat it as a fresh host; recover or remove that exact installation deliberately." } remove_portal_runtime_preserving_data() { local portal_dir="${1:-${PORTAL_DIR}}" local expected_portal_dir="${2:-${PORTAL_DIR}}" local portal_attestation="" portal_attestation="$(python3 - "${portal_dir}" "${expected_portal_dir}" <<'PY' import os import stat import sys path, expected = sys.argv[1:] for value in (path, expected): if not os.path.isabs(value) or value != os.path.normpath(value) or value == os.path.sep: raise SystemExit("Portal runtime path must be a bounded canonical absolute path") if path != expected: raise SystemExit("Portal runtime path does not match the fixed installation boundary") current = os.path.sep parts = path.strip(os.path.sep).split(os.path.sep) for index, component in enumerate(parts): current = os.path.join(current, component) if not os.path.lexists(current): print("absent") raise SystemExit(0) info = os.lstat(current) if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) or info.st_uid != 0: raise SystemExit("Portal runtime path crosses an unsafe ownership boundary") if info.st_mode & 0o022: if index == len(parts) - 1 or not info.st_mode & stat.S_ISVTX: raise SystemExit("Portal runtime path crosses a writable ownership boundary") if os.path.realpath(path) != path: raise SystemExit("Portal runtime path does not resolve to its fixed boundary") info = os.lstat(path) print(f"{info.st_dev}:{info.st_ino}") PY )" || return 1 [[ "${portal_attestation}" != "absent" ]] || return 0 # Pin the attested directory while preparing rsync and prove the shell did # not reopen a different inode. Root-owned non-writable parents prevent an # unprivileged rename after this check. local portal_fd actual_portal_attestation exec {portal_fd}< "${portal_dir}" || return 1 actual_portal_attestation="$(stat -Lc '%d:%i' "/proc/$$/fd/${portal_fd}" 2>/dev/null)" \ || { exec {portal_fd}<&-; return 1; } [[ "${actual_portal_attestation}" == "${portal_attestation}" ]] \ || { exec {portal_fd}<&-; return 1; } # Empty-source rsync deletes only runtime payload. These exclusions are the # same persistent boundaries used by update/reinstall, plus the retained # environment needed to reconnect a custom database on the next install. local empty_source empty_source="$(mktemp -d)" mkdir -p "${empty_source}/backend" "${empty_source}/frontend" if ! rsync -a --delete \ --exclude='/projects' \ --exclude='/apps' \ --exclude='/assets' \ --exclude='/upload-temp' \ --exclude='/.data' \ --exclude='/backend/.data' \ --exclude='/backend/.env.production' \ --exclude='/backend/.env' \ --exclude='/frontend/.env' \ "${empty_source}/" "${portal_dir}/"; then rm -rf -- "${empty_source}" exec {portal_fd}<&- return 1 fi rmdir -- "${empty_source}/backend" "${empty_source}/frontend" "${empty_source}" \ || { exec {portal_fd}<&-; return 1; } exec {portal_fd}<&- } remove_portal_backup_automation() { # Backup automation is Portal runtime, not user data. Leaving these units # enabled after either uninstall mode creates dangling timers whose ExecStart # points into the removed Portal tree. Stop known instances before removing # the exact installer-owned unit files; retained archives and backup settings # are deliberately untouched and will be reused on a later install. local timer_units=( bridgesllm-backup-daily.timer bridgesllm-backup-comprehensive.timer bridgesllm-backup-monthly.timer ) local service_units=( bridgesllm-backup@daily.service bridgesllm-backup@comprehensive.service bridgesllm-backup@monthly.service ) systemctl disable --now "${timer_units[@]}" >/dev/null 2>&1 || true systemctl stop "${service_units[@]}" >/dev/null 2>&1 || true # A failed stop cannot be treated as cosmetic before Clean slate removes the # database and filesystem. Read back every known unit and refuse to continue # while a timer or backup process is active, activating, or deactivating. local unit active_state for unit in "${timer_units[@]}" "${service_units[@]}"; do active_state="$(systemctl show --property=ActiveState --value "${unit}" 2>/dev/null)" \ || fail "Portal backup unit ${unit} state could not be verified; uninstall was aborted before data removal." case "${active_state}" in inactive|failed) ;; *) fail "Portal backup unit ${unit} did not stop cleanly; uninstall was aborted before data removal." ;; esac done rm -f -- \ /etc/systemd/system/bridgesllm-backup@.service \ /etc/systemd/system/bridgesllm-backup-daily.timer \ /etc/systemd/system/bridgesllm-backup-comprehensive.timer \ /etc/systemd/system/bridgesllm-backup-monthly.timer \ /etc/systemd/system/timers.target.wants/bridgesllm-backup-daily.timer \ /etc/systemd/system/timers.target.wants/bridgesllm-backup-comprehensive.timer \ /etc/systemd/system/timers.target.wants/bridgesllm-backup-monthly.timer systemctl daemon-reload >/dev/null 2>&1 \ || fail "Systemd could not forget removed Portal backup automation; uninstall was aborted." systemctl reset-failed "${service_units[@]}" >/dev/null 2>&1 || true } portal_remote_desktop_account_matches_contract() { local rd_user="bridgesrd" local rd_home="/home/bridgesrd" local rd_marker="${rd_home}/.bridgesllm-portal-managed" local passwd_entry="" passwd_entry="$(getent passwd "${rd_user}" 2>/dev/null || true)" [[ -n "${passwd_entry}" ]] || return 1 local account_name _password account_uid account_gid _gecos account_home account_shell IFS=: read -r account_name _password account_uid account_gid _gecos account_home account_shell <<< "${passwd_entry}" [[ "${account_name}" == "${rd_user}" \ && "${account_uid}" =~ ^[0-9]+$ \ && "${account_gid}" =~ ^[0-9]+$ \ && ${account_uid} -lt 1000 \ && "${account_home}" == "${rd_home}" \ && "${account_shell}" == "/bin/bash" \ && -d "${rd_home}" \ && ! -L "${rd_home}" ]] || return 1 [[ "$(stat -c '%u:%g' "${rd_home}" 2>/dev/null || true)" == "${account_uid}:${account_gid}" ]] \ || return 1 if [[ -f "${rd_marker}" && ! -L "${rd_marker}" \ && "$(stat -c '%u:%g:%a' "${rd_marker}" 2>/dev/null || true)" == '0:0:600' \ && "$(cat -- "${rd_marker}" 2>/dev/null || true)" == 'managed-by=bridgesllm-portal-remote-desktop-v1' ]]; then return 0 fi # Releases predating the ownership marker are still safely attributable when # both exact Portal-authored units remain. Refuse account deletion on partial, # linked, or administrator-shaped state. local vnc_unit="/etc/systemd/system/bridges-rd-xtigervnc.service" local websockify_unit="/etc/systemd/system/bridges-rd-websockify.service" [[ -f "${vnc_unit}" && ! -L "${vnc_unit}" \ && -f "${websockify_unit}" && ! -L "${websockify_unit}" ]] || return 1 grep -Fx 'Description=Bridges Remote Desktop Xtigervnc :1' "${vnc_unit}" >/dev/null 2>&1 \ && grep -Fx 'ExecStart=/usr/local/bin/bridges-rd-xtigervnc-start.sh' "${vnc_unit}" >/dev/null 2>&1 \ && grep -Fx 'Description=Bridges Remote Desktop noVNC Websockify' "${websockify_unit}" >/dev/null 2>&1 \ && grep -Fx 'User=bridgesrd' "${websockify_unit}" >/dev/null 2>&1 } remove_portal_remote_desktop_launcher_lock() { local lock_path="${1:-/var/lib/bridgesllm/remote-desktop-ai-launchers.lock}" local lock_fd lock_fd_identity lock_path_identity if [[ ! -e "${lock_path}" && ! -L "${lock_path}" ]]; then return 0 fi [[ -f "${lock_path}" && ! -L "${lock_path}" ]] \ || fail "Managed AI launcher lifecycle lock is linked or non-regular; Remote Desktop uninstall was aborted." [[ "$(stat -c '%u:%g:%a' "${lock_path}")" == '0:0:600' ]] \ || fail "Managed AI launcher lifecycle lock is not root-owned mode 0600; Remote Desktop uninstall was aborted." exec {lock_fd}<>"${lock_path}" \ || fail "Managed AI launcher lifecycle lock could not be opened safely; Remote Desktop uninstall was aborted." flock -x "${lock_fd}" \ || fail "Managed AI launcher lifecycle lock could not be acquired; Remote Desktop uninstall was aborted." [[ -f "${lock_path}" && ! -L "${lock_path}" && "$(stat -c '%u:%g:%a' "${lock_path}")" == '0:0:600' ]] \ || fail "Managed AI launcher lifecycle lock changed while uninstall waited for it." lock_fd_identity="$(stat -Lc '%d:%i' "/proc/$$/fd/${lock_fd}")" lock_path_identity="$(stat -Lc '%d:%i' "${lock_path}")" [[ "${lock_fd_identity}" == "${lock_path_identity}" ]] \ || fail "Managed AI launcher lifecycle lock identity changed while uninstall waited for it." rm -f -- "${lock_path}" [[ ! -e "${lock_path}" && ! -L "${lock_path}" ]] \ || fail "Managed AI launcher lifecycle lock survived Remote Desktop uninstall." exec {lock_fd}>&- } remove_portal_remote_desktop_open_handoffs() { local production_root='/var/lib/bridgesllm/remote-desktop-open' local handoff_root group_entry group_name group_password desktop_gid group_members handoff_root="$(update_transaction_state_path "${production_root}")" \ || fail "Remote Desktop file handoff path could not be bounded safely; uninstall was aborted." if [[ ! -e "${handoff_root}" && ! -L "${handoff_root}" ]]; then return 0 fi group_entry="$(getent group bridgesrd 2>/dev/null)" \ || fail "Remote Desktop group identity could not be resolved while file handoffs remain; uninstall was aborted." [[ -n "${group_entry}" && "${group_entry}" != *$'\n'* ]] \ || fail "Remote Desktop group identity is ambiguous while file handoffs remain; uninstall was aborted." IFS=: read -r group_name group_password desktop_gid group_members <<< "${group_entry}" [[ "${group_name}" == 'bridgesrd' && "${desktop_gid}" =~ ^[1-9][0-9]*$ ]] \ || fail "Remote Desktop group identity is invalid while file handoffs remain; uninstall was aborted." python3 - "${handoff_root}" "${desktop_gid}" <<'PY' \ || fail "Remote Desktop file handoffs failed exact lifecycle attestation; uninstall was aborted without deleting an unvalidated target." import os import re import stat import sys root_path, desktop_gid_text = sys.argv[1:] desktop_gid = int(desktop_gid_text) request_pattern = re.compile(r"[a-f0-9]{32}\Z") directory_flags = ( os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) ) file_flags = ( os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) ) if ( os.geteuid() != 0 or desktop_gid <= 0 or not os.path.isabs(root_path) or os.path.normpath(root_path) != root_path or os.path.basename(root_path) != "remote-desktop-open" ): raise SystemExit(1) def inode_identity(details): return ( details.st_dev, details.st_ino, details.st_mode, details.st_nlink, details.st_uid, details.st_gid, ) def stable_directory_identity(details): return ( details.st_dev, details.st_ino, stat.S_IFMT(details.st_mode), stat.S_IMODE(details.st_mode), details.st_uid, details.st_gid, ) def mount_id(descriptor): with open( f"/proc/self/fdinfo/{descriptor}", "r", encoding="ascii", errors="strict", ) as handle: for line in handle: if line.startswith("mnt_id:\t"): value = line.split("\t", 1)[1].strip() if value.isdigit(): return int(value) raise SystemExit(1) def stat_entry(name, directory_fd): return os.stat(name, dir_fd=directory_fd, follow_symlinks=False) def assert_directory(details, uid, gid, mode, device): if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != uid or details.st_gid != gid or stat.S_IMODE(details.st_mode) != mode or details.st_dev != device ): raise SystemExit(1) def assert_file(details, gids, mode, device): if ( not stat.S_ISREG(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != 0 or details.st_gid not in gids or stat.S_IMODE(details.st_mode) != mode or details.st_nlink != 1 or details.st_dev != device ): raise SystemExit(1) # Every ancestor is root-owned and non-writable before the final parent is # pinned. That makes the descriptor-relative identity proofs below stable # against unprivileged rename/rebind attempts. parent_path = os.path.dirname(root_path) current = os.path.sep for component in parent_path.strip(os.path.sep).split(os.path.sep): if not component: continue current = os.path.join(current, component) details = os.lstat(current) if ( not stat.S_ISDIR(details.st_mode) or stat.S_ISLNK(details.st_mode) or details.st_uid != 0 or details.st_mode & 0o022 ): raise SystemExit(1) parent_fd = os.open(parent_path, directory_flags) root_fd = -1 directory_records = [] try: root_name = os.path.basename(root_path) try: root_entry = stat_entry(root_name, parent_fd) except FileNotFoundError: raise SystemExit(0) root_fd = os.open(root_name, directory_flags, dir_fd=parent_fd) root_opened = os.fstat(root_fd) if inode_identity(root_opened) != inode_identity(root_entry): raise SystemExit(1) assert_directory(root_opened, 0, desktop_gid, 0o710, root_opened.st_dev) root_device = root_opened.st_dev if root_device != os.fstat(parent_fd).st_dev or mount_id(root_fd) != mount_id(parent_fd): raise SystemExit(1) request_names = sorted(os.listdir(root_fd)) if len(request_names) > 32 or any(not request_pattern.fullmatch(name) for name in request_names): raise SystemExit(1) # Pin and prove the complete tree before deleting any member. A malformed # sibling therefore cannot turn a partial cleanup into guessed ownership. for request_name in request_names: request_entry = stat_entry(request_name, root_fd) request_fd = os.open(request_name, directory_flags, dir_fd=root_fd) request_opened = os.fstat(request_fd) try: if inode_identity(request_opened) != inode_identity(request_entry): raise SystemExit(1) assert_directory(request_opened, 0, desktop_gid, 0o750, root_device) if mount_id(request_fd) != mount_id(root_fd): raise SystemExit(1) file_names = sorted(os.listdir(request_fd)) if len(file_names) > 2: raise SystemExit(1) snapshot_names = [name for name in file_names if name != ".reservation"] if len(snapshot_names) > 1: raise SystemExit(1) file_records = [] reservation_bytes = None snapshot_details = None for file_name in file_names: is_reservation = file_name == ".reservation" expected_gids = ( {0} if is_reservation else ({0, desktop_gid} if ".reservation" in file_names else {desktop_gid}) ) expected_mode = 0o400 if file_name == ".reservation" else 0o440 file_entry = stat_entry(file_name, request_fd) file_fd = os.open(file_name, file_flags, dir_fd=request_fd) file_opened = os.fstat(file_fd) try: if inode_identity(file_opened) != inode_identity(file_entry): raise SystemExit(1) assert_file(file_opened, expected_gids, expected_mode, root_device) if mount_id(file_fd) != mount_id(root_fd): raise SystemExit(1) if is_reservation: payload = os.read(file_fd, 16) if os.read(file_fd, 1): raise SystemExit(1) try: reservation_text = payload.decode("ascii") except UnicodeDecodeError: raise SystemExit(1) if not re.fullmatch(r"0|[1-9][0-9]*", reservation_text): raise SystemExit(1) reservation_bytes = int(reservation_text) if reservation_bytes > 256 * 1024 * 1024: raise SystemExit(1) else: snapshot_details = file_opened if snapshot_details.st_size > 256 * 1024 * 1024: raise SystemExit(1) file_records.append((file_name, file_fd, inode_identity(file_opened))) file_fd = -1 finally: if file_fd >= 0: os.close(file_fd) if snapshot_details is not None and reservation_bytes is not None: if ( snapshot_details.st_gid == 0 and snapshot_details.st_size > reservation_bytes ) or ( snapshot_details.st_gid == desktop_gid and snapshot_details.st_size != reservation_bytes ): raise SystemExit(1) directory_records.append(( request_name, request_fd, stable_directory_identity(request_opened), file_records, )) request_fd = -1 finally: if request_fd >= 0: os.close(request_fd) # Revalidate the full pinned tree once more before the first unlink. The # per-entry check immediately before each mutation then catches a late # replacement without ever following it. if inode_identity(stat_entry(root_name, parent_fd)) != inode_identity(root_opened): raise SystemExit(1) for request_name, request_fd, request_identity, file_records in directory_records: if stable_directory_identity(stat_entry(request_name, root_fd)) != request_identity: raise SystemExit(1) if stable_directory_identity(os.fstat(request_fd)) != request_identity: raise SystemExit(1) for file_name, file_fd, file_identity in file_records: if inode_identity(stat_entry(file_name, request_fd)) != file_identity: raise SystemExit(1) if inode_identity(os.fstat(file_fd)) != file_identity: raise SystemExit(1) for request_name, request_fd, request_identity, file_records in directory_records: for file_name, file_fd, file_identity in file_records: if ( inode_identity(stat_entry(file_name, request_fd)) != file_identity or inode_identity(os.fstat(file_fd)) != file_identity ): raise SystemExit(1) os.unlink(file_name, dir_fd=request_fd) if os.fstat(file_fd).st_nlink != 0: raise SystemExit(1) os.close(file_fd) os.fsync(request_fd) if os.listdir(request_fd): raise SystemExit(1) if ( stable_directory_identity(stat_entry(request_name, root_fd)) != request_identity or stable_directory_identity(os.fstat(request_fd)) != request_identity ): raise SystemExit(1) os.rmdir(request_name, dir_fd=root_fd) os.close(request_fd) os.fsync(root_fd) if os.listdir(root_fd): raise SystemExit(1) if ( stable_directory_identity(stat_entry(root_name, parent_fd)) != stable_directory_identity(root_opened) or stable_directory_identity(os.fstat(root_fd)) != stable_directory_identity(root_opened) ): raise SystemExit(1) os.rmdir(root_name, dir_fd=parent_fd) os.fsync(parent_fd) try: stat_entry(root_name, parent_fd) except FileNotFoundError: pass else: raise SystemExit(1) finally: for _request_name, request_fd, _request_identity, file_records in directory_records: for _file_name, file_fd, _file_identity in file_records: try: os.close(file_fd) except OSError: pass try: os.close(request_fd) except OSError: pass if root_fd >= 0: os.close(root_fd) os.close(parent_fd) PY } portal_remote_desktop_watchdog_runtime_absent() { [[ ! -e /run/bridges-rd && ! -L /run/bridges-rd ]] } remove_portal_remote_desktop_runtime() { local data_choice="${1:-1}" [[ "${data_choice}" == "1" || "${data_choice}" == "2" ]] \ || fail "Invalid Remote Desktop uninstall data choice; expected 1 or 2." local rd_user="bridgesrd" local rd_account_is_managed=false # Account ownership governs the process boundary in both uninstall modes. # Keep Data preserves the profile, but detached XFCE/browser processes owned # by an attested Portal account must not outlive the removed services. A # foreign or drifted bridgesrd identity remains completely untouched. if portal_remote_desktop_account_matches_contract; then rd_account_is_managed=true fi # Stop the out-of-process watchdog first so it cannot race teardown by # restarting a Remote Desktop unit while uninstall is removing its files. local rd_health_units=( bridges-rd-healthcheck.timer bridges-rd-healthcheck.service ) systemctl disable --now bridges-rd-healthcheck.timer >/dev/null 2>&1 || true systemctl stop "${rd_health_units[@]}" >/dev/null 2>&1 || true systemctl kill --kill-who=all bridges-rd-healthcheck.service >/dev/null 2>&1 || true local rd_unit rd_unit_state rd_unit_load_state for rd_unit in "${rd_health_units[@]}"; do rd_unit_load_state="$(systemctl show --property=LoadState --value "${rd_unit}" 2>/dev/null)" \ || fail "Remote Desktop watchdog unit ${rd_unit} load state could not be verified; uninstall was aborted before files or accounts were removed." rd_unit_state="$(systemctl show --property=ActiveState --value "${rd_unit}" 2>/dev/null)" \ || fail "Remote Desktop watchdog unit ${rd_unit} active state could not be verified; uninstall was aborted before files or accounts were removed." case "${rd_unit_state}" in inactive|failed) ;; *) fail "Remote Desktop watchdog unit ${rd_unit} is still ${rd_unit_state:-unknown} (${rd_unit_load_state:-unknown}); uninstall was aborted before files or accounts were removed." ;; esac done local rd_units=( bridges-rd-xtigervnc.service bridges-rd-websockify.service bridges-rd-vnc.service ) systemctl disable --now "${rd_units[@]}" >/dev/null 2>&1 || true systemctl stop "${rd_units[@]}" >/dev/null 2>&1 || true systemctl kill --kill-who=all "${rd_units[@]}" >/dev/null 2>&1 || true for rd_unit in "${rd_units[@]}"; do rd_unit_load_state="$(systemctl show --property=LoadState --value "${rd_unit}" 2>/dev/null)" \ || fail "Remote Desktop unit ${rd_unit} load state could not be verified; uninstall was aborted before files or accounts were removed." rd_unit_state="$(systemctl show --property=ActiveState --value "${rd_unit}" 2>/dev/null)" \ || fail "Remote Desktop unit ${rd_unit} active state could not be verified; uninstall was aborted before files or accounts were removed." case "${rd_unit_state}" in inactive|failed) ;; *) fail "Remote Desktop unit ${rd_unit} is still ${rd_unit_state:-unknown} (${rd_unit_load_state:-unknown}); uninstall was aborted before files or accounts were removed." ;; esac done if pgrep -f '/usr/local/bin/bridges-rd-healthcheck\.sh|/usr/local/bin/bridges-rd-xtigervnc-start\.sh|/usr/local/bin/bridges-rd-session-guard\.sh|/usr/bin/Xtigervnc[[:space:]]+:1([[:space:]]|$)|/usr/bin/websockify([[:space:]].*)?127\.0\.0\.1:6080([[:space:]]|$)' >/dev/null 2>&1; then fail "Remote Desktop host processes are still running; uninstall was aborted before files or accounts were removed." fi # A uid-wide kill crosses the service boundary and is safe only after the # account itself passed the Portal ownership contract. Exact managed service # processes were already stopped and attested above. Preserve every process # and home file belonging to an unattested/foreign bridgesrd account. if [[ "${rd_account_is_managed}" == "true" ]] && id "${rd_user}" >/dev/null 2>&1; then local rd_process_status=0 if pkill -TERM -u "${rd_user}" >/dev/null 2>&1; then : else rd_process_status=$? [[ ${rd_process_status} -eq 1 ]] \ || fail "Remote Desktop user processes could not be signaled safely; uninstall was aborted before files or accounts were removed." fi local wait_attempt rd_processes_remain=false for wait_attempt in 1 2 3 4 5; do if pgrep -u "${rd_user}" >/dev/null 2>&1; then rd_processes_remain=true else rd_process_status=$? [[ ${rd_process_status} -eq 1 ]] \ || fail "Remote Desktop user process state could not be verified; uninstall was aborted before files or accounts were removed." rd_processes_remain=false break fi sleep 0.2 done if [[ "${rd_processes_remain}" == "true" ]]; then if pkill -KILL -u "${rd_user}" >/dev/null 2>&1; then : else rd_process_status=$? [[ ${rd_process_status} -eq 1 ]] \ || fail "Remote Desktop user processes could not be force-stopped safely; uninstall was aborted before files or accounts were removed." fi fi if pgrep -u "${rd_user}" >/dev/null 2>&1; then fail "Remote Desktop user processes are still running; uninstall was aborted before files or accounts were removed." else rd_process_status=$? [[ ${rd_process_status} -eq 1 ]] \ || fail "Remote Desktop user process convergence could not be verified; uninstall was aborted before files or accounts were removed." fi fi # Portal was fenced and stopped before this lifecycle phase. The exact RD # units and, for an attested account, every bridgesrd process are now also # proven stopped, so no product writer can race removal of ephemeral file # handoffs. Both uninstall modes remove these runtime-only snapshots. remove_portal_remote_desktop_open_handoffs local ai_provider_launcher='/usr/local/bin/bridges-rd-ai-launchers.sh' if [[ "${rd_account_is_managed}" == "true" ]]; then if [[ -f "${ai_provider_launcher}" && ! -L "${ai_provider_launcher}" ]]; then local ai_remove_args=(remove) [[ "${data_choice}" == '2' ]] && ai_remove_args+=(--purge-profiles) "${ai_provider_launcher}" "${ai_remove_args[@]}" >/dev/null \ || fail "Managed AI provider Remote Desktop launchers could not be removed safely." fi remove_portal_remote_desktop_launcher_lock fi rm -f -- \ /etc/systemd/system/bridges-rd-healthcheck.service \ /etc/systemd/system/bridges-rd-healthcheck.timer \ /etc/systemd/system/bridges-rd-xtigervnc.service \ /etc/systemd/system/bridges-rd-websockify.service \ /etc/systemd/system/bridges-rd-vnc.service \ /etc/systemd/system/timers.target.wants/bridges-rd-healthcheck.timer \ /etc/systemd/system/multi-user.target.wants/bridges-rd-xtigervnc.service \ /etc/systemd/system/multi-user.target.wants/bridges-rd-websockify.service \ /etc/systemd/system/multi-user.target.wants/bridges-rd-vnc.service \ /usr/local/bin/bridges-rd-web-open.sh \ /usr/local/bin/bridges-rd-shared-chrome.sh \ /usr/local/bin/bridges-rd-openclaw-ui.sh \ /usr/local/bin/bridges-rd-ai-launchers.sh \ /usr/local/bin/bridges-rd-websockify-launcher.sh \ /usr/local/bin/bridges-rd-healthcheck.sh \ /usr/local/bin/bridges-rd-xtigervnc-start.sh \ /usr/local/bin/bridges-rd-session-guard.sh \ /usr/local/bin/bridges-rd-window-fit.sh \ /usr/local/share/pixmaps/bridges-shared-browser.svg \ /usr/local/share/pixmaps/bridges-openclaw-ui.svg if [[ "${data_choice}" == "2" && "${rd_account_is_managed}" == "true" ]]; then rm -f -- \ "/home/${rd_user}/Desktop/Shared Chrome.desktop" \ "/home/${rd_user}/Desktop/OpenClaw Web UI.desktop" \ "/home/${rd_user}/.bridges-rd-env" \ "/home/${rd_user}/.Xauthority" fi rm -rf -- /tmp/bridges-rd-runtime /run/bridges-rd rm -f -- /run/lock/bridges-rd-healthcheck.lock portal_remote_desktop_watchdog_runtime_absent \ || fail "Remote Desktop watchdog runtime state could not be removed safely." systemctl daemon-reload >/dev/null 2>&1 \ || fail "Systemd could not forget removed Remote Desktop units; uninstall was aborted." for rd_unit in "${rd_health_units[@]}"; do rd_unit_load_state="$(systemctl show --property=LoadState --value "${rd_unit}" 2>/dev/null)" \ || fail "Removed Remote Desktop watchdog unit ${rd_unit} could not be re-inspected." [[ "${rd_unit_load_state}" == "not-found" ]] \ || fail "Remote Desktop watchdog unit ${rd_unit} still has load state ${rd_unit_load_state:-unknown} after removal." done systemctl reset-failed "${rd_health_units[@]}" "${rd_units[@]}" >/dev/null 2>&1 || true systemctl unmask \ tigervncserver@:1.service tigervncserver@1.service \ vncserver@:1.service vncserver@1.service >/dev/null 2>&1 || true if [[ "${data_choice}" == "2" ]]; then if [[ "${rd_account_is_managed}" == "true" ]]; then rm -rf -- /var/log/bridges-rd userdel --remove "${rd_user}" >/dev/null 2>&1 \ || fail "Portal-owned Remote Desktop account could not be removed safely." if getent group "${rd_user}" >/dev/null 2>&1; then groupdel "${rd_user}" >/dev/null 2>&1 || true fi ok "Portal-owned Remote Desktop account and profile removed" elif id "${rd_user}" >/dev/null 2>&1; then warn "The bridgesrd account did not match Portal ownership safeguards; its home was preserved." fi fi } assert_no_managed_project_runtime_residuals() { local query kind selector residuals name local queries=( 'container|com.bridgesllm.project-egress.runtime-fingerprint' 'container|com.bridgesllm.project-egress.policy' 'container|com.bridgesllm.project-workload.policy' 'container|com.bridgesllm.ollama-project.policy' 'container|com.bridgesllm.project-runtime=true' 'container|com.bridgesllm.project-git=true' 'container|io.bridgesllm.managed=agent-zero-project' 'volume|io.bridgesllm.managed=agent-zero-project' 'network|io.bridgesllm.managed=agent-zero-project' 'network|com.bridgesllm.project-egress.policy' ) for query in "${queries[@]}"; do IFS='|' read -r kind selector <<<"${query}" case "${kind}" in container) residuals="$(docker container ls --all --quiet --filter "label=${selector}" 2>/dev/null)" \ || fail "Docker project-container state could not be authoritatively inspected." ;; volume) residuals="$(docker volume ls --quiet --filter "label=${selector}" 2>/dev/null)" \ || fail "Docker project-volume state could not be authoritatively inspected." ;; network) residuals="$(docker network ls --quiet --filter "label=${selector}" 2>/dev/null)" \ || fail "Docker project-network state could not be authoritatively inspected." ;; *) fail "Internal project-runtime residual query was invalid." ;; esac [[ -z "${residuals//[[:space:]]/}" ]] \ || fail "Managed project resources remain but their cleanup helper is unavailable; clean-slate uninstall was aborted." done # Labels are ownership evidence, but cannot be the only discovery key: a # crashed or partially migrated runtime can retain an exact reserved Portal # name after its labels or database identity disappear. Enumerate every # Docker namespace read-only and fail closed on only the exact deterministic # product shapes. The signed preflight performs attested cleanup on the first # run; a helperless repeat never guesses that a name-shaped resource is safe # to delete. residuals="$(docker container ls --all --format '{{.Names}}' 2>/dev/null)" \ || fail "Docker container names could not be authoritatively inspected for project residue." while IFS= read -r name; do [[ -z "${name}" ]] && continue if [[ "${name}" =~ ^p4e-proxy-[a-f0-9]{20}$ \ || "${name}" =~ ^p4ol-[a-f0-9]{24}$ \ || "${name}" =~ ^p4cx-[a-f0-9]{24}$ \ || "${name}" =~ ^p4cc-[a-f0-9]{24}$ \ || "${name}" =~ ^p4ag-[a-f0-9]{24}$ \ || "${name}" =~ ^p4oc-[a-f0-9]{16}-[a-z0-9._-]{1,32}-[a-f0-9]{8}$ \ || "${name}" =~ ^bridgesllm-a0p-[a-f0-9]{24}$ \ || "${name}" =~ ^bridgesllm-project-(app|job|git)-[a-f0-9]{20}$ ]]; then fail "Reserved Portal project container residue remains but its cleanup helper is unavailable." fi done <<<"${residuals}" residuals="$(docker network ls --format '{{.Name}}' 2>/dev/null)" \ || fail "Docker network names could not be authoritatively inspected for project residue." while IFS= read -r name; do [[ -z "${name}" ]] && continue if [[ "${name}" =~ ^p4e-(in|out)-[a-f0-9]{20}$ ]]; then fail "Reserved Portal project network residue remains but its cleanup helper is unavailable." fi done <<<"${residuals}" residuals="$(docker volume ls --format '{{.Name}}' 2>/dev/null)" \ || fail "Docker volume names could not be authoritatively inspected for project residue." while IFS= read -r name; do [[ -z "${name}" ]] && continue if [[ "${name}" =~ ^bridgesllm-a0p-[a-f0-9]{24}-usr$ ]]; then fail "Reserved Portal project volume residue remains but its cleanup helper is unavailable." fi done <<<"${residuals}" # A repeat Clean slate can continue without the deleted helper only after a # global host-firewall scan proves that neither exact Portal signatures nor # ambiguous Portal-shaped chains remain. The first run removes these through # the signed TypeScript preflight; this is deliberately verification-only. local firewall_tool firewall_rules for firewall_tool in iptables ip6tables; do command -v "${firewall_tool}" >/dev/null 2>&1 \ || fail "${firewall_tool} is unavailable; project firewall cleanup could not be verified." firewall_rules="$(${firewall_tool} -w -S 2>/dev/null)" \ || fail "${firewall_tool} project firewall state could not be authoritatively inspected." if grep -E -- '(^-N (P4E-|A0P-)[A-Za-z0-9_.:-]+$)|( --comment "?(p4e-v1|a0p-v3):[a-f0-9]{64}:[a-f0-9]{64}"?( |$))|( -[jg] (P4E-|A0P-)[A-Za-z0-9_.:-]+( |$))' \ <<<"${firewall_rules}" >/dev/null; then fail "Managed or ambiguous project firewall residue remains; clean-slate uninstall was aborted." fi done } quiesce_managed_project_runtime_containers() { command -v docker >/dev/null 2>&1 || fail "Docker is unavailable; retained Project runtimes could not be stopped safely." python3 - <<'PY' import json import re import subprocess DOCKER = "/usr/bin/docker" selectors = { "com.bridgesllm.project-egress.policy": {"portal-project-egress-v1"}, "com.bridgesllm.project-workload.policy": {"portal-project-workload-v1"}, "com.bridgesllm.ollama-project.policy": {"portal-ollama-project-sandbox-v1"}, "com.bridgesllm.project-runtime": {"true"}, "com.bridgesllm.project-git": {"true"}, "io.bridgesllm.managed": {"agent-zero-project"}, } def run(args, *, allow=(0,)): result = subprocess.run( [DOCKER, *args], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8"}, ) if result.returncode not in allow: raise RuntimeError("Docker command failed") return result.stdout identifiers = [line.strip() for line in run( ["container", "ls", "--all", "--no-trunc", "--format", "{{.ID}}"], ).splitlines() if line.strip()] managed = [] for identifier in identifiers: if not re.fullmatch(r"[a-f0-9]{64}", identifier): raise RuntimeError("Docker returned an invalid container identity") raw = run(["container", "inspect", identifier]) payload = json.loads(raw) if not isinstance(payload, list) or len(payload) != 1 or not isinstance(payload[0], dict): raise RuntimeError("Docker inspection was ambiguous") container = payload[0] labels = ((container.get("Config") or {}).get("Labels") or {}) if not isinstance(labels, dict): raise RuntimeError("Docker labels were invalid") claims = [(key, labels.get(key)) for key in selectors if key in labels] runtime_fingerprint = labels.get("com.bridgesllm.project-egress.runtime-fingerprint") if runtime_fingerprint is not None and ( not isinstance(runtime_fingerprint, str) or not re.fullmatch(r"[a-f0-9]{64}", runtime_fingerprint) ): raise RuntimeError("Project runtime fingerprint is invalid") if not claims and runtime_fingerprint is None: continue if claims and not any(value in selectors[key] for key, value in claims): raise RuntimeError("Portal-shaped Project labels contradict the managed runtime contract") # Every recognized ownership label present must carry its exact product # value. This prevents a foreign/drifted container from being stopped just # because it copied one reserved key. if any(value not in selectors[key] for key, value in claims): raise RuntimeError("Project runtime ownership labels are inconsistent") name = str(container.get("Name") or "").lstrip("/") if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", name): raise RuntimeError("Managed Project container name is invalid") managed.append((identifier, name, bool((container.get("State") or {}).get("Running")))) for identifier, _name, running in managed: if running: run(["container", "stop", "--time", "30", identifier]) for identifier, _name, _running in managed: raw = run(["container", "inspect", identifier]) payload = json.loads(raw) if bool(((payload[0].get("State") or {}).get("Running"))): raise RuntimeError("Managed Project container did not remain stopped") print(f"Stopped {sum(1 for _identifier, _name, running in managed if running)} managed Project container(s); retained data and identities were preserved.") PY } quiesce_project_runtimes_preserving_data() { info "Stopping managed Project runtimes while retaining sources, sessions, and identities..." quiesce_managed_project_runtime_containers \ || fail "Managed Project runtimes could not be stopped without crossing their exact ownership boundary." local agent_zero_helper="${PORTAL_DIR}/installer/agent-zero-runtime.sh" if [[ -f "${agent_zero_helper}" && ! -L "${agent_zero_helper}" ]]; then bash "${agent_zero_helper}" quiesce \ || fail "Managed Agent Zero could not be stopped while preserving its data." elif [[ -e "${agent_zero_helper}" || -L "${agent_zero_helper}" ]]; then fail "Agent Zero lifecycle helper is unsafe; retained-data uninstall was aborted." fi } # Keep Data stops managed runtimes for a safe teardown; the pre-quiesce # running set is the user's runtime-running intent and must survive to the # reconnect. First writer wins: a resumed phase re-run observes containers the # earlier attempt already stopped and must not overwrite the original record. record_retained_runtime_intent() { local intent_file="${PORTAL_DIR}/backend/.data/retained-runtime-intent.json" local agent_zero_running="false" local agent_zero_helper="${PORTAL_DIR}/installer/agent-zero-runtime.sh" if [[ -f "${agent_zero_helper}" && ! -L "${agent_zero_helper}" ]]; then agent_zero_running="$(bash "${agent_zero_helper}" runtime-active)" \ || fail "Agent Zero running state could not be recorded before quiescence." [[ "${agent_zero_running}" == "true" || "${agent_zero_running}" == "false" ]] \ || fail "Agent Zero running state was unreadable before quiescence." fi python3 - "${intent_file}" "${agent_zero_running}" <<'PY' \ || fail "The retained runtime-running intent could not be recorded before quiescence." import datetime import json import os import re import stat import subprocess import sys target, agent_zero_running = sys.argv[1:] if os.geteuid() != 0 or agent_zero_running not in {"true", "false"}: raise SystemExit(1) try: existing = os.lstat(target) except FileNotFoundError: existing = None if existing is not None: if (not stat.S_ISREG(existing.st_mode) or stat.S_ISLNK(existing.st_mode) or existing.st_uid != 0 or existing.st_mode & 0o022): raise SystemExit("retained runtime intent already exists but is unsafe") raise SystemExit(0) directory = os.path.dirname(target) os.makedirs(directory, mode=0o700, exist_ok=True) info = os.lstat(directory) if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022): raise SystemExit("retained runtime intent directory is unsafe") DOCKER = "/usr/bin/docker" selectors = { "com.bridgesllm.project-egress.policy": {"portal-project-egress-v1"}, "com.bridgesllm.project-workload.policy": {"portal-project-workload-v1"}, "com.bridgesllm.ollama-project.policy": {"portal-ollama-project-sandbox-v1"}, "com.bridgesllm.project-runtime": {"true"}, "com.bridgesllm.project-git": {"true"}, "io.bridgesllm.managed": {"agent-zero-project"}, } def run(arguments): result = subprocess.run( [DOCKER, *arguments], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8"}, ) if result.returncode != 0: raise RuntimeError("Docker command failed") return result.stdout running = [] if os.path.exists(DOCKER): for identifier in run(["container", "ls", "--no-trunc", "--format", "{{.ID}}"]).splitlines(): identifier = identifier.strip() if not identifier: continue if not re.fullmatch(r"[a-f0-9]{64}", identifier): raise RuntimeError("Docker returned an invalid container identity") payload = json.loads(run(["container", "inspect", identifier])) if not isinstance(payload, list) or len(payload) != 1: raise RuntimeError("Docker inspection was ambiguous") container = payload[0] labels = ((container.get("Config") or {}).get("Labels") or {}) claims = [(key, labels.get(key)) for key in selectors if key in labels] runtime_fingerprint = labels.get("com.bridgesllm.project-egress.runtime-fingerprint") if runtime_fingerprint is not None and ( not isinstance(runtime_fingerprint, str) or not re.fullmatch(r"[a-f0-9]{64}", runtime_fingerprint) ): raise RuntimeError("Project runtime fingerprint is invalid") if not claims and runtime_fingerprint is None: continue if any(value not in selectors[key] for key, value in claims): raise RuntimeError("Project runtime labels contradict the managed contract") if bool((container.get("State") or {}).get("Running")): running.append(identifier) document = { "schema": "bridgesllm.retained-runtime-intent.v1", "recordedAt": datetime.datetime.now(datetime.timezone.utc) .replace(microsecond=0).isoformat().replace("+00:00", "Z"), "projectContainers": sorted(running), "agentZeroRunning": agent_zero_running == "true", } descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(descriptor, "w", encoding="utf-8") as handle: json.dump(document, handle, indent=2, sort_keys=True) handle.write("\n") handle.flush() os.fsync(handle.fileno()) directory_fd = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY } # Retire the reconnect intent as a durable directory-entry update. The file # contains no reusable credential, but resurrecting it after a crash could # restart an old runtime set during a later repair. durably_remove_retained_runtime_intent() { local intent_file="${PORTAL_DIR}/backend/.data/retained-runtime-intent.json" python3 - "${intent_file}" "${PORTAL_DIR}/backend/.data" <<'PY' import os import stat import sys path, expected_parent = sys.argv[1:] if ( not os.path.isabs(path) or os.path.normpath(path) != path or os.path.dirname(path) != expected_parent or os.path.basename(path) != "retained-runtime-intent.json" ): raise SystemExit(1) parent = os.lstat(expected_parent) if ( not stat.S_ISDIR(parent.st_mode) or stat.S_ISLNK(parent.st_mode) or parent.st_uid != 0 or parent.st_gid != 0 or parent.st_mode & 0o022 ): raise SystemExit(1) try: os.unlink(path) except FileNotFoundError: pass directory_fd = os.open(expected_parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) PY } # Reconnect restores the recorded intent explicitly but never fails the # install over it: a runtime that disappeared or whose ownership labels # drifted is reported and skipped, and the Portal's own boot attestation # remains the authority over what may keep running afterwards. restore_retained_runtime_intent() { local intent_file="${PORTAL_DIR}/backend/.data/retained-runtime-intent.json" [[ -e "${intent_file}" || -L "${intent_file}" ]] || return 0 if [[ ! -f "${intent_file}" || -L "${intent_file}" ]]; then warn "Retained runtime-running intent is not a regular file; no runtime was started from it." durably_remove_retained_runtime_intent \ || warn "The rejected retained runtime intent could not be removed durably." return 0 fi local summary if ! summary="$(python3 - "${intent_file}" <<'PY' import json import os import re import stat import subprocess import sys path = sys.argv[1] info = os.lstat(path) if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022 or info.st_size <= 0 or info.st_size > 65536): raise SystemExit("retained runtime intent is unsafe") document = json.load(open(path, "r", encoding="utf-8")) containers = document.get("projectContainers") agent_zero = document.get("agentZeroRunning") if (document.get("schema") != "bridgesllm.retained-runtime-intent.v1" or not isinstance(containers, list) or len(containers) > 256 or not all(isinstance(item, str) and re.fullmatch(r"[a-f0-9]{64}", item) for item in containers) or len(set(containers)) != len(containers) or not isinstance(agent_zero, bool)): raise SystemExit("retained runtime intent is invalid") DOCKER = "/usr/bin/docker" selectors = { "com.bridgesllm.project-egress.policy": {"portal-project-egress-v1"}, "com.bridgesllm.project-workload.policy": {"portal-project-workload-v1"}, "com.bridgesllm.ollama-project.policy": {"portal-ollama-project-sandbox-v1"}, "com.bridgesllm.project-runtime": {"true"}, "com.bridgesllm.project-git": {"true"}, "io.bridgesllm.managed": {"agent-zero-project"}, } def run(arguments, tolerate=False): result = subprocess.run( [DOCKER, *arguments], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8"}, ) if result.returncode != 0 and not tolerate: raise RuntimeError("Docker command failed") return result.returncode, result.stdout started = already = missing = drifted = failed = 0 if containers and not os.path.exists(DOCKER): missing = len(containers) containers = [] for identifier in containers: status, raw = run(["container", "inspect", identifier], tolerate=True) if status != 0: missing += 1 continue payload = json.loads(raw) if not isinstance(payload, list) or len(payload) != 1: drifted += 1 continue container = payload[0] labels = ((container.get("Config") or {}).get("Labels") or {}) claims = [(key, labels.get(key)) for key in selectors if key in labels] runtime_fingerprint = labels.get("com.bridgesllm.project-egress.runtime-fingerprint") if runtime_fingerprint is not None and ( not isinstance(runtime_fingerprint, str) or not re.fullmatch(r"[a-f0-9]{64}", runtime_fingerprint) ): drifted += 1 continue if ( (not claims and runtime_fingerprint is None) or any(value not in selectors[key] for key, value in claims) ): drifted += 1 continue if bool((container.get("State") or {}).get("Running")): already += 1 continue status, _raw = run(["container", "start", identifier], tolerate=True) if status == 0: started += 1 else: failed += 1 print( f"started={started} already={already} missing={missing} " f"drifted={drifted} failed={failed} agentZero={'true' if agent_zero else 'false'}" ) PY )"; then warn "Retained runtime-running intent could not be applied; no runtime was started from it." durably_remove_retained_runtime_intent \ || warn "The rejected retained runtime intent could not be removed durably." return 0 fi info "Retained runtime intent: ${summary%% agentZero=*}" if [[ "${summary}" != *"missing=0"* || "${summary}" != *"drifted=0"* || "${summary}" != *"failed=0"* ]]; then warn "Some previously running Project runtimes were not restored (${summary%% agentZero=*}); they will be rebuilt on demand." fi if [[ "${summary}" == *"agentZero=true"* ]]; then local agent_zero_helper="${PORTAL_DIR}/installer/agent-zero-runtime.sh" if [[ -f "${agent_zero_helper}" && ! -L "${agent_zero_helper}" ]]; then bash "${agent_zero_helper}" resume \ || warn "The managed Agent Zero runtime did not resume; it can be reconciled from Settings." else warn "Agent Zero was running before uninstall but its lifecycle helper is unavailable; it was not restarted." fi fi durably_remove_retained_runtime_intent \ || warn "The applied retained runtime intent could not be retired durably; it will be rechecked on the next repair." } run_project_runtime_clean_slate_preflight() { local portal_dir="${1:-${PORTAL_DIR}}" local preflight="${portal_dir}/backend/dist/cli/projectRuntimeUninstallPreflight.js" local env_file="${portal_dir}/backend/.env.production" if [[ -L "${preflight}" || ( -e "${preflight}" && ! -f "${preflight}" ) \ || -L "${env_file}" || ( -e "${env_file}" && ! -f "${env_file}" ) ]]; then fail "Project-runtime cleanup helper or environment is unsafe; clean-slate uninstall was aborted." fi if [[ ! -f "${preflight}" || ! -f "${env_file}" ]]; then handle_helperless_clean_slate_residue project return 0 fi info "Removing managed project runtimes and provider resources..." /usr/bin/timeout --foreground --kill-after=15s 1800s \ /usr/bin/node "${preflight}" \ --env-file "${env_file}" \ --max-runtime-seconds 1740 \ || fail "Managed project resources could not be removed safely; clean-slate uninstall was aborted." } remove_managed_agent_zero_for_clean_uninstall() { local portal_dir="${1:-${PORTAL_DIR}}" local helper="${portal_dir}/installer/agent-zero-runtime.sh" if [[ -L "${helper}" || ( -e "${helper}" && ! -f "${helper}" ) ]]; then fail "Agent Zero cleanup helper is unsafe; clean-slate uninstall was aborted." fi if [[ ! -f "${helper}" ]]; then handle_helperless_clean_slate_residue agent-zero return 0 fi info "Removing managed Agent Zero runtime..." bash "${helper}" uninstall \ || fail "Managed Agent Zero runtime could not be removed safely; clean-slate uninstall was aborted." } assert_no_managed_agent_zero_runtime_residuals() { local names name volumes path names="$(docker container ls --all --format '{{.Names}}' 2>/dev/null)" \ || fail "Docker container state could not be authoritatively inspected for Agent Zero residue." while IFS= read -r name; do [[ -z "${name}" ]] && continue if [[ "${name}" == 'bridgesllm-agent-zero' \ || "${name}" == 'bridgesllm-agent-zero-rollback' \ || "${name}" =~ ^bridgesllm-agent-zero-pre-rollback-[0-9]{8}T[0-9]{6}Z$ ]]; then fail "Managed Agent Zero container residue remains but its cleanup helper is unavailable." fi done <<<"${names}" volumes="$(docker volume ls --format '{{.Name}}' 2>/dev/null)" \ || fail "Docker volume state could not be authoritatively inspected for Agent Zero residue." grep -Fx 'bridgesllm-agent-zero-usr' <<<"${volumes}" >/dev/null \ && fail "Managed Agent Zero volume residue remains but its cleanup helper is unavailable." for path in \ /etc/bridgesllm/agent-zero.env \ /var/lib/bridgesllm/agent-zero-runtime \ /var/backups/bridgesllm/agent-zero; do [[ ! -e "${path}" && ! -L "${path}" ]] \ || fail "Managed Agent Zero filesystem residue remains but its cleanup helper is unavailable." done } assert_no_agent_zero_project_bridge_residuals() { local bridge_service='bridgesllm-agent-zero-project-model-bridge.service' local load_state active_state process_status path load_state="$(systemctl show --property=LoadState --value "${bridge_service}" 2>/dev/null)" \ || fail "Agent Zero Project model bridge load state could not be verified." active_state="$(systemctl show --property=ActiveState --value "${bridge_service}" 2>/dev/null)" \ || fail "Agent Zero Project model bridge active state could not be verified." [[ "${load_state}" == 'not-found' && ( "${active_state}" == 'inactive' || "${active_state}" == 'failed' ) ]] \ || fail "Agent Zero Project model bridge residue remains but its lifecycle helper is unavailable." for path in \ /etc/systemd/system/bridgesllm-agent-zero-project-model-bridge.service \ /etc/bridgesllm/agent-zero-project-model-bridge.env \ /var/lib/bridgesllm/agent-zero-project-model-bridge; do [[ ! -e "${path}" && ! -L "${path}" ]] \ || fail "Agent Zero Project model bridge filesystem residue remains but its lifecycle helper is unavailable." done id -u bridgesllm-a0-bridge >/dev/null 2>&1 \ && fail "Agent Zero Project model bridge account residue remains but its lifecycle helper is unavailable." getent group bridgesllm-a0-bridge >/dev/null 2>&1 \ && fail "Agent Zero Project model bridge group residue remains but its lifecycle helper is unavailable." if pgrep -f -- '/opt/bridgesllm/portal/backend/dist/agents/providers/agentZero/AgentZeroProjectModelBridge\.js([[:space:]]|$)' >/dev/null 2>&1; then fail "Agent Zero Project model bridge process residue remains but its lifecycle helper is unavailable." else process_status=$? [[ ${process_status} -eq 1 ]] \ || fail "Agent Zero Project model bridge process state could not be verified." fi } # Detect or remove leftover managed runtime residue on the helperless repeat # clean-slate path. Every deletion is scoped to the exact deterministic product # shapes the fail-closed asserts recognize; nothing else on the host is # touched. "report" is read-only and prints `present= bytes=` # followed by one human line per finding. "wipe" saves a full iptables and # ip6tables backup before deleting anything, then removes only the matched # resources; the caller must re-run the fail-closed asserts as the # authoritative absence proof. managed_runtime_residue_tool() { local mode="$1" local scope="${2:-all}" local backup_dir="${3:-}" local plan_path="${4:-}" local expected_plan_digest="${5:-}" python3 - "${mode}" "${scope}" "${backup_dir}" \ "${plan_path}" "${expected_plan_digest}" <<'PY' import datetime import hashlib import json import os import re import shlex import shutil import stat import subprocess import sys import tempfile mode, scope, backup_dir, plan_path, expected_plan_digest = sys.argv[1:] if ( mode not in {"report", "seal-plan", "quiesce", "wipe", "prove-absent"} or scope not in {"all", "project", "agent-zero"} ): raise SystemExit("invalid managed-residue tool invocation") if os.geteuid() != 0: raise SystemExit("the managed-residue tool requires root") TEST_ROOT = os.environ.get("BRIDGESLLM_RESIDUE_TEST_ROOT", "") if TEST_ROOT: if ( not re.fullmatch(r"/tmp/bridgesllm-residue-fixture-[A-Za-z0-9._-]+", TEST_ROOT) or os.path.normpath(TEST_ROOT) != TEST_ROOT or os.path.realpath(TEST_ROOT) != TEST_ROOT ): raise SystemExit("the managed-residue fixture root is unsafe") test_info = os.lstat(TEST_ROOT) if ( not stat.S_ISDIR(test_info.st_mode) or stat.S_ISLNK(test_info.st_mode) or test_info.st_uid != 0 or test_info.st_gid != 0 or stat.S_IMODE(test_info.st_mode) != 0o700 ): raise SystemExit("the managed-residue fixture root is unsafe") def mapped_path(path): if not TEST_ROOT: return path if not path.startswith("/"): raise SystemExit("fixture mapping requires an absolute path") result = os.path.normpath(TEST_ROOT + path) if result != TEST_ROOT and not result.startswith(TEST_ROOT + os.sep): raise SystemExit("fixture path escaped its root") return result TRUSTED_BINARY_NAMES = { "docker", "du", "iptables", "ip6tables", "iptables-save", "ip6tables-save", } TRUSTED_PRODUCTION_BINARY_DIRS = ( "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", ) class BinaryAttestationError(RuntimeError): pass def path_is_within(path, root): try: return os.path.commonpath((path, root)) == root except ValueError: return False def attest_directory(path): info = os.lstat(path) if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 ): raise BinaryAttestationError("binary path crosses an unsafe directory") def resolve_attested_binary(path, anchor): requested = os.path.normpath(path) if ( not os.path.isabs(requested) or not path_is_within(requested, anchor) ): raise BinaryAttestationError("binary path escaped its trust anchor") attest_directory(anchor) seen = set() for _ in range(40): if requested in seen: raise BinaryAttestationError("binary path contains a symlink cycle") seen.add(requested) relative = os.path.relpath(requested, anchor) parts = [] if relative == "." else relative.split(os.sep) current = anchor redirected = None for index, component in enumerate(parts): current = os.path.join(current, component) info = os.lstat(current) if stat.S_ISLNK(info.st_mode): if info.st_uid != 0 or info.st_gid != 0: raise BinaryAttestationError( "binary path contains an unowned symlink") target = os.readlink(current) target_path = ( target if os.path.isabs(target) else os.path.join(os.path.dirname(current), target) ) suffix = parts[index + 1:] redirected = os.path.normpath( os.path.join(target_path, *suffix)) if not path_is_within(redirected, anchor): raise BinaryAttestationError( "binary symlink escaped its trust anchor") break if index < len(parts) - 1: attest_directory(current) continue if ( not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 or not info.st_mode & 0o111 ): raise BinaryAttestationError( "binary target failed its ownership contract") return current if redirected is None: raise BinaryAttestationError("binary path did not resolve to a file") requested = redirected raise BinaryAttestationError("binary path exceeded the symlink safety limit") def attest_trusted_binary(candidate): if ( not isinstance(candidate, str) or not os.path.isabs(candidate) or os.path.normpath(candidate) != candidate or os.path.basename(candidate) not in TRUSTED_BINARY_NAMES ): raise BinaryAttestationError("binary frontend is outside the allowlist") if TEST_ROOT: binary_dir = os.path.join(TEST_ROOT, "bin") if os.path.dirname(candidate) != binary_dir: raise BinaryAttestationError( "fixture binary frontend escaped its sealed directory") resolved = resolve_attested_binary(candidate, TEST_ROOT) if not path_is_within(resolved, binary_dir): raise BinaryAttestationError( "fixture binary target escaped its sealed directory") return resolved if os.path.dirname(candidate) not in TRUSTED_PRODUCTION_BINARY_DIRS: raise BinaryAttestationError( "binary frontend escaped the production allowlist") resolved = resolve_attested_binary(candidate, "/") if not resolved.startswith(("/usr/local/", "/usr/")): raise BinaryAttestationError( "binary target escaped the immutable system tree") return resolved def trusted_binary(name): if name not in TRUSTED_BINARY_NAMES: return None directories = ( (os.path.join(TEST_ROOT, "bin"),) if TEST_ROOT else TRUSTED_PRODUCTION_BINARY_DIRS ) for directory in directories: candidate = os.path.join(directory, name) try: attest_trusted_binary(candidate) except (BinaryAttestationError, FileNotFoundError, OSError): continue # Keep the sealed frontend path: multicall tools such as # xtables-nft-multi select iptables/ip6tables behavior from argv[0]. return candidate return None DOCKER = trusted_binary("docker") if DOCKER is None: raise SystemExit("docker is unavailable; managed residue could not be inspected") MAX_COMMAND_BYTES = 16 * 1024 * 1024 MAX_RESOURCES = 4096 SAFE_ENV = { "PATH": ( os.path.join(TEST_ROOT, "bin") + ":/usr/bin:/bin" if TEST_ROOT else "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ), "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8", } if TEST_ROOT: SAFE_ENV["BRIDGESLLM_RESIDUE_TEST_ROOT"] = TEST_ROOT def run_result(args, tolerate_failure=False): try: resolved = attest_trusted_binary(args[0]) flags = os.O_RDONLY | os.O_CLOEXEC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW descriptor = os.open(resolved, flags) try: opened = os.fstat(descriptor) current = os.stat(resolved, follow_symlinks=False) if ( not stat.S_ISREG(opened.st_mode) or opened.st_uid != 0 or opened.st_gid != 0 or opened.st_mode & 0o022 or not opened.st_mode & 0o111 or (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino) ): raise BinaryAttestationError( "binary changed before its pinned execution") result = subprocess.run( args, executable=f"/proc/self/fd/{descriptor}", pass_fds=(descriptor,), capture_output=True, text=True, timeout=90, env=SAFE_ENV, ) finally: os.close(descriptor) except ( BinaryAttestationError, FileNotFoundError, OSError, subprocess.TimeoutExpired, ): raise SystemExit( f"{os.path.basename(args[0])} failed its pinned binary " f"attestation while handling managed {scope} residue") if len(result.stdout.encode("utf-8", "replace")) > MAX_COMMAND_BYTES: raise SystemExit("command output exceeded the managed-residue safety limit") if result.returncode != 0 and not tolerate_failure: raise SystemExit( f"{os.path.basename(args[0])} failed with exit code " f"{result.returncode} while handling managed {scope} residue") return result def run(args, tolerate_failure=False): return run_result(args, tolerate_failure=tolerate_failure).stdout PROJECT_CONTAINER_RES = [ re.compile(r"^p4e-proxy-[a-f0-9]{20}$"), re.compile(r"^p4ol-[a-f0-9]{24}$"), re.compile(r"^p4cx-[a-f0-9]{24}$"), re.compile(r"^p4cc-[a-f0-9]{24}$"), re.compile(r"^p4ag-[a-f0-9]{24}$"), re.compile(r"^p4oc-[a-f0-9]{16}-[a-z0-9._-]{1,32}-[a-f0-9]{8}$"), re.compile(r"^bridgesllm-a0p-[a-f0-9]{24}$"), re.compile(r"^bridgesllm-project-(app|job|git)-[a-f0-9]{20}$"), ] PROJECT_CONTAINER_LABELS = [ "com.bridgesllm.project-egress.runtime-fingerprint", "com.bridgesllm.project-egress.policy=portal-project-egress-v1", "com.bridgesllm.project-workload.policy=portal-project-workload-v1", "com.bridgesllm.ollama-project.policy=portal-ollama-project-sandbox-v1", "com.bridgesllm.project-runtime=true", "com.bridgesllm.project-git=true", "io.bridgesllm.managed=agent-zero-project", ] PROJECT_CONTAINER_LABEL_RULES = { "com.bridgesllm.project-egress.runtime-fingerprint": lambda value: bool(re.fullmatch(r"[a-f0-9]{64}", value)), "com.bridgesllm.project-egress.policy": lambda value: value == "portal-project-egress-v1", "com.bridgesllm.project-workload.policy": lambda value: value == "portal-project-workload-v1", "com.bridgesllm.ollama-project.policy": lambda value: value == "portal-ollama-project-sandbox-v1", "com.bridgesllm.project-runtime": lambda value: value == "true", "com.bridgesllm.project-git": lambda value: value == "true", "io.bridgesllm.managed": lambda value: value == "agent-zero-project", } PROJECT_NETWORK_RE = re.compile(r"^p4e-(in|out)-[a-f0-9]{20}$") PROJECT_NETWORK_LABELS = [ "io.bridgesllm.managed=agent-zero-project", "com.bridgesllm.project-egress.policy=portal-project-egress-v1", ] PROJECT_NETWORK_LABEL_RULES = { "io.bridgesllm.managed": lambda value: value == "agent-zero-project", "com.bridgesllm.project-egress.policy": lambda value: value == "portal-project-egress-v1", } PROJECT_VOLUME_RE = re.compile(r"^bridgesllm-a0p-[a-f0-9]{24}-usr$") PROJECT_VOLUME_LABELS = ["io.bridgesllm.managed=agent-zero-project"] PROJECT_VOLUME_LABEL_RULES = { "io.bridgesllm.managed": lambda value: value == "agent-zero-project", } AGENT_ZERO_CONTAINER_RES = [ re.compile(r"^bridgesllm-agent-zero$"), re.compile(r"^bridgesllm-agent-zero-rollback$"), re.compile(r"^bridgesllm-agent-zero-pre-rollback-[0-9]{8}T[0-9]{6}Z$"), ] AGENT_ZERO_VOLUME = "bridgesllm-agent-zero-usr" AGENT_ZERO_IMAGES = { "agent0ai/agent-zero@sha256:9b48534c1279fb831513b8c970e2d9004e7a2a6708a4d53a91a76d24a4f9f7eb", "agent0ai/agent-zero@sha256:da107b689828124369d83f017b9664493c0699c60e57809fbd32f647078de49c", } AGENT_ZERO_PATHS = [ mapped_path("/etc/bridgesllm/agent-zero.env"), mapped_path("/var/lib/bridgesllm/agent-zero-runtime"), mapped_path("/var/backups/bridgesllm/agent-zero"), ] MANAGED_CHAIN_RE = re.compile(r"^(P4E-|A0P-)[A-Za-z0-9_.:-]+$") MANAGED_COMMENT_RE = re.compile(r"^(p4e-v1|a0p-v3):[a-f0-9]{64}:[a-f0-9]{64}$") P4E_PROJECT_CHAIN_RE = re.compile(r"^P4E-[A-F0-9]{23}$") A0P_PROJECT_CHAIN_RE = re.compile(r"^A0P-[A-F0-9]{24}$") P4E_MASTER_CHAIN = "P4E-MASTER-V1" P4E_HOST_CHAIN = "P4E-HOST-V1" want_project = scope in {"all", "project"} want_agent_zero = scope in {"all", "agent-zero"} def docker_names(kind): args = [DOCKER, kind, "ls", "--format", "{{.Names}}" if kind == "container" else "{{.Name}}"] if kind == "container": args.insert(3, "--all") names = {line.strip() for line in run(args).splitlines() if line.strip()} if len(names) > MAX_RESOURCES: raise SystemExit("Docker name inventory exceeded the safety limit") return names def docker_identities(kind): args = [DOCKER, kind, "ls"] if kind == "container": args.extend(["--all", "--no-trunc", "--format", "{{.ID}}"]) elif kind == "network": args.extend(["--no-trunc", "--format", "{{.ID}}"]) else: args.extend(["--format", "{{.Name}}"]) values = [line.strip() for line in run(args).splitlines() if line.strip()] if ( len(values) > MAX_RESOURCES or len(values) != len(set(values)) or ( kind in {"container", "network"} and not all(re.fullmatch(r"[a-f0-9]{64}", value) for value in values) ) or ( kind == "volume" and not all(re.fullmatch( r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", value) for value in values) ) ): raise SystemExit("Docker identity inventory was invalid") return set(values) def docker_label_ids(kind, label): args = [DOCKER, kind, "ls", "--quiet", "--filter", f"label={label}"] if kind == "container": args.insert(3, "--all") args.insert(4, "--no-trunc") elif kind == "network": args.insert(3, "--no-trunc") identities = {line.strip() for line in run(args).splitlines() if line.strip()} if len(identities) > MAX_RESOURCES: raise SystemExit("Docker label inventory exceeded the safety limit") return identities def normalized_labels(value): if value is None: return {} if not isinstance(value, dict): raise SystemExit("Docker returned invalid resource labels") if not all(isinstance(key, str) and isinstance(item, str) for key, item in value.items()): raise SystemExit("Docker returned invalid resource labels") return dict(value) def inspect_resource(kind, reference, tolerate_missing=False, include_size=False): args = [DOCKER, kind, "inspect"] if kind == "container" and include_size: args.append("--size") args.append(reference) result = run_result(args, tolerate_failure=tolerate_missing) if result.returncode != 0: return None try: payload = json.loads(result.stdout) except json.JSONDecodeError: raise SystemExit("Docker inspection returned invalid JSON") if not isinstance(payload, list) or len(payload) != 1 or not isinstance(payload[0], dict): raise SystemExit("Docker inspection was ambiguous") item = payload[0] if kind == "container": identity = item.get("Id") name = str(item.get("Name") or "").lstrip("/") labels = normalized_labels((item.get("Config") or {}).get("Labels")) mounts = item.get("Mounts") if ( not isinstance(mounts, list) or not all(isinstance(mount, dict) for mount in mounts) ): raise SystemExit("Docker returned invalid container mounts") mounts = sorted( mounts, key=lambda mount: json.dumps( mount, sort_keys=True, separators=(",", ":") ), ) stable = { "id": identity, "name": name, "config": item.get("Config"), "hostConfig": item.get("HostConfig"), # Docker does not guarantee inspect ordering for this set. "mounts": mounts, } size = item.get("SizeRw") running = bool((item.get("State") or {}).get("Running")) restart_policy = str( (((item.get("HostConfig") or {}).get("RestartPolicy") or {}).get("Name")) or "no" ) elif kind == "network": identity = item.get("Id") name = item.get("Name") labels = normalized_labels(item.get("Labels")) stable = { key: item.get(key) for key in ( "Id", "Name", "Driver", "Scope", "Internal", "Attachable", "Ingress", "IPAM", "Options", "Labels", ) } size = None running = False restart_policy = "no" else: identity = item.get("Name") name = identity labels = normalized_labels(item.get("Labels")) stable = { key: item.get(key) for key in ( "Name", "Driver", "Scope", "Mountpoint", "Labels", "Options", "CreatedAt", ) } size = None running = False restart_policy = "no" if not isinstance(name, str) or not re.fullmatch( r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", name): raise SystemExit("Docker returned an invalid resource name") if kind != "volume" and ( not isinstance(identity, str) or not re.fullmatch(r"[a-f0-9]{64}", identity)): raise SystemExit("Docker returned an invalid immutable resource identity") canonical = json.dumps(stable, sort_keys=True, separators=(",", ":")) return { "kind": kind, "identity": identity, "name": name, "labels": labels, "stable": stable, "attestation": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), "size": size, "mountpoint": item.get("Mountpoint") if kind == "volume" else None, "running": running, "restartPolicy": restart_policy, } def valid_label_claim(labels, rules): claims = [(key, validator) for key, validator in rules.items() if key in labels] return bool(claims) and all(validator(labels[key]) for key, validator in claims) def contradictory_managed_labels(kind, labels): rules = { "container": PROJECT_CONTAINER_LABEL_RULES, "network": PROJECT_NETWORK_LABEL_RULES, "volume": PROJECT_VOLUME_LABEL_RULES, }[kind] return any( key in labels and not validator(labels[key]) for key, validator in rules.items() ) def exact_managed_name(kind, name): if kind == "container": return ( want_project and any(pattern.fullmatch(name) for pattern in PROJECT_CONTAINER_RES) ) or ( want_agent_zero and any(pattern.fullmatch(name) for pattern in AGENT_ZERO_CONTAINER_RES) ) if kind == "network": return want_project and bool(PROJECT_NETWORK_RE.fullmatch(name)) return ( want_project and bool(PROJECT_VOLUME_RE.fullmatch(name)) ) or ( want_agent_zero and name == AGENT_ZERO_VOLUME ) def exact_managed_labels(kind, labels): if not want_project: return False rules = { "container": PROJECT_CONTAINER_LABEL_RULES, "network": PROJECT_NETWORK_LABEL_RULES, "volume": PROJECT_VOLUME_LABEL_RULES, }[kind] return valid_label_claim(labels, rules) def agent_zero_container_contract(record): if not any( pattern.fullmatch(record["name"]) for pattern in AGENT_ZERO_CONTAINER_RES ): return True stable = record["stable"] config = stable.get("config") or {} host = stable.get("hostConfig") or {} mounts = stable.get("mounts") or [] labels = normalized_labels(config.get("Labels")) pre_rollback = bool(re.fullmatch( r"bridgesllm-agent-zero-pre-rollback-[0-9]{8}T[0-9]{6}Z", record["name"], )) managed_label = labels.get("io.bridgesllm.agent-zero.managed") if ( config.get("Image") not in AGENT_ZERO_IMAGES or labels.get("io.bridgesllm.agent-zero.version") != "2.5" or managed_label not in {None, "true"} or ((host.get("RestartPolicy") or {}).get("Name")) != "unless-stopped" or len(host.get("PortBindings") or {}) != 1 or (host.get("PortBindings") or {}).get("80/tcp") != [{"HostIp": "127.0.0.1", "HostPort": "50001"}] or len(mounts) != 2 or (pre_rollback and record["running"]) ): return False data_mounts = [ mount for mount in mounts if isinstance(mount, dict) and mount.get("Destination") == "/a0/usr" ] auth_mounts = [ mount for mount in mounts if isinstance(mount, dict) and mount.get("Destination") == "/a0/.env" ] if len(data_mounts) != 1 or len(auth_mounts) != 1: return False data_mount = data_mounts[0] auth_mount = auth_mounts[0] return ( data_mount.get("Type") == "volume" and data_mount.get("RW") is True and ( pre_rollback and bool(re.fullmatch( r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", str(data_mount.get("Name") or ""), )) or data_mount.get("Name") == AGENT_ZERO_VOLUME ) and auth_mount.get("Type") == "bind" and auth_mount.get("Source") == mapped_path("/etc/bridgesllm/agent-zero.env") and auth_mount.get("RW") is False ) def agent_zero_volume_contract(record): if record["name"] != AGENT_ZERO_VOLUME: return True stable = record["stable"] labels = normalized_labels(stable.get("Labels")) root = run([DOCKER, "info", "--format", "{{.DockerRootDir}}"]).strip() expected_mountpoint = os.path.join( root.rstrip("/"), "volumes", AGENT_ZERO_VOLUME, "_data") try: mount_info = os.lstat(expected_mountpoint) except FileNotFoundError: return False if ( not os.path.isabs(root) or stable.get("Driver") != "local" or stable.get("Scope") != "local" or labels not in ({}, {"io.bridgesllm.agent-zero.managed": "true"}) or stable.get("Mountpoint") != expected_mountpoint or not os.path.isdir(expected_mountpoint) or os.path.islink(expected_mountpoint) or os.path.realpath(expected_mountpoint) != expected_mountpoint or mount_info.st_uid != 0 or mount_info.st_gid != 0 ): return False references = run([ DOCKER, "container", "ls", "--all", "--no-trunc", "--filter", f"volume={AGENT_ZERO_VOLUME}", "--format", "{{.ID}}", ]).splitlines() selected_ids = { item["identity"] for item in container_records if any( pattern.fullmatch(item["name"]) for pattern in AGENT_ZERO_CONTAINER_RES ) } return all( re.fullmatch(r"[a-f0-9]{64}", reference.strip()) and reference.strip() in selected_ids for reference in references if reference.strip() ) def canonical_records(kind, references): records = {} for reference in sorted(references): record = inspect_resource( kind, reference, include_size=kind == "container") if record is None: raise SystemExit("Docker residue disappeared during its initial attestation") if contradictory_managed_labels(kind, record["labels"]): raise SystemExit( f"Docker {kind} carries a contradictory Portal ownership label") if not ( exact_managed_name(kind, record["name"]) or exact_managed_labels(kind, record["labels"]) ): raise SystemExit( f"Docker {kind} has a contradictory or incomplete Portal ownership contract") if kind == "container" and not agent_zero_container_contract(record): raise SystemExit( "Agent Zero container residue failed its exact managed contract") if kind == "volume" and not agent_zero_volume_contract(record): raise SystemExit( "Agent Zero volume residue failed its exact managed contract") existing = records.get(record["identity"]) if existing is not None and existing["attestation"] != record["attestation"]: raise SystemExit( "Docker returned contradictory duplicate " f"{kind} identity for {record['name']}" ) records[record["identity"]] = record return [records[identity] for identity in sorted(records)] container_refs, network_refs, volume_refs = set(), set(), set() container_names = docker_names("container") network_names = docker_names("network") volume_names = docker_names("volume") if want_project: container_refs |= {name for name in container_names if any(pattern.fullmatch(name) for pattern in PROJECT_CONTAINER_RES)} network_refs |= {name for name in network_names if PROJECT_NETWORK_RE.fullmatch(name)} volume_refs |= {name for name in volume_names if PROJECT_VOLUME_RE.fullmatch(name)} for label in PROJECT_CONTAINER_LABELS: container_refs |= docker_label_ids("container", label) for label in PROJECT_NETWORK_LABELS: network_refs |= docker_label_ids("network", label) for label in PROJECT_VOLUME_LABELS: volume_refs |= docker_label_ids("volume", label) if want_agent_zero: container_refs |= {name for name in container_names if any(pattern.fullmatch(name) for pattern in AGENT_ZERO_CONTAINER_RES)} volume_refs |= {name for name in volume_names if name == AGENT_ZERO_VOLUME} container_records = canonical_records("container", container_refs) network_records = canonical_records("network", network_refs) volume_records = canonical_records("volume", volume_refs) def decode_mount_path(value): return ( value.replace("\\040", " ") .replace("\\011", "\t") .replace("\\012", "\n") .replace("\\134", "\\") ) def current_mountpoints(): mountinfo_path = mapped_path("/proc/self/mountinfo") try: with open(mountinfo_path, "r", encoding="utf-8") as handle: return { mapped_path(decode_mount_path(columns[4])) if TEST_ROOT else decode_mount_path(columns[4]) for line in handle if len((columns := line.split())) >= 5 } except OSError: raise SystemExit("mount topology could not be inspected") def descriptor_mount_id(descriptor): try: with open( f"/proc/self/fdinfo/{descriptor}", "r", encoding="utf-8", ) as handle: matches = [ line.split(":", 1)[1].strip() for line in handle if line.startswith("mnt_id:") ] except OSError: raise SystemExit("open descriptor mount identity could not be read") if ( len(matches) != 1 or not re.fullmatch(r"[0-9]+", matches[0]) or int(matches[0]) <= 0 ): raise SystemExit("open descriptor mount identity was invalid") return int(matches[0]) def path_record(path): if path not in AGENT_ZERO_PATHS or not os.path.isabs(path): raise SystemExit("managed residue path escaped its exact ownership boundary") if not os.path.lexists(path): return None current = TEST_ROOT if TEST_ROOT else "/" relative = os.path.relpath(path, current) if relative == ".." or relative.startswith(".." + os.sep): raise SystemExit("managed residue path escaped its fixture root") for component in relative.split(os.sep)[:-1]: current = os.path.join(current, component) info = os.lstat(current) if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 ): raise SystemExit("managed residue path crosses an unsafe ancestor") parent = os.path.dirname(path) parent_info = os.lstat(parent) try: info = os.lstat(path) except FileNotFoundError: return None if ( stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or os.path.realpath(path) != path ): raise SystemExit("managed residue path is unsafe") expected_type = "file" if path.endswith(".env") else "directory" for mountpoint in current_mountpoints(): if mountpoint == path or mountpoint.startswith(path + os.sep): raise SystemExit("managed Agent Zero residue contains a mount boundary") if expected_type == "file": if ( not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 ): raise SystemExit("managed Agent Zero environment residue is unsafe") else: if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700: raise SystemExit("managed Agent Zero directory residue is unsafe") for entry in os.scandir(path): entry_info = entry.stat(follow_symlinks=False) if ( stat.S_ISLNK(entry_info.st_mode) or entry_info.st_uid != 0 or entry_info.st_gid != 0 ): raise SystemExit( "managed Agent Zero residue has an unsafe top-level entry") return { "path": path, "device": info.st_dev, "inode": info.st_ino, "mode": info.st_mode, "uid": info.st_uid, "gid": info.st_gid, "type": expected_type, "parentDevice": parent_info.st_dev, "parentInode": parent_info.st_ino, "parentMode": parent_info.st_mode, } path_records = [] if want_agent_zero: for candidate in AGENT_ZERO_PATHS: record = path_record(candidate) if record is not None: path_records.append(record) paths = [record["path"] for record in path_records] def shaped_path_quarantines(): discovered = set() for path in AGENT_ZERO_PATHS: parent = os.path.dirname(path) try: info = os.lstat(parent) except FileNotFoundError: continue if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 ): raise SystemExit( "managed residue quarantine parent is unsafe") prefix = f".{os.path.basename(path)}.bridgesllm-wipe-" for name in os.listdir(parent): if name.startswith(prefix): discovered.add(os.path.join(parent, name)) return discovered quarantine_candidates = ( shaped_path_quarantines() if want_agent_zero else set()) if ( quarantine_candidates and mode in {"report", "seal-plan", "quiesce"} ): raise SystemExit( "an unexpected managed-residue quarantine requires recovery") def normalize_firewall_line(value): normalized = value.strip() normalized = re.sub(r'--comment "([^"]*)"', r"--comment \1", normalized) normalized = re.sub( r" -j REJECT --reject-with (?:icmp6?-port-unreachable|icmp-port-unreachable)$", " -j REJECT", normalized, ) return normalized def managed_firewall_comment(line): matches = re.findall(r"(?:^| )--comment ([^ ]+)(?= |$)", line) return matches[0] if len(matches) == 1 else None def comment_matches_chain(comment, chain): p4e = re.fullmatch(r"p4e-v1:[a-f0-9]{64}:([a-f0-9]{64})", comment) if p4e and P4E_PROJECT_CHAIN_RE.fullmatch(chain): return chain == f"P4E-{p4e.group(1)[:23].upper()}" a0p = re.fullmatch(r"a0p-v3:[a-f0-9]{64}:([a-f0-9]{64})", comment) return bool( a0p and A0P_PROJECT_CHAIN_RE.fullmatch(chain) and chain == f"A0P-{a0p.group(1)[:24].upper()}" ) def reference_lines(lines, chain): target = re.compile(rf" (?:-j|-g) {re.escape(chain)}(?: |$)") return [ line for line in lines if not line.startswith(f"-A {chain} ") and target.search(line) ] def valid_p4e_chain_rules(lines, chain, comment): escaped_chain = re.escape(chain) escaped_comment = re.escape(comment) cidr = r"[0-9A-Fa-f:.]+/[0-9]{1,3}" deny = re.compile( rf"^-A {escaped_chain} -d {cidr} -m comment --comment " rf"{escaped_comment} -j REJECT$") ipv4_return = re.compile( rf"^-A {escaped_chain} -p tcp -m multiport --dports 80,443 " rf"-m comment --comment {escaped_comment} -j RETURN$") ipv6_return = re.compile( rf"^-A {escaped_chain} -d 2000::/3 -p tcp -m multiport " rf"--dports 80,443 -m comment --comment {escaped_comment} -j RETURN$") final_reject = re.compile( rf"^-A {escaped_chain} -m comment --comment " rf"{escaped_comment} -j REJECT$") allowed = (deny, ipv4_return, ipv6_return, final_reject) return ( len(lines) >= 2 and all(any(pattern.fullmatch(line) for pattern in allowed) for line in lines) and sum( bool(ipv4_return.fullmatch(line) or ipv6_return.fullmatch(line)) for line in lines ) == 1 and bool(final_reject.fullmatch(lines[-1])) ) def valid_a0p_chain_rules(lines, chain, comment): escaped_chain = re.escape(chain) escaped_comment = re.escape(comment) established = re.compile( rf"^-A {escaped_chain} -m conntrack --ctstate RELATED,ESTABLISHED " rf"-m comment --comment {escaped_comment} -j ACCEPT$") destination = re.compile( rf"^-A {escaped_chain} -d [0-9.]+/32 -p tcp -m tcp " rf"--dport [0-9]{{1,5}} -m comment --comment " rf"{escaped_comment} -j ACCEPT$") reject = re.compile( rf"^-A {escaped_chain} -m comment --comment " rf"{escaped_comment} -j REJECT$") return ( len(lines) == 4 and bool(established.fullmatch(lines[0])) and bool(destination.fullmatch(lines[1])) and bool(destination.fullmatch(lines[2])) and bool(reject.fullmatch(lines[3])) ) def parse_managed_firewall(tool, binary, payload): lines = [ normalize_firewall_line(line) for line in payload.splitlines() if line.strip() ] if len(lines) > MAX_RESOURCES: raise SystemExit(f"{tool} firewall inventory exceeded the safety limit") declarations = { match.group(1) for line in lines if (match := re.fullmatch(r"-N ([A-Za-z0-9_.:-]+)", line)) } broad = {name for name in declarations if MANAGED_CHAIN_RE.fullmatch(name)} project_chains = sorted( name for name in broad if P4E_PROJECT_CHAIN_RE.fullmatch(name) or A0P_PROJECT_CHAIN_RE.fullmatch(name) ) shared_chains = { name for name in (P4E_MASTER_CHAIN, P4E_HOST_CHAIN) if name in declarations } if broad != set(project_chains) | shared_chains: raise SystemExit(f"{tool} contains an ambiguous Portal-shaped chain") consumed = set() for chain in project_chains: local_rules = [line for line in lines if line.startswith(f"-A {chain} ")] comments = { comment for line in local_rules if (comment := managed_firewall_comment(line)) } if len(comments) != 1: raise SystemExit(f"{tool} contains an ambiguous Portal chain") comment = next(iter(comments)) if ( not all(managed_firewall_comment(line) == comment for line in local_rules) or not comment_matches_chain(comment, chain) or ( chain.startswith("P4E-") and not valid_p4e_chain_rules(local_rules, chain, comment) ) or ( chain.startswith("A0P-") and not valid_a0p_chain_rules(local_rules, chain, comment) ) ): raise SystemExit(f"{tool} contains an ambiguous Portal chain") parents = reference_lines(lines, chain) escaped_chain = re.escape(chain) escaped_comment = re.escape(comment) cidr = r"[0-9A-Fa-f:.]+/[0-9]{1,3}" if chain.startswith("P4E-"): parent_pattern = re.compile( rf"^-A {P4E_MASTER_CHAIN} -s {cidr} -m comment --comment " rf"{escaped_comment} -j {escaped_chain}$") else: parent_pattern = re.compile( rf"^-A (?:INPUT|DOCKER-USER) -s {cidr} -m comment --comment " rf"{escaped_comment} -j {escaped_chain}$") if not all(parent_pattern.fullmatch(line) for line in parents): raise SystemExit(f"{tool} contains an ambiguous Portal chain reference") consumed.update(local_rules) consumed.update(parents) consumed.add(f"-N {chain}") host_rules = [ line for line in lines if line.startswith(f"-A {P4E_HOST_CHAIN} ") ] # Match what the Portal actually writes into the host chain. Every managed # subnet gets a conntrack RELATED,ESTABLISHED ACCEPT followed by a REJECT # (see projectEgressPlane.ts), `iptables -S` quotes comments, and REJECT # carries a --reject-with suffix. The previous pattern accepted none of # those, so any box that had ever run a Project with egress control failed # this check and could not be uninstalled at all. # # Still fail-closed: ACCEPT is only recognised together with the conntrack # match, so a bare permissive ACCEPT in this chain remains ambiguous. host_comment = r"\"?p4e-v1:[a-f0-9]{64}:[a-f0-9]{64}\"?" host_cidr = r"[0-9A-Fa-f:.]+/[0-9]{1,3}" host_established_pattern = re.compile( rf"^-A {P4E_HOST_CHAIN} -s {host_cidr} " rf"-m conntrack --ctstate RELATED,ESTABLISHED " rf"-m comment --comment {host_comment} -j ACCEPT$") host_reject_pattern = re.compile( rf"^-A {P4E_HOST_CHAIN} -s {host_cidr} " rf"-m comment --comment {host_comment} " rf"-j REJECT(?: --reject-with [a-z0-9-]+)?$") if not all( host_established_pattern.fullmatch(line) or host_reject_pattern.fullmatch(line) for line in host_rules ): raise SystemExit(f"{tool} contains ambiguous Portal host rules") consumed.update(host_rules) master_rules = [ line for line in lines if line.startswith(f"-A {P4E_MASTER_CHAIN} ") ] recognized_master = { line for chain in project_chains if chain.startswith("P4E-") for line in reference_lines(lines, chain) } if any(line not in recognized_master for line in master_rules): raise SystemExit(f"{tool} contains ambiguous Portal master rules") master_parents = reference_lines(lines, P4E_MASTER_CHAIN) host_parents = reference_lines(lines, P4E_HOST_CHAIN) if ( any(line != f"-A DOCKER-USER -j {P4E_MASTER_CHAIN}" for line in master_parents) or any(line != f"-A INPUT -j {P4E_HOST_CHAIN}" for line in host_parents) or ( P4E_MASTER_CHAIN not in declarations and (master_rules or master_parents) ) or ( P4E_HOST_CHAIN not in declarations and (host_rules or host_parents) ) ): raise SystemExit(f"{tool} contains ambiguous Portal shared-chain rules") consumed.update(master_parents) consumed.update(host_parents) for chain in shared_chains: consumed.add(f"-N {chain}") for line in lines: shaped = ( bool(re.match(r"^-N (?:P4E-|A0P-)", line)) or bool(re.search(r" --comment (?:p4e-v1|a0p-v3):", line)) or bool(re.search(r" (?:-j|-g) (?:P4E-|A0P-)", line)) ) if shaped and line not in consumed: raise SystemExit(f"{tool} contains unrecognized Portal-shaped rules") managed_chains = sorted(set(project_chains) | shared_chains) references = [] for chain in managed_chains: for line in reference_lines(lines, chain): parent = line.split()[1] if parent not in managed_chains: tokens = shlex.split(line) references.append((parent, tokens[2:], line)) return { "binary": binary, "chains": managed_chains, "references": references, "snapshot": "\n".join(lines), } firewall = {} if want_project: for tool in ("iptables", "ip6tables"): binary = trusted_binary(tool) if binary is None: raise SystemExit(f"{tool} is unavailable; firewall residue could not be inspected") payload = run([binary, "-w", "-S"]) if mode in {"report", "seal-plan", "prove-absent"}: state = parse_managed_firewall(tool, binary, payload) firewall[tool] = state else: firewall[tool] = { "binary": binary, "currentSnapshot": "\n".join( normalize_firewall_line(line) for line in payload.splitlines() if line.strip() ), } def remove_first_line(lines, target): updated = list(lines) try: updated.remove(target) except ValueError: raise SystemExit("managed firewall plan could not be constructed") return updated def firewall_mutation_plan(state): lines = state["snapshot"].splitlines() if state["snapshot"] else [] initial = "\n".join(lines) mutations = [] for parent, spec, line in state["references"]: lines = remove_first_line(lines, line) mutations.append({ "args": ["-D", parent, *spec], "after": "\n".join(lines), }) for chain in state["chains"]: local_rules = [ line for line in lines if line.startswith(f"-A {chain} ") ] for line in local_rules: tokens = shlex.split(line) if len(tokens) < 3 or tokens[:2] != ["-A", chain]: raise SystemExit( "managed firewall rule could not be sealed exactly") lines = remove_first_line(lines, line) mutations.append({ "args": ["-D", chain, *tokens[2:]], "after": "\n".join(lines), }) for chain in state["chains"]: lines = remove_first_line(lines, f"-N {chain}") mutations.append({ "args": ["-X", chain], "after": "\n".join(lines), }) return { "initial": initial, "mutations": mutations, "final": "\n".join(lines), } def record_scopes(record): scopes = [] if record["kind"] == "container": if ( any(pattern.fullmatch(record["name"]) for pattern in PROJECT_CONTAINER_RES) or valid_label_claim( record["labels"], PROJECT_CONTAINER_LABEL_RULES) ): scopes.append("project") if any( pattern.fullmatch(record["name"]) for pattern in AGENT_ZERO_CONTAINER_RES ): scopes.append("agent-zero") elif record["kind"] == "network": if ( PROJECT_NETWORK_RE.fullmatch(record["name"]) or valid_label_claim(record["labels"], PROJECT_NETWORK_LABEL_RULES) ): scopes.append("project") else: if ( PROJECT_VOLUME_RE.fullmatch(record["name"]) or valid_label_claim(record["labels"], PROJECT_VOLUME_LABEL_RULES) ): scopes.append("project") if record["name"] == AGENT_ZERO_VOLUME: scopes.append("agent-zero") if not scopes: raise SystemExit("managed Docker residue had no exact lifecycle scope") return sorted(set(scopes)) def create_plan_document(): docker_plan = {} for kind, records in ( ("container", container_records), ("network", network_records), ("volume", volume_records), ): docker_plan[kind] = [ { "identity": record["identity"], "name": record["name"], "attestation": record["attestation"], "scopes": record_scopes(record), } for record in records ] firewall_plan = { tool: firewall_mutation_plan(state) for tool, state in sorted(firewall.items()) } return { "schema": "bridgesllm.uninstall-residue-plan.v1", "docker": docker_plan, "paths": path_records, "firewall": firewall_plan, } def canonical_plan(document): return json.dumps(document, sort_keys=True, separators=(",", ":")) def plan_digest(document): return hashlib.sha256(canonical_plan(document).encode("utf-8")).hexdigest() plan_document = create_plan_document() if mode in {"report", "seal-plan"} else None current_plan_digest = plan_digest(plan_document) if plan_document is not None else "" present = bool( container_records or network_records or volume_records or paths or any( state.get("chains") or state.get("references") for state in firewall.values() ) ) def size_of_path(path): du = trusted_binary("du") if du is None: return None result = run_result([du, "-sb", "--", path], tolerate_failure=True) try: return int(result.stdout.split()[0]) if result.returncode == 0 else None except (ValueError, IndexError): return None if mode == "report": estimated = 0 size_complete = True for record in container_records: try: estimated += max(int(record["size"]), 0) except (TypeError, ValueError): size_complete = False for record in volume_records: mountpoint = record["mountpoint"] if mountpoint and os.path.isdir(mountpoint): measured = size_of_path(mountpoint) if measured is None: size_complete = False else: estimated += measured else: size_complete = False for path in paths: measured = size_of_path(path) if measured is None: size_complete = False else: estimated += measured print( f"present={'true' if present else 'false'} " f"planDigest={current_plan_digest} " f"sizeComplete={'true' if size_complete else 'false'} bytes={estimated}") for record in container_records: print(f"Docker container: {record['name']}") for record in network_records: print(f"Docker network: {record['name']}") for record in volume_records: print(f"Docker volume: {record['name']}") for path in paths: print(f"File tree: {path}") for tool, state in sorted(firewall.items()): if state["chains"] or state["references"]: print(f"Firewall ({tool}): {len(state['chains'])} Portal chain(s), " f"{len(state['references'])} referencing rule(s)") raise SystemExit(0) if mode == "seal-plan": plan_root = mapped_path( "/var/lib/bridgesllm-installer/uninstall/transactions") if ( not re.fullmatch(r"[a-f0-9]{64}", expected_plan_digest) or expected_plan_digest != current_plan_digest or not os.path.isabs(plan_path) or os.path.normpath(plan_path) != plan_path or os.path.basename(plan_path) != "residue-plan.json" or not re.fullmatch( re.escape(plan_root) + r"/[a-f0-9]{32}/residue-plan\.json", plan_path) ): raise SystemExit( "managed residue changed after confirmation; restart uninstall to review it") parent = os.path.dirname(plan_path) parent_info = os.lstat(parent) if ( not stat.S_ISDIR(parent_info.st_mode) or stat.S_ISLNK(parent_info.st_mode) or parent_info.st_uid != 0 or parent_info.st_gid != 0 or stat.S_IMODE(parent_info.st_mode) != 0o700 or os.path.realpath(parent) != parent ): raise SystemExit("the uninstall residue-plan directory is unsafe") envelope = { "schema": "bridgesllm.sealed-uninstall-residue-plan.v1", "planDigest": current_plan_digest, "plan": plan_document, } payload = json.dumps(envelope, indent=2, sort_keys=True) + "\n" descriptor = os.open( plan_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600, ) with os.fdopen(descriptor, "w", encoding="utf-8") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) parent_fd = os.open( parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(parent_fd) finally: os.close(parent_fd) raise SystemExit(0) if not present and mode not in {"quiesce", "wipe", "prove-absent"}: raise SystemExit(0) def load_sealed_plan(): plan_root = mapped_path( "/var/lib/bridgesllm-installer/uninstall/transactions") if ( not os.path.isabs(plan_path) or os.path.normpath(plan_path) != plan_path or os.path.basename(plan_path) != "residue-plan.json" or not re.fullmatch( re.escape(plan_root) + r"/[a-f0-9]{32}/residue-plan\.json", plan_path) ): raise SystemExit("the uninstall residue plan escaped its transaction boundary") info = os.lstat(plan_path) if ( not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or stat.S_IMODE(info.st_mode) != 0o600 or info.st_size <= 0 or info.st_size > 8 * 1024 * 1024 ): raise SystemExit("the uninstall residue plan is unsafe") with open(plan_path, "r", encoding="utf-8") as handle: envelope = json.load(handle) if ( not isinstance(envelope, dict) or set(envelope) != {"schema", "planDigest", "plan"} or envelope.get("schema") != "bridgesllm.sealed-uninstall-residue-plan.v1" or not re.fullmatch(r"[a-f0-9]{64}", str(envelope.get("planDigest", ""))) or not isinstance(envelope.get("plan"), dict) or plan_digest(envelope["plan"]) != envelope["planDigest"] ): raise SystemExit("the uninstall residue plan failed integrity validation") document = envelope["plan"] if ( document.get("schema") != "bridgesllm.uninstall-residue-plan.v1" or set(document) != {"schema", "docker", "paths", "firewall"} or set(document.get("docker", {})) != {"container", "network", "volume"} or not isinstance(document.get("paths"), list) or not isinstance(document.get("firewall"), dict) ): raise SystemExit("the uninstall residue plan has an invalid schema") return document sealed_plan = load_sealed_plan() global_identities_by_kind = { kind: docker_identities(kind) for kind in ("container", "network", "volume") } planned_by_kind = {} for kind in ("container", "network", "volume"): planned = sealed_plan["docker"].get(kind) if not isinstance(planned, list) or len(planned) > MAX_RESOURCES: raise SystemExit("the uninstall residue plan exceeds its Docker limit") selected = {} for record in planned: if ( not isinstance(record, dict) or set(record) != { "identity", "name", "attestation", "scopes", } or not isinstance(record.get("scopes"), list) or not record["scopes"] or any(item not in {"project", "agent-zero"} for item in record["scopes"]) or record["scopes"] != sorted(set(record["scopes"])) or not re.fullmatch(r"[a-f0-9]{64}", str(record.get("attestation", ""))) or not isinstance(record.get("name"), str) or not re.fullmatch( r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", record["name"], ) ): raise SystemExit("the uninstall residue plan has an invalid Docker record") key = record.get("identity") if ( ( kind in {"container", "network"} and not isinstance(key, str) ) or ( kind in {"container", "network"} and not re.fullmatch(r"[a-f0-9]{64}", key) ) or ( kind == "volume" and key != record["name"] ) ): raise SystemExit( "the uninstall residue plan has an invalid Docker identity") if scope == "all" or scope in record["scopes"]: if key in selected: raise SystemExit( "the uninstall residue plan has duplicate Docker identities") selected[key] = record planned_by_kind[kind] = selected for kind, current_records in ( ("container", container_records), ("network", network_records), ("volume", volume_records), ): present = {} listed_identities = global_identities_by_kind[kind] for identity, planned in planned_by_kind[kind].items(): inspected = inspect_resource( kind, identity, tolerate_missing=True, include_size=kind == "container", ) if mode == "prove-absent": if identity in listed_identities or inspected is not None: raise SystemExit( f"a user-confirmed Docker {kind} still exists at " "terminal cleanup") continue if inspected is None: if identity in listed_identities: raise SystemExit( f"a listed user-confirmed Docker {kind} could not be " "re-inspected by its full identity") if mode == "quiesce": raise SystemExit( f"a user-confirmed Docker {kind} disappeared before " "Safe cleanup") # A wipe can resume after an earlier invocation durably removed # an object but crashed before advancing the uninstall phase. continue if ( identity not in listed_identities or inspected["identity"] != identity or inspected["name"] != planned["name"] or inspected["attestation"] != planned["attestation"] ): raise SystemExit( f"a user-confirmed Docker {kind} changed identity, name, " "labels, or topology after confirmation") present[identity] = planned current = {record["identity"]: record for record in current_records} if ( len(current) != len(current_records) or set(current) != set(present) ): raise SystemExit( "the current and user-confirmed managed Docker inventories differ") for identity, record in current.items(): planned = present[identity] if ( record["name"] != planned["name"] or record["attestation"] != planned["attestation"] ): raise SystemExit( "managed Docker residue changed after the user's confirmation") transaction_directory = os.path.dirname(plan_path) transaction_id = os.path.basename(transaction_directory) sealed_plan_digest = plan_digest(sealed_plan) receipt_directory = os.path.join( transaction_directory, "residue-path-deletions") def integer_field(record, name, *, positive=False): value = record.get(name) return ( isinstance(value, int) and not isinstance(value, bool) and (value > 0 if positive else value >= 0) ) all_planned_paths = {} for record in sealed_plan["paths"]: if ( not isinstance(record, dict) or set(record) != { "path", "device", "inode", "mode", "uid", "gid", "type", "parentDevice", "parentInode", "parentMode", } or record.get("path") not in AGENT_ZERO_PATHS or record.get("type") not in {"file", "directory"} or not all(integer_field(record, name, positive=name.endswith("Inode") or name == "inode") for name in ( "device", "inode", "mode", "uid", "gid", "parentDevice", "parentInode", "parentMode", )) or record["uid"] != 0 or record["gid"] != 0 or ( record["type"] == "file" and ( not stat.S_ISREG(record["mode"]) or stat.S_IMODE(record["mode"]) != 0o600 ) ) or ( record["type"] == "directory" and ( not stat.S_ISDIR(record["mode"]) or stat.S_IMODE(record["mode"]) != 0o700 ) ) or not stat.S_ISDIR(record["parentMode"]) or record["parentMode"] & 0o022 ): raise SystemExit( "the uninstall residue plan has an invalid filesystem record") if record["path"] in all_planned_paths: raise SystemExit( "the uninstall residue plan has duplicate filesystem paths") all_planned_paths[record["path"]] = record planned_paths = all_planned_paths if want_agent_zero else {} current_paths = {record["path"]: record for record in path_records} if ( len(current_paths) != len(path_records) or any(path not in planned_paths for path in current_paths) ): raise SystemExit( "the current and user-confirmed managed filesystem inventories differ") def record_digest(record): return hashlib.sha256( canonical_plan(record).encode("utf-8") ).hexdigest() def quarantine_path(record): path = record["path"] name = ( f".{os.path.basename(path)}.bridgesllm-wipe-" f"{transaction_id}-{record_digest(record)[:32]}" ) return os.path.join(os.path.dirname(path), name) def receipt_paths(record): name = hashlib.sha256( record["path"].encode("utf-8") ).hexdigest() + ".json" path = os.path.join(receipt_directory, name) return path, os.path.join(receipt_directory, f".{name}.tmp") def safe_receipt_file(path, *, allow_empty=False): try: info = os.lstat(path) except FileNotFoundError: return False return ( stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode) and info.st_uid == 0 and info.st_gid == 0 and info.st_nlink == 1 and stat.S_IMODE(info.st_mode) == 0o600 and (allow_empty or info.st_size > 0) and info.st_size <= 65536 ) def expected_receipt(record): return { "schema": "bridgesllm.uninstall-path-deletion.v1", "transactionId": transaction_id, "planDigest": sealed_plan_digest, "path": record["path"], "quarantine": quarantine_path(record), "recordDigest": record_digest(record), } def receipt_state(record): receipt, temporary = receipt_paths(record) temporary_exists = os.path.lexists(temporary) if temporary_exists and not safe_receipt_file( temporary, allow_empty=True): raise SystemExit("the path-deletion receipt temporary is unsafe") if not os.path.lexists(receipt): return False, temporary_exists if not safe_receipt_file(receipt): raise SystemExit("the path-deletion receipt is unsafe") try: with open(receipt, "r", encoding="utf-8") as handle: document = json.load(handle) except (OSError, ValueError): raise SystemExit("the path-deletion receipt is invalid") if document != expected_receipt(record): raise SystemExit( "the path-deletion receipt does not match the sealed record") return True, temporary_exists def assert_parent_identity(record): parent = os.path.dirname(record["path"]) info = os.lstat(parent) if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 or os.path.realpath(parent) != parent or info.st_dev != record["parentDevice"] or info.st_ino != record["parentInode"] or info.st_mode != record["parentMode"] ): raise SystemExit( "managed residue parent changed after the user's confirmation") def opened_parent(record): parent = os.path.dirname(record["path"]) flags = ( os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) ) descriptor = os.open(parent, flags) opened = os.fstat(descriptor) current = os.lstat(parent) if ( opened.st_dev != record["parentDevice"] or opened.st_ino != record["parentInode"] or opened.st_mode != record["parentMode"] or opened.st_uid != 0 or opened.st_gid != 0 or (opened.st_dev, opened.st_ino, opened.st_mode) != (current.st_dev, current.st_ino, current.st_mode) ): os.close(descriptor) raise SystemExit( "managed residue parent raced its sealed identity") return descriptor def durably_confirm_path_absence(record, quarantine): parent_fd = opened_parent(record) try: # A prior invocation can die after unlink/rmdir and before its parent # fsync. Seeing the names absent is not yet a crash-durable deletion # proof: flush that exact sealed parent, then re-check both entries # through the bound descriptor before allowing a receipt to adopt the # already-missing state. os.fsync(parent_fd) for candidate in (record["path"], quarantine): try: os.stat( os.path.basename(candidate), dir_fd=parent_fd, follow_symlinks=False, ) except FileNotFoundError: continue raise SystemExit( "managed filesystem residue reappeared while sealing absence") if TEST_ROOT: trace = os.path.join( TEST_ROOT, "state", "managed-path-parent-fsync.log") with open(trace, "a", encoding="utf-8") as handle: handle.write(record["path"] + "\n") finally: os.close(parent_fd) def assert_quarantine_identity(record, quarantine): info = os.lstat(quarantine) if ( stat.S_ISLNK(info.st_mode) or info.st_dev != record["device"] or info.st_ino != record["inode"] or info.st_mode != record["mode"] or info.st_uid != record["uid"] or info.st_gid != record["gid"] or ( record["type"] == "file" and not stat.S_ISREG(info.st_mode) ) or ( record["type"] == "directory" and not stat.S_ISDIR(info.st_mode) ) ): raise SystemExit( "managed residue quarantine does not match its sealed inode") for mountpoint in current_mountpoints(): if ( mountpoint == quarantine or mountpoint.startswith(quarantine + os.sep) ): raise SystemExit( "managed residue quarantine contains a mount boundary") def validate_receipt_directory(): if not os.path.lexists(receipt_directory): return False info = os.lstat(receipt_directory) if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or stat.S_IMODE(info.st_mode) != 0o700 or os.path.realpath(receipt_directory) != receipt_directory ): raise SystemExit("the path-deletion receipt directory is unsafe") allowed = set() for record in planned_paths.values(): allowed.update(receipt_paths(record)) for name in os.listdir(receipt_directory): candidate = os.path.join(receipt_directory, name) if candidate not in allowed: raise SystemExit( "the path-deletion receipt directory contains an unknown entry") return True receipt_directory_exists = validate_receipt_directory() planned_path_states = {} expected_quarantines = { quarantine_path(record) for record in planned_paths.values() } if quarantine_candidates - expected_quarantines: raise SystemExit("an unexpected managed-residue quarantine exists") for path, record in planned_paths.items(): assert_parent_identity(record) quarantine = quarantine_path(record) prefix = f".{os.path.basename(path)}.bridgesllm-wipe-" for name in os.listdir(os.path.dirname(path)): candidate = os.path.join(os.path.dirname(path), name) if name.startswith(prefix) and candidate != quarantine: raise SystemExit( "an unexpected managed-residue quarantine exists") original_exists = os.path.lexists(path) quarantine_exists = os.path.lexists(quarantine) if original_exists and quarantine_exists: raise SystemExit( "managed residue exists at both original and quarantine paths") has_receipt, has_temporary = ( receipt_state(record) if receipt_directory_exists else (False, False) ) if mode == "quiesce": if ( not original_exists or quarantine_exists or has_receipt or has_temporary or current_paths.get(path) != record ): raise SystemExit( "managed filesystem residue changed before Safe cleanup") planned_path_states[path] = { "record": record, "quarantine": quarantine, "state": "original", } continue if mode == "prove-absent": # A receipt proves *this* step deleted the path. It is not the only # honest route to absence: the residue plan is sealed before the # ordinary uninstall steps run, and those steps legitimately remove # some of the same paths (the Agent Zero runtime, its env file, and # its backups). Demanding a receipt for a path an attested earlier # step already removed made Complete wipe unable to converge in the # normal case, leaving a boot fence and a half-uninstalled host. # # Absence with no quarantine and no temporary is still absence: a # partially-moved deletion would have left one of those behind, and # both are still refused below. if ( original_exists or quarantine_exists or has_temporary ): raise SystemExit( "sealed managed filesystem residue is not durably absent") durably_confirm_path_absence(record, quarantine) planned_path_states[path] = { "record": record, "quarantine": quarantine, "state": "deleted", } continue if original_exists: if ( quarantine_exists or has_receipt or has_temporary or current_paths.get(path) != record ): raise SystemExit( "managed filesystem residue changed before Complete wipe") state = "original" elif quarantine_exists: assert_quarantine_identity(record, quarantine) state = "quarantine" elif not has_temporary: # Either this step already deleted it (receipt present), or an earlier # attested uninstall step did. Both end at the same verified absence; # only an interrupted move leaves a quarantine or temporary, and those # are handled above and below. durably_confirm_path_absence(record, quarantine) state = "deleted" else: raise SystemExit( "sealed managed filesystem residue disappeared while a deletion " "temporary was still present") planned_path_states[path] = { "record": record, "quarantine": quarantine, "state": state, } if mode == "prove-absent": for tool, state in firewall.items(): if state["chains"] or state["references"]: raise SystemExit( f"user-confirmed managed {tool} residue remains at terminal cleanup") print("The sealed managed-residue inventory is absent.") raise SystemExit(0) firewall_positions = {} if want_project: for tool in ("iptables", "ip6tables"): planned = sealed_plan["firewall"].get(tool) current = firewall.get(tool, {}).get("currentSnapshot") if ( not isinstance(planned, dict) or set(planned) != {"initial", "mutations", "final"} or not isinstance(planned.get("initial"), str) or not isinstance(planned.get("mutations"), list) or not isinstance(planned.get("final"), str) or current is None ): raise SystemExit("the uninstall residue plan has invalid firewall state") states = [planned["initial"]] for mutation in planned["mutations"]: if ( not isinstance(mutation, dict) or set(mutation) != {"args", "after"} or not isinstance(mutation.get("args"), list) or not all(isinstance(item, str) for item in mutation["args"]) or not isinstance(mutation.get("after"), str) ): raise SystemExit("the uninstall residue plan has an invalid firewall mutation") states.append(mutation["after"]) matches = [index for index, state in enumerate(states) if state == current] if len(matches) != 1: raise SystemExit( "firewall state changed outside the user-confirmed cleanup plan") firewall_positions[tool] = matches[0] if mode == "quiesce": stopped = 0 for record in container_records: current = inspect_resource("container", record["identity"], tolerate_missing=True) if current is None or current["attestation"] != record["attestation"]: raise SystemExit( "a managed container changed before Safe cleanup could stop it") if current["running"]: run([DOCKER, "container", "stop", "--time", "30", record["identity"]]) stopped += 1 verified = inspect_resource("container", record["identity"]) if ( verified is None or verified["attestation"] != record["attestation"] or verified["running"] ): raise SystemExit( "a managed container did not remain stopped for Safe cleanup") print( f"Stopped {stopped} managed leftover container(s); " "Docker objects and host firewall rules were retained.") raise SystemExit(0) if not backup_dir or not os.path.isabs(backup_dir) or os.path.normpath(backup_dir) != backup_dir: raise SystemExit("a firewall backup directory is required before any wipe") BACKUP_BASE = mapped_path("/var/backups/bridgesllm") backup_name = os.path.basename(backup_dir) backup_match = re.fullmatch(r"firewall-wipe-([a-f0-9]{32})", backup_name) if os.path.dirname(backup_dir) != BACKUP_BASE or backup_match is None: raise SystemExit("the firewall backup directory is outside its transaction boundary") transaction_id = backup_match.group(1) if os.path.basename(os.path.dirname(plan_path)) != transaction_id: raise SystemExit("the firewall backup does not match the consent transaction") def fsync_dir(path): descriptor = os.open( path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(descriptor) finally: os.close(descriptor) def safe_root_directory(path, exact_mode=None): try: info = os.lstat(path) except FileNotFoundError: return False return ( stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode) and info.st_uid == 0 and info.st_gid == 0 and not info.st_mode & 0o022 and (exact_mode is None or stat.S_IMODE(info.st_mode) == exact_mode) and os.path.realpath(path) == path ) def safe_backup_file(path): try: info = os.lstat(path) except FileNotFoundError: return False return ( stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode) and info.st_uid == 0 and info.st_gid == 0 and info.st_nlink == 1 and stat.S_IMODE(info.st_mode) == 0o600 ) def write_exclusive(path, payload): descriptor = os.open( path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600, ) with os.fdopen(descriptor, "w", encoding="utf-8") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) def digest(payload): return hashlib.sha256(payload.encode("utf-8")).hexdigest() def validate_sealed_backup(): manifest_path = os.path.join(backup_dir, "manifest.json") if not safe_backup_file(manifest_path): return False try: with open(manifest_path, "r", encoding="utf-8") as handle: manifest = json.load(handle) except (OSError, ValueError): raise SystemExit("the existing firewall backup manifest is invalid") if ( set(manifest) != {"schema", "transactionId", "createdAt", "snapshots"} or manifest.get("schema") != "bridgesllm.firewall-wipe-backup.v1" or manifest.get("transactionId") != transaction_id or not isinstance(manifest.get("createdAt"), str) or set(manifest.get("snapshots", {})) != {"iptables", "ip6tables"} ): raise SystemExit("the existing firewall backup manifest is invalid") for tool in ("iptables", "ip6tables"): record = manifest["snapshots"].get(tool) rules_path = os.path.join(backup_dir, f"{tool}.rules") if ( not isinstance(record, dict) or set(record) != {"bytes", "sha256"} or not isinstance(record.get("bytes"), int) or record["bytes"] < 0 or not isinstance(record.get("sha256"), str) or not re.fullmatch(r"[a-f0-9]{64}", record["sha256"]) or not safe_backup_file(rules_path) ): raise SystemExit("the existing firewall backup is incomplete or unsafe") with open(rules_path, "r", encoding="utf-8") as handle: payload = handle.read(MAX_COMMAND_BYTES + 1) if ( len(payload.encode("utf-8")) != record["bytes"] or digest(payload) != record["sha256"] ): raise SystemExit("the existing firewall backup does not match its manifest") return True for parent in (mapped_path("/var"), mapped_path("/var/backups")): if not safe_root_directory(parent): raise SystemExit("the firewall backup parent is unsafe") if not os.path.lexists(BACKUP_BASE): os.mkdir(BACKUP_BASE, 0o700) fsync_dir(mapped_path("/var/backups")) if not safe_root_directory(BACKUP_BASE): raise SystemExit("the firewall backup root is unsafe") try: os.mkdir(backup_dir, 0o700) fsync_dir(BACKUP_BASE) except FileExistsError: pass if not safe_root_directory(backup_dir, 0o700): raise SystemExit("the firewall backup directory is unsafe") if not validate_sealed_backup(): # A missing manifest is an interrupted pre-delete seal. This version never # mutates Docker, files, or firewall state until the manifest is durable, # so only exact root-owned partial files may be refreshed here. allowed_partial = {"iptables.rules", "ip6tables.rules", "manifest.json"} for name in os.listdir(backup_dir): path = os.path.join(backup_dir, name) if name not in allowed_partial or not safe_backup_file(path): raise SystemExit("the incomplete firewall backup contains an unsafe entry") os.unlink(path) fsync_dir(backup_dir) snapshots = {} for tool in ("iptables", "ip6tables"): saver = trusted_binary(f"{tool}-save") if saver is None: raise SystemExit( f"{tool}-save is unavailable; refusing to touch the firewall without a backup") snapshot = run([saver]) write_exclusive(os.path.join(backup_dir, f"{tool}.rules"), snapshot) snapshots[tool] = { "bytes": len(snapshot.encode("utf-8")), "sha256": digest(snapshot), } manifest = { "schema": "bridgesllm.firewall-wipe-backup.v1", "transactionId": transaction_id, "createdAt": datetime.datetime.now(datetime.timezone.utc) .replace(microsecond=0).isoformat().replace("+00:00", "Z"), "snapshots": snapshots, } write_exclusive( os.path.join(backup_dir, "manifest.json"), json.dumps(manifest, indent=2, sort_keys=True) + "\n", ) fsync_dir(backup_dir) if not validate_sealed_backup(): raise SystemExit("the firewall backup could not be durably sealed") def fixture_path_crash(point): if not TEST_ROOT: return control = os.path.join( TEST_ROOT, "state", "managed-path-crash-point") try: with open(control, "r", encoding="utf-8") as handle: requested = handle.read(128).strip() except FileNotFoundError: return if requested != point: return os.unlink(control) raise SystemExit(f"fixture crash after managed path {point}") def ensure_receipt_directory(): info = os.lstat(transaction_directory) if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or stat.S_IMODE(info.st_mode) != 0o700 or os.path.realpath(transaction_directory) != transaction_directory ): raise SystemExit( "the uninstall transaction directory is unsafe for deletion receipts") if not os.path.lexists(receipt_directory): os.mkdir(receipt_directory, 0o700) fsync_dir(transaction_directory) if not validate_receipt_directory(): raise SystemExit("the path-deletion receipt directory could not be sealed") def write_path_receipt(record): receipt, temporary = receipt_paths(record) has_receipt, has_temporary = receipt_state(record) if has_receipt: if has_temporary: os.unlink(temporary) fsync_dir(receipt_directory) return if has_temporary: os.unlink(temporary) fsync_dir(receipt_directory) payload = json.dumps( expected_receipt(record), indent=2, sort_keys=True) + "\n" descriptor = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600, ) try: with os.fdopen(descriptor, "w", encoding="utf-8") as handle: descriptor = -1 handle.write(payload) handle.flush() os.fsync(handle.fileno()) finally: if descriptor >= 0: os.close(descriptor) if os.path.lexists(receipt): raise SystemExit("the path-deletion receipt raced its atomic publish") os.replace(temporary, receipt) fsync_dir(receipt_directory) has_receipt, has_temporary = receipt_state(record) if not has_receipt or has_temporary: raise SystemExit("the path-deletion receipt was not committed durably") def assert_opened_inode(descriptor, record): opened = os.fstat(descriptor) if ( opened.st_dev != record["device"] or opened.st_ino != record["inode"] or opened.st_mode != record["mode"] or opened.st_uid != record["uid"] or opened.st_gid != record["gid"] ): raise SystemExit( "managed residue inode changed before quarantine") def prepare_path_quarantine(state): record = state["record"] original = record["path"] quarantine = state["quarantine"] if state["state"] == "deleted": return parent_fd = opened_parent(record) try: original_name = os.path.basename(original) quarantine_name = os.path.basename(quarantine) if state["state"] == "original": flags = ( os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0) ) if record["type"] == "directory": flags |= getattr(os, "O_DIRECTORY", 0) inode_fd = os.open( original_name, flags, dir_fd=parent_fd) try: assert_opened_inode(inode_fd, record) try: os.stat( quarantine_name, dir_fd=parent_fd, follow_symlinks=False, ) except FileNotFoundError: pass else: raise SystemExit( "managed residue quarantine appeared before rename") os.rename( original_name, quarantine_name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd, ) moved = os.stat( quarantine_name, dir_fd=parent_fd, follow_symlinks=False, ) if ( moved.st_dev != record["device"] or moved.st_ino != record["inode"] or moved.st_mode != record["mode"] ): raise SystemExit( "managed residue quarantine changed during rename") try: os.stat( original_name, dir_fd=parent_fd, follow_symlinks=False, ) except FileNotFoundError: pass else: raise SystemExit( "managed residue original survived quarantine rename") os.fsync(parent_fd) finally: os.close(inode_fd) state["state"] = "quarantine" fixture_path_crash("quarantine") else: assert_quarantine_identity(record, quarantine) finally: os.close(parent_fd) write_path_receipt(record) fixture_path_crash("receipt") if planned_paths: ensure_receipt_directory() for path in sorted(planned_path_states): prepare_path_quarantine(planned_path_states[path]) def current_firewall_snapshot(binary): return "\n".join( normalize_firewall_line(line) for line in run([binary, "-w", "-S"]).splitlines() if line.strip() ) def validate_firewall_mutation_args(args): if not args or args[0] not in {"-D", "-X"}: return False if args[0] == "-X": return len(args) == 2 and bool( P4E_PROJECT_CHAIN_RE.fullmatch(args[1]) or A0P_PROJECT_CHAIN_RE.fullmatch(args[1]) or args[1] in {P4E_MASTER_CHAIN, P4E_HOST_CHAIN} ) if len(args) < 4 or not re.fullmatch( r"[A-Za-z0-9_.:-]+", args[1]): return False targets = [ args[index + 1] for index, token in enumerate(args[:-1]) if token in {"-j", "-g"} ] managed_parent = bool( P4E_PROJECT_CHAIN_RE.fullmatch(args[1]) or A0P_PROJECT_CHAIN_RE.fullmatch(args[1]) or args[1] in {P4E_MASTER_CHAIN, P4E_HOST_CHAIN} ) managed_target = len(targets) == 1 and bool( P4E_PROJECT_CHAIN_RE.fullmatch(targets[0]) or A0P_PROJECT_CHAIN_RE.fullmatch(targets[0]) or targets[0] in {P4E_MASTER_CHAIN, P4E_HOST_CHAIN} ) return len(targets) == 1 and (managed_parent or managed_target) if want_project: for tool in ("iptables", "ip6tables"): state = firewall[tool] planned = sealed_plan["firewall"][tool] position = firewall_positions[tool] expected = ( planned["initial"] if position == 0 else planned["mutations"][position - 1]["after"] ) for mutation in planned["mutations"][position:]: args = mutation["args"] after = mutation["after"] if not validate_firewall_mutation_args(args): raise SystemExit( "the sealed firewall cleanup plan contains an unsafe command") if current_firewall_snapshot(state["binary"]) != expected: raise SystemExit( "firewall state raced the scoped cleanup operation") run([state["binary"], "-w", *args]) if current_firewall_snapshot(state["binary"]) != after: raise SystemExit( "firewall state did not match the sealed cleanup result") expected = after final = parse_managed_firewall( tool, state["binary"], run([state["binary"], "-w", "-S"]), ) if final["chains"] or final["references"]: raise SystemExit("Portal firewall residue remained after the sealed wipe") removed_count = len(planned["mutations"]) if removed_count: print( f"Applied {removed_count} sealed {tool} cleanup operation(s); " "the original full firewall backup was preserved.") for kind, records in ( ("container", container_records), ("network", network_records), ("volume", volume_records), ): for record in records: current = inspect_resource(kind, record["identity"], tolerate_missing=True) if current is None or current["attestation"] != record["attestation"]: raise SystemExit(f"Docker {kind} changed immediately before deletion") if kind == "container": run([DOCKER, "container", "rm", "--force", record["identity"]]) elif kind == "network": run([DOCKER, "network", "rm", record["identity"]]) else: run([DOCKER, "volume", "rm", record["identity"]]) remaining_identities = docker_identities(kind) if record["identity"] in remaining_identities: raise SystemExit(f"Docker {kind} remained after its sealed deletion") if inspect_resource( kind, record["identity"], tolerate_missing=True ) is not None: raise SystemExit( f"Docker {kind} remained inspectable after its sealed deletion") print(f"Removed Docker {kind} {record['name']}") def secure_remove_quarantine(state): record = state["record"] path = state["quarantine"] has_receipt, has_temporary = receipt_state(record) if not has_receipt or has_temporary: raise SystemExit( "managed residue quarantine lacks its durable deletion receipt") if os.path.lexists(record["path"]): raise SystemExit( "managed residue original reappeared before quarantine deletion") assert_quarantine_identity(record, path) parent_fd = opened_parent(record) quarantine_name = os.path.basename(path) flags = ( os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) ) root_fd = os.open(quarantine_name, flags, dir_fd=parent_fd) try: root_info = os.fstat(root_fd) root_mount_id = descriptor_mount_id(root_fd) if ( root_info.st_dev != record["device"] or root_info.st_ino != record["inode"] or root_info.st_mode != record["mode"] or root_mount_id != descriptor_mount_id(parent_fd) ): raise SystemExit( "managed Agent Zero directory changed mount or inode " "identity immediately before deletion") def remove_contents( parent_fd, parent_path, root_device, root_mount_id ): if descriptor_mount_id(parent_fd) != root_mount_id: raise SystemExit( "managed Agent Zero residue crossed a mount identity") for entry in list(os.scandir(parent_fd)): child_path = os.path.join(parent_path, entry.name) before = os.stat( entry.name, dir_fd=parent_fd, follow_symlinks=False) if stat.S_ISDIR(before.st_mode): if ( before.st_dev != root_device or any( mountpoint == child_path or mountpoint.startswith(child_path + os.sep) for mountpoint in current_mountpoints() ) ): raise SystemExit( "managed Agent Zero residue gained a mount boundary") child_fd = os.open( entry.name, flags, dir_fd=parent_fd) try: opened = os.fstat(child_fd) if ( (opened.st_dev, opened.st_ino) != ( before.st_dev, before.st_ino ) or descriptor_mount_id(child_fd) != root_mount_id ): raise SystemExit( "managed residue directory raced or crossed " "a mount during deletion") remove_contents( child_fd, child_path, root_device, root_mount_id, ) finally: os.close(child_fd) after = os.stat( entry.name, dir_fd=parent_fd, follow_symlinks=False) if (after.st_dev, after.st_ino) != ( before.st_dev, before.st_ino ): raise SystemExit( "managed residue directory raced deletion") os.rmdir(entry.name, dir_fd=parent_fd) fixture_path_crash("mid-delete") else: after = os.stat( entry.name, dir_fd=parent_fd, follow_symlinks=False) if (after.st_dev, after.st_ino, after.st_mode) != ( before.st_dev, before.st_ino, before.st_mode ): raise SystemExit("managed residue entry raced deletion") os.unlink(entry.name, dir_fd=parent_fd) fixture_path_crash("mid-delete") remove_contents( root_fd, path, root_info.st_dev, root_mount_id) rebound = os.stat( quarantine_name, dir_fd=parent_fd, follow_symlinks=False, ) if ( rebound.st_dev != record["device"] or rebound.st_ino != record["inode"] or rebound.st_mode != record["mode"] ): raise SystemExit( "managed residue quarantine changed during deletion") os.rmdir(quarantine_name, dir_fd=parent_fd) fixture_path_crash("directory-root-unlink") os.fsync(parent_fd) finally: os.close(root_fd) os.close(parent_fd) def secure_remove_quarantined_file(state): record = state["record"] quarantine = state["quarantine"] has_receipt, has_temporary = receipt_state(record) if not has_receipt or has_temporary: raise SystemExit( "managed residue quarantine lacks its durable deletion receipt") if os.path.lexists(record["path"]): raise SystemExit( "managed residue original reappeared before quarantine deletion") assert_quarantine_identity(record, quarantine) parent_fd = opened_parent(record) try: name = os.path.basename(quarantine) before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) if ( before.st_dev != record["device"] or before.st_ino != record["inode"] or before.st_mode != record["mode"] or before.st_nlink != 1 ): raise SystemExit( "managed residue file changed before quarantine deletion") os.unlink(name, dir_fd=parent_fd) fixture_path_crash("file-root-unlink") os.fsync(parent_fd) fixture_path_crash("mid-delete") finally: os.close(parent_fd) for path in sorted(planned_path_states): state = planned_path_states[path] if state["state"] == "deleted": continue if state["state"] != "quarantine": raise SystemExit( "managed filesystem residue was not quarantined before deletion") if state["record"]["type"] == "directory": secure_remove_quarantine(state) else: secure_remove_quarantined_file(state) if ( os.path.lexists(state["record"]["path"]) or os.path.lexists(state["quarantine"]) ): raise SystemExit( "managed filesystem residue remained after quarantine deletion") state["state"] = "deleted" print(f"Removed {state['record']['path']}") PY } resolve_uninstall_residue_policy() { local policy policy="$(read_uninstall_transaction_field residuePolicy 2>/dev/null || true)" case "${policy}" in wipe) printf 'wipe' ;; *) printf 'safe' ;; esac } # Resolve leftover managed runtime residue on the helperless repeat clean-slate # path according to the policy the user confirmed: "safe" finishes the # uninstall and leaves the residue untouched (the host firewall is never # edited), "wipe" removes exactly the recognized product shapes after a full # firewall backup and then re-proves absence through the fail-closed asserts. handle_helperless_clean_slate_residue() { local scope="$1" local report first_line policy transaction_id transaction_dir plan_path backup_dir policy="$(resolve_uninstall_residue_policy)" transaction_id="$(read_uninstall_transaction_field transactionId)" \ || fail "The uninstall transaction id could not be read before residue handling." transaction_dir="$(read_uninstall_transaction_field transactionDir)" \ || fail "The uninstall transaction directory could not be read before residue handling." plan_path="$(read_uninstall_transaction_field residuePlan)" \ || fail "The uninstall residue plan could not be read." [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ \ && "${transaction_dir}" == "${UNINSTALL_TRANSACTIONS_ROOT}/${transaction_id}" \ && "${plan_path}" == "${transaction_dir}/residue-plan.json" ]] \ || fail "The uninstall residue plan escaped its exact transaction boundary." if [[ "${policy}" == "wipe" ]]; then backup_dir="/var/backups/bridgesllm/firewall-wipe-${transaction_id}" info "Complete wipe: removing leftover managed runtime residue (firewall backup: ${backup_dir})..." managed_runtime_residue_tool wipe "${scope}" "${backup_dir}" "${plan_path}" \ || fail "Leftover managed runtime residue could not be removed completely; clean-slate uninstall was aborted. The firewall backup (if written) is at ${backup_dir}." case "${scope}" in project) assert_no_managed_project_runtime_residuals ;; agent-zero) assert_no_managed_agent_zero_runtime_residuals ;; all) assert_no_managed_project_runtime_residuals assert_no_managed_agent_zero_runtime_residuals ;; esac if [[ -d "${backup_dir}" && ! -L "${backup_dir}" ]]; then UNINSTALL_RESIDUE_WIPE_BACKUP_DIR="${backup_dir}" ok "Leftover managed runtime residue removed; firewall backup kept at ${backup_dir}" else info "Managed runtime residue was already absent; no firewall backup was needed." fi # Absence is confirmed at each recorded location, not proven host-wide. Say # so plainly rather than let "Complete wipe" imply more than it verifies. info "Complete wipe confirms each managed path is gone from the location it was recorded at." info "A managed path copied or moved elsewhere before this uninstall is not tracked, and that copy remains. Review this host directly if it held sensitive data." else report="$(managed_runtime_residue_tool report "${scope}")" \ || fail "Managed runtime residue could not be authoritatively inspected; clean-slate uninstall was aborted." first_line="$(head -n 1 <<<"${report}")" if [[ "${first_line}" == present=false* ]]; then info "Managed runtime residue is already absent; the cleanup helper is no longer required." return 0 fi [[ "${first_line}" == present=true* ]] \ || fail "Managed runtime residue inspection returned an unreadable result; clean-slate uninstall was aborted." managed_runtime_residue_tool quiesce "${scope}" "" "${plan_path}" \ || fail "Safe cleanup could not stop and verify every confirmed managed container; clean-slate uninstall was aborted before deleting data." warn "Safe cleanup: leftover managed runtime residue stays on this server (chosen at confirmation):" tail -n +2 <<<"${report}" | sed 's/^/ /' warn "Re-run the installer with --uninstall and choose Complete wipe to remove it later." fi } attest_clean_slate_tree() { local path="$1" local expected_path="$2" local contract="${3:-generic}" local related_root="${4:-}" local mountinfo_file="${5:-/proc/self/mountinfo}" python3 - "${path}" "${expected_path}" "${contract}" "${related_root}" "${mountinfo_file}" <<'PY' import os import re import stat import sys path, expected, contract, related, mountinfo_path = sys.argv[1:] for value in (path, expected): if not os.path.isabs(value) or value != os.path.normpath(value) or value == os.path.sep: raise SystemExit("Clean-slate path must be a bounded canonical absolute path") if path != expected: raise SystemExit("Clean-slate path does not match its exact ownership boundary") current = os.path.sep parts = path.strip(os.path.sep).split(os.path.sep) for index, component in enumerate(parts): current = os.path.join(current, component) if not os.path.lexists(current): print("absent") raise SystemExit(0) info = os.lstat(current) if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) or info.st_uid != 0: raise SystemExit("Clean-slate path crosses an unsafe ownership boundary") if info.st_mode & 0o022: if index == len(parts) - 1 or not info.st_mode & stat.S_ISVTX: raise SystemExit("Clean-slate path crosses a writable ownership boundary") if os.path.realpath(path) != path: raise SystemExit("Clean-slate path does not resolve to its exact boundary") def decode_mount(value: str) -> str: return (value.replace("\\040", " ").replace("\\011", "\t") .replace("\\012", "\n").replace("\\134", "\\")) try: with open(mountinfo_path, "r", encoding="utf-8") as handle: mountpoints = [decode_mount(line.split()[4]) for line in handle if len(line.split()) >= 5] except OSError as error: raise SystemExit(f"Mount topology could not be inspected: {error}") for mountpoint in mountpoints: if mountpoint == path or mountpoint.startswith(path + os.sep): raise SystemExit("Clean-slate path contains a mount or bind-mount boundary") def owned_dir(candidate: str) -> bool: try: info = os.lstat(candidate) except FileNotFoundError: return False return stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o022 def owned_file(candidate: str) -> bool: try: info = os.lstat(candidate) except FileNotFoundError: return False return stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 and not info.st_mode & 0o022 def safe_owned_root(candidate: str) -> bool: if not candidate or not os.path.isabs(candidate) or candidate != os.path.normpath(candidate): return False current = os.path.sep components = candidate.strip(os.path.sep).split(os.path.sep) for index, component in enumerate(components): current = os.path.join(current, component) try: info = os.lstat(current) except FileNotFoundError: return False if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) or info.st_uid != 0: return False if info.st_mode & 0o022 and (index == len(components) - 1 or not info.st_mode & stat.S_ISVTX): return False return os.path.realpath(candidate) == candidate if contract == "portal": # A legacy Portal root is attributed by owned, Portal-specific directory # markers. `projects` is the usual one, but a legacy tree that never had a # Project created still contains the other Portal-owned roots. Requiring # `projects` unconditionally made clean-slate uninstall abort outright on # such a tree -- and it aborted *after* the database had already been # dropped, leaving the host neither working nor uninstalled. # # Accept either marker set. Both are equally Portal-specific, so this does # not widen attribution to a foreign /portal: an unrelated directory of # that name still has none of these owned subdirectories and is still # refused. # A legacy Portal root is attributed by owned, Portal-specific markers. # Two distinct shapes exist in the wild and BOTH must be accepted: # # - a runtime tree, identified by owned `.data` or by signed # backend/frontend package identities; and # - a data-only tree left by older installs, which has none of those but # does have Portal's own data roots (app-zips, apps, project-zips). # # Requiring the runtime markers unconditionally made clean-slate uninstall # abort on the data-only shape -- the common case on an upgraded host -- # exactly as requiring `projects` did. Either shape is sufficient identity; # a foreign directory called /portal has neither and is still refused. portal_data_markers = [ name for name in ("projects", "app-zips", "project-zips", "apps") if owned_dir(os.path.join(path, name)) ] attributed_by_data = ( "projects" in portal_data_markers or len(portal_data_markers) >= 2 ) attributed_by_runtime = owned_dir(os.path.join(path, ".data")) if not attributed_by_runtime: backend_package = os.path.join(path, "backend", "package.json") frontend_package = os.path.join(path, "frontend", "package.json") if owned_file(backend_package) and owned_file(frontend_package): import json try: with open(backend_package, "r", encoding="utf-8") as handle: backend = json.load(handle) with open(frontend_package, "r", encoding="utf-8") as handle: frontend = json.load(handle) except (OSError, ValueError): raise SystemExit("Legacy Portal package identity markers are unreadable") if (backend.get("name") != "bridgesllm-backend" or frontend.get("name") != "bridgesllm-frontend"): raise SystemExit("Legacy Portal package identity markers do not match") attributed_by_runtime = True if not attributed_by_data and not attributed_by_runtime: raise SystemExit("Legacy Portal root lacks any owned Portal identity marker") elif contract == "apps": if not safe_owned_root(related) or not owned_dir(os.path.join(related, "projects")): raise SystemExit("Legacy app root cannot be tied to an owned Portal projects root") uuid_prefix = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}-") for name in os.listdir(path): candidate = os.path.join(path, name) if not uuid_prefix.match(name) or not owned_dir(candidate): raise SystemExit("Legacy app root contains an unattributable entry") owner_id = name[:36] if not owned_dir(os.path.join(related, "projects", owner_id)): raise SystemExit("Legacy app entry has no matching owned Portal project identity") elif contract == "portal-files": modern_marker = os.path.join(related, "portal", "installer", "install.sh") if related else "" legacy_projects = os.path.join(related, "projects") if related else "" legacy_state = os.path.join(related, ".data") if related else "" if not safe_owned_root(related) or not ((modern_marker and owned_file(modern_marker)) or (legacy_projects and owned_dir(legacy_projects) and owned_dir(legacy_state))): raise SystemExit("Portal file root cannot be tied to an owned Portal installation") elif contract == "stalwart-legacy-store": marker = os.path.join(path, "data", "IDENTITY") if not owned_file(marker): raise SystemExit("Legacy Stalwart store lacks its owned identity marker") with open(marker, "r", encoding="ascii") as handle: identity = handle.read(128).strip() if not re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}", identity): raise SystemExit("Legacy Stalwart identity marker is malformed") elif contract == "stalwart-legacy-mail": marker = os.path.join(path, "etc", "config.toml") if not owned_file(marker): raise SystemExit("Legacy Stalwart mail root lacks its owned config marker") with open(marker, "r", encoding="utf-8") as handle: first_line = handle.readline().rstrip("\r\n") if first_line not in {"# Stalwart Mail Server - BridgesLLM", "# Stalwart Mail Server — BridgesLLM Portal"}: raise SystemExit("Legacy Stalwart config marker is not Portal-authored") elif contract == "install-root": marker = os.path.join(path, "portal", "installer", "install.sh") if not owned_file(marker): raise SystemExit("Install root lacks the Portal installer ownership marker") elif contract != "generic": raise SystemExit("Unknown clean-slate ownership contract") info = os.lstat(path) print(f"{info.st_dev}:{info.st_ino}") PY } remove_attested_clean_slate_tree() { local path="$1" local expected_path="$2" local contract="${3:-generic}" local related_root="${4:-}" local mountinfo_file="${5:-/proc/self/mountinfo}" local attestation attestation="$(attest_clean_slate_tree "${path}" "${expected_path}" "${contract}" "${related_root}" "${mountinfo_file}")" \ || return 1 [[ "${attestation}" != "absent" ]] || return 0 rm -rf --one-file-system -- "${path}" || return 1 [[ ! -e "${path}" && ! -L "${path}" ]] } # Prove every legacy tree can be attributed BEFORE anything is destroyed. # # Clean-slate uninstall is a phase machine whose destructive steps (Agent Zero, # Stalwart, the Portal database and role) all run before legacy attribution. # When attribution failed, the database was already gone and the host was left # neither working nor uninstalled -- and re-running could not finish, because # the same gate failed again with nothing left to roll back to. # # This runs the identical attestations read-only, up front, so an # unattributable tree fails while the system is still intact and the operator # still has a working Portal. assert_clean_slate_legacy_attribution() { local legacy_portal_root="${1:-/portal}" local legacy_apps_root="${2:-/var/www/bridgesllm-apps}" local portal_files_root="${3:-/var/portal-files}" local related_install_root="${4:-${INSTALL_ROOT}}" if [[ -e "${legacy_apps_root}" || -L "${legacy_apps_root}" ]]; then attest_clean_slate_tree \ "${legacy_apps_root}" "${legacy_apps_root}" apps "${legacy_portal_root}" >/dev/null \ || fail "Legacy hosted-app data cannot be attributed. Nothing was removed. Resolve this before uninstalling." fi if [[ -e "${portal_files_root}" || -L "${portal_files_root}" ]]; then if ! attest_clean_slate_tree \ "${portal_files_root}" "${portal_files_root}" portal-files "${related_install_root}" >/dev/null 2>&1 \ && ! attest_clean_slate_tree \ "${portal_files_root}" "${portal_files_root}" portal-files "${legacy_portal_root}" >/dev/null 2>&1; then fail "Portal upload data at ${portal_files_root} cannot be attributed to this Portal, so nothing was removed. Inspect that path and move or delete it if it is not Portal data." fi fi if [[ -e "${legacy_portal_root}" || -L "${legacy_portal_root}" ]]; then attest_clean_slate_tree \ "${legacy_portal_root}" "${legacy_portal_root}" portal "${related_install_root}" >/dev/null \ || fail "Legacy Portal data at ${legacy_portal_root} cannot be attributed to this Portal, so nothing was removed. Inspect that path: if it is not Portal data, move or delete it and retry; a stray ${legacy_portal_root} can be created by running Portal tooling without PORTAL_DATA_ROOT set." fi return 0 } remove_legacy_portal_data_for_clean_uninstall() { local legacy_portal_root="${1:-/portal}" local legacy_apps_root="${2:-/var/www/bridgesllm-apps}" local portal_files_root="${3:-/var/portal-files}" local expected_legacy_portal_root="${4:-/portal}" local expected_legacy_apps_root="${5:-/var/www/bridgesllm-apps}" local expected_portal_files_root="${6:-/var/portal-files}" local related_install_root="${7:-${INSTALL_ROOT}}" # App ownership is tied to user/project identities in the legacy Portal # root, so it must be checked and removed before that root disappears. if [[ -e "${legacy_apps_root}" || -L "${legacy_apps_root}" ]]; then remove_attested_clean_slate_tree \ "${legacy_apps_root}" "${expected_legacy_apps_root}" apps "${legacy_portal_root}" \ || fail "Legacy hosted-app data could not be attributed and removed safely; clean-slate uninstall was aborted." fi if [[ -e "${portal_files_root}" || -L "${portal_files_root}" ]]; then if remove_attested_clean_slate_tree \ "${portal_files_root}" "${expected_portal_files_root}" portal-files "${related_install_root}" 2>/dev/null; then : elif remove_attested_clean_slate_tree \ "${portal_files_root}" "${expected_portal_files_root}" portal-files "${legacy_portal_root}"; then : else fail "Portal upload data could not be attributed and removed safely; clean-slate uninstall was aborted." fi fi if [[ -e "${legacy_portal_root}" || -L "${legacy_portal_root}" ]]; then remove_attested_clean_slate_tree \ "${legacy_portal_root}" "${expected_legacy_portal_root}" portal \ || fail "Legacy Portal data could not be attributed and removed safely; clean-slate uninstall was aborted." fi } remove_install_root_for_clean_uninstall() { local install_root="${1:-${INSTALL_ROOT}}" local expected_install_root="${2:-${INSTALL_ROOT}}" if [[ ! -f "${install_root}/portal/installer/install.sh" \ && ( -e "${install_root}/.retained-install-v1.json" \ || -L "${install_root}/.retained-install-v1.json" ) ]]; then verify_retained_install_receipt \ "${install_root}/portal" \ "${install_root}/.retained-install-v1.json" \ "${install_root}/.retained-install-tree-v1.json" \ || fail "The retained install receipt no longer matches its exact Portal tree; clean-slate removal was paused." remove_attested_clean_slate_tree \ "${install_root}" "${expected_install_root}" generic \ || fail "The retained Portal install root could not be removed safely." return 0 fi remove_attested_clean_slate_tree "${install_root}" "${expected_install_root}" install-root \ || fail "Portal install root could not be attributed and removed safely; clean-slate uninstall was aborted." } attest_stalwart_compose_tree() { local stalwart_dir="$1" local expected_stalwart_dir="$2" local tree_attestation tree_attestation="$(attest_clean_slate_tree "${stalwart_dir}" "${expected_stalwart_dir}" generic)" \ || return 1 [[ "${tree_attestation}" != "absent" ]] || return 1 python3 - "${stalwart_dir}" <<'PY' import os import stat import sys root = sys.argv[1] compose = os.path.join(root, "docker-compose.yml") expected = """version: '3.8' services: stalwart: image: stalwartlabs/stalwart:v0.15.5 container_name: stalwart-mail restart: unless-stopped ports: - "25:25" - "587:587" - "993:993" - "127.0.0.1:8580:8080" volumes: - ./data:/opt/stalwart """ info = os.lstat(compose) if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 or info.st_size > 4096: raise SystemExit("Managed Stalwart compose file is unsafe") with open(compose, "r", encoding="utf-8") as handle: if handle.read(4097) != expected: raise SystemExit("Managed Stalwart compose content has drifted") marker = os.path.join(root, "data", "etc", "config.toml") marker_info = os.lstat(marker) if not stat.S_ISREG(marker_info.st_mode) or stat.S_ISLNK(marker_info.st_mode) or marker_info.st_uid != 0 or marker_info.st_nlink != 1 or marker_info.st_mode & 0o022: raise SystemExit("Managed Stalwart data marker is unsafe") with open(marker, "r", encoding="utf-8") as handle: if handle.readline().rstrip("\r\n") != "# Stalwart Mail Server — BridgesLLM Portal": raise SystemExit("Managed Stalwart data marker is not Portal-authored") PY } attest_stalwart_container_contract() { local inspect_json="$1" local mode="$2" local expected_source="$3" printf '%s' "${inspect_json}" | python3 /dev/fd/3 "${mode}" "${expected_source}" 3<<'PY' import json import os import sys mode, expected_source = sys.argv[1:] try: payload = json.load(sys.stdin) except (ValueError, TypeError): raise SystemExit("Stalwart container inspection was not valid JSON") if isinstance(payload, list): if len(payload) != 1: raise SystemExit("Stalwart inspection returned an ambiguous container set") payload = payload[0] if not isinstance(payload, dict) or payload.get("Name") != "/stalwart-mail": raise SystemExit("Stalwart container identity does not match") config = payload.get("Config") or {} host = payload.get("HostConfig") or {} expected_image = "stalwartlabs/stalwart:v0.15.5" if mode == "modern" else "stalwartlabs/stalwart:latest" if config.get("Image") != expected_image or host.get("Privileged") is not False: raise SystemExit("Stalwart image or privilege contract does not match") restart = host.get("RestartPolicy") or {} if restart.get("Name") != "unless-stopped" or restart.get("MaximumRetryCount") not in (0, None): raise SystemExit("Stalwart restart contract does not match") expected_ports = { "25/tcp": [{"HostIp": "", "HostPort": "25"}], "587/tcp": [{"HostIp": "", "HostPort": "587"}], "993/tcp": [{"HostIp": "", "HostPort": "993"}], "8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": "8580"}], } if host.get("PortBindings") != expected_ports: raise SystemExit("Stalwart port contract does not match") mounts = payload.get("Mounts") or [] if len(mounts) != 1: raise SystemExit("Stalwart mount contract is ambiguous") mount = mounts[0] if (mount.get("Type") != "bind" or os.path.normpath(mount.get("Source", "")) != expected_source or mount.get("Destination") != "/opt/stalwart" or mount.get("RW") is not True): raise SystemExit("Stalwart data mount contract does not match") state = payload.get("State") or {} print("running" if state.get("Running") is True else "stopped") PY } stop_managed_stalwart_for_clean_uninstall() { local stalwart_dir="${1:-${INSTALL_ROOT}/stalwart}" local expected_stalwart_dir="${2:-${INSTALL_ROOT}/stalwart}" local legacy_store="${3:-/var/stalwart}" local legacy_mail="${4:-/var/stalwart-mail}" local expected_legacy_store="${5:-/var/stalwart}" local expected_legacy_mail="${6:-/var/stalwart-mail}" local stalwart_container_names="" stalwart_container_names="$(docker container ls --all --format '{{.Names}}' 2>/dev/null)" \ || fail "Docker state could not be inspected before clean-slate uninstall." local mode="legacy" local expected_source="${expected_legacy_store}" if [[ -e "${stalwart_dir}" || -L "${stalwart_dir}" ]]; then attest_stalwart_compose_tree "${stalwart_dir}" "${expected_stalwart_dir}" \ || fail "Managed Stalwart compose/data ownership could not be attested; clean-slate uninstall was aborted." mode="modern" expected_source="${expected_stalwart_dir}/data" fi if grep -Fx 'stalwart-mail' <<< "${stalwart_container_names}" >/dev/null; then if [[ "${mode}" == "legacy" ]]; then [[ "$(attest_clean_slate_tree "${legacy_store}" "${expected_legacy_store}" stalwart-legacy-store)" != "absent" ]] \ || fail "Legacy Stalwart store ownership could not be attested; clean-slate uninstall was aborted." [[ "$(attest_clean_slate_tree "${legacy_mail}" "${expected_legacy_mail}" stalwart-legacy-mail)" != "absent" ]] \ || fail "Legacy Stalwart mail ownership could not be attested; clean-slate uninstall was aborted." fi local inspect_json container_state inspect_json="$(docker container inspect stalwart-mail 2>/dev/null)" \ || fail "The stalwart-mail container could not be inspected; clean-slate uninstall was aborted." container_state="$(attest_stalwart_container_contract "${inspect_json}" "${mode}" "${expected_source}")" \ || fail "The stalwart-mail container does not match the Portal ownership contract; clean-slate uninstall was aborted." info "Stopping mail server..." if [[ "${container_state}" == "running" ]]; then docker container stop --time 30 stalwart-mail >/dev/null 2>&1 \ || fail "Managed Stalwart mail could not be stopped; clean-slate uninstall was aborted." fi docker container rm stalwart-mail >/dev/null 2>&1 \ || fail "Managed Stalwart mail container could not be removed; clean-slate uninstall was aborted." fi stalwart_container_names="$(docker container ls --all --format '{{.Names}}' 2>/dev/null)" \ || fail "Docker state could not be re-inspected after Stalwart cleanup." grep -Fx 'stalwart-mail' <<< "${stalwart_container_names}" >/dev/null \ && fail "The managed stalwart-mail container still exists; clean-slate uninstall was aborted." # These are the two supported legacy stores. They are removed only after the # exact legacy container is absent and each root independently proves Portal # ownership. Modern installs keep their data under INSTALL_ROOT. if [[ -e "${legacy_store}" || -L "${legacy_store}" ]]; then remove_attested_clean_slate_tree "${legacy_store}" "${expected_legacy_store}" stalwart-legacy-store \ || fail "Legacy Stalwart store could not be removed safely; clean-slate uninstall was aborted." fi if [[ -e "${legacy_mail}" || -L "${legacy_mail}" ]]; then remove_attested_clean_slate_tree "${legacy_mail}" "${expected_legacy_mail}" stalwart-legacy-mail \ || fail "Legacy Stalwart mail root could not be removed safely; clean-slate uninstall was aborted." fi } is_installer_default_database_url() { local database_url="${1:-}" [[ -n "${database_url}" ]] || return 1 # Parse through the same credential-safe helper used by migrations and # backups. Percent-encoding and non-identity libpq options are valid, but an # alternate scheme, endpoint, account, database, missing password, fragment, # or query-string identity override is not the installer-owned fallback. [[ "$(pg_url_component "${database_url}" host 2>/dev/null)" == '127.0.0.1' \ && "$(pg_url_component "${database_url}" port 2>/dev/null)" == '5432' \ && "$(pg_url_component "${database_url}" database 2>/dev/null)" == 'bridgesllm_portal' \ && "$(pg_url_component "${database_url}" user 2>/dev/null)" == 'blp' ]] \ || return 1 pg_url_component "${database_url}" password >/dev/null 2>&1 || return 1 printf '%s' "${database_url}" | python3 /dev/fd/3 3<<'PY2' import sys from urllib.parse import urlsplit raw = sys.stdin.read() try: parsed = urlsplit(raw) except ValueError: raise SystemExit(1) raise SystemExit(0 if parsed.scheme == "postgresql" else 1) PY2 } default_portal_database_catalog_is_exclusive() { # The URL proves only the configured endpoint/name. It does not prove that a # pre-existing role/database was created for this Portal. Require the target # database to be owned by blp and reject every blp-owned database or shared # dependency outside that target before issuing any terminating or DROP SQL. local owner_match other_database_count external_dependency_count owner_match="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba WHERE d.datname = 'bridgesllm_portal' AND r.rolname = 'blp';")" \ || return 1 [[ "${owner_match//[[:space:]]/}" == '1' ]] || return 1 other_database_count="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT count(*) FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba WHERE r.rolname = 'blp' AND d.datname <> 'bridgesllm_portal';")" \ || return 1 [[ "${other_database_count//[[:space:]]/}" == '0' ]] || return 1 external_dependency_count="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="WITH target AS (SELECT oid FROM pg_database WHERE datname = 'bridgesllm_portal'), portal_role AS (SELECT oid FROM pg_roles WHERE rolname = 'blp') SELECT count(*) FROM pg_shdepend dep CROSS JOIN target CROSS JOIN portal_role WHERE dep.refclassid = 'pg_authid'::regclass AND dep.refobjid = portal_role.oid AND NOT (dep.dbid = target.oid OR (dep.dbid = 0 AND dep.classid = 'pg_database'::regclass AND dep.objid = target.oid));")" \ || return 1 [[ "${external_dependency_count//[[:space:]]/}" == '0' ]] } remove_default_portal_database_for_clean_uninstall() { info "Removing default local Portal database..." sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'bridgesllm_portal' AND pid <> pg_backend_pid();" >/dev/null \ || fail "Default local Portal database connections could not be stopped; clean-slate uninstall was aborted." sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --command='DROP DATABASE IF EXISTS bridgesllm_portal;' >/dev/null \ || fail "Default local Portal database could not be removed; clean-slate uninstall was aborted." local remaining_role_dependencies="" remaining_role_dependencies="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT count(*) FROM pg_shdepend dep JOIN pg_roles r ON dep.refclassid = 'pg_authid'::regclass AND dep.refobjid = r.oid WHERE r.rolname = 'blp';")" \ || fail "Default local Portal role dependencies could not be re-inspected after database removal." [[ "${remaining_role_dependencies//[[:space:]]/}" == '0' ]] \ || fail "The blp role gained or retained external dependencies; role removal was aborted." sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --command='DROP ROLE IF EXISTS blp;' >/dev/null \ || fail "Default local Portal database role could not be removed; clean-slate uninstall was aborted." local database_present="" database_present="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT 1 FROM pg_database WHERE datname = 'bridgesllm_portal';")" \ || fail "Default local Portal database removal could not be verified." [[ -z "${database_present//[[:space:]]/}" ]] \ || fail "Default local Portal database still exists; clean-slate uninstall was aborted." local role_present="" role_present="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT 1 FROM pg_roles WHERE rolname = 'blp';")" \ || fail "Default local Portal database role removal could not be verified." [[ -z "${role_present//[[:space:]]/}" ]] \ || fail "Default local Portal database role still exists; clean-slate uninstall was aborted." ok "Default local Portal database and role removed" } remove_configured_default_database_for_clean_uninstall() { local configured_url="${1:-}" if ! is_installer_default_database_url "${configured_url}"; then warn "The configured database is missing, unreadable, or not the exact installer-owned local default; local PostgreSQL databases and roles were preserved." return 0 fi command -v psql >/dev/null 2>&1 \ || fail "The exact installer-owned database is configured, but PostgreSQL tools are unavailable; clean-slate uninstall was paused before forgetting its credentials." local database_present role_present database_present="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT count(*) FROM pg_database WHERE datname = 'bridgesllm_portal';")" \ || fail "The configured local Portal database could not be inspected; clean-slate uninstall was paused." role_present="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT count(*) FROM pg_roles WHERE rolname = 'blp';")" \ || fail "The configured local Portal role could not be inspected; clean-slate uninstall was paused." database_present="${database_present//[[:space:]]/}" role_present="${role_present//[[:space:]]/}" [[ "${database_present}" == "0" || "${database_present}" == "1" ]] \ || fail "The local Portal database catalog returned an invalid identity count." [[ "${role_present}" == "0" || "${role_present}" == "1" ]] \ || fail "The local Portal role catalog returned an invalid identity count." if [[ "${database_present}" == "0" && "${role_present}" == "0" ]]; then ok "Default local Portal database and role are already absent" return 0 fi if [[ "${database_present}" == "1" && "${role_present}" == "1" ]]; then default_portal_database_catalog_is_exclusive \ || fail "The local blp role/database is externally owned, shared, or ambiguous; clean-slate uninstall was paused before deleting it." remove_default_portal_database_for_clean_uninstall return 0 fi if [[ "${database_present}" == "0" && "${role_present}" == "1" ]]; then local remaining_role_dependencies remaining_role_dependencies="$(sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --tuples-only --no-align \ --command="SELECT count(*) FROM pg_shdepend dep JOIN pg_roles r ON dep.refclassid = 'pg_authid'::regclass AND dep.refobjid = r.oid WHERE r.rolname = 'blp';")" \ || fail "The partially removed Portal role could not be re-inspected." [[ "${remaining_role_dependencies//[[:space:]]/}" == "0" ]] \ || fail "The partially removed blp role has external dependencies; clean-slate uninstall was paused." sudo -u postgres psql --no-psqlrc --dbname=postgres --set=ON_ERROR_STOP=1 \ --command='DROP ROLE IF EXISTS blp;' >/dev/null \ || fail "The partially removed Portal role could not be removed." ok "Partially removed default local Portal database state converged" return 0 fi fail "The default Portal database exists without its expected owner role; clean-slate uninstall was paused rather than guessing ownership." } prepare_uninstall_transaction() { local mode="$1" local env_file="${2:-${PORTAL_DIR}/backend/.env.production}" local residue_policy="${3:-safe}" [[ "${mode}" == "keep" || "${mode}" == "clean" ]] || return 1 [[ "${residue_policy}" == "safe" || "${residue_policy}" == "wipe" ]] || return 1 local transaction_id transaction_dir env_snapshot="" residue_plan="" transaction_id="$(openssl rand -hex 16)" || return 1 [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 transaction_dir="${UNINSTALL_TRANSACTIONS_ROOT}/${transaction_id}" [[ ! -e "${UNINSTALL_ACTIVE_JOURNAL}" && ! -L "${UNINSTALL_ACTIVE_JOURNAL}" ]] \ || return 1 # The shared journal root is named explicitly. `install -d` applies -m only # to the last component it creates, so letting it appear implicitly as a # parent leaves it 0755 -- which later fails the update admission contract # and makes every future update on this host refuse to start. install -d -m 700 -o root -g root \ "${UPDATE_STATE_ROOT}" \ "${UNINSTALL_STATE_ROOT}" "${UNINSTALL_TRANSACTIONS_ROOT}" "${transaction_dir}" \ || return 1 if [[ -e "${env_file}" || -L "${env_file}" ]]; then [[ -f "${env_file}" && ! -L "${env_file}" ]] || return 1 assert_env_file_no_duplicate_keys "${env_file}" || return 1 env_snapshot="${transaction_dir}/portal.env" install -m 600 -o root -g root -- "${env_file}" "${env_snapshot}" || return 1 cmp -s -- "${env_file}" "${env_snapshot}" || return 1 fi if [[ "${mode}" == "clean" ]]; then [[ "${UNINSTALL_RESIDUE_PLAN_DIGEST}" =~ ^[a-f0-9]{64}$ ]] || return 1 residue_plan="${transaction_dir}/residue-plan.json" if ! managed_runtime_residue_tool seal-plan all "" \ "${residue_plan}" "${UNINSTALL_RESIDUE_PLAN_DIGEST}"; then rm -rf --one-file-system -- "${transaction_dir}" >/dev/null 2>&1 || true fsync_uninstall_directory "${UNINSTALL_TRANSACTIONS_ROOT}" >/dev/null 2>&1 || true return 1 fi fi python3 - "${UNINSTALL_ACTIVE_JOURNAL}" "${transaction_id}" "${mode}" \ "${transaction_dir}" "${env_snapshot}" "${VERSION}" "${residue_policy}" \ "${residue_plan}" "${UNINSTALL_RESIDUE_PLAN_DIGEST}" <<'PY' import datetime import json import os import re import stat import sys import tempfile ( target, transaction_id, mode, transaction_dir, env_snapshot, version, residue_policy, residue_plan, residue_plan_digest, ) = sys.argv[1:] if residue_policy not in {"safe", "wipe"}: raise SystemExit(1) if os.geteuid() != 0: raise SystemExit(1) if not all(os.path.isabs(value) and os.path.normpath(value) == value for value in (target, transaction_dir)): raise SystemExit(1) if env_snapshot and (not os.path.isabs(env_snapshot) or os.path.normpath(env_snapshot) != env_snapshot): raise SystemExit(1) if mode == "clean": if ( residue_plan != os.path.join(transaction_dir, "residue-plan.json") or not re.fullmatch(r"[a-f0-9]{64}", residue_plan_digest) ): raise SystemExit(1) elif residue_plan or residue_plan_digest: raise SystemExit(1) document = { "schema": "bridgesllm.uninstall-transaction.v1", "transactionId": transaction_id, "mode": mode, "phase": "prepared", "residuePolicy": residue_policy, "residuePlan": residue_plan or None, "residuePlanDigest": residue_plan_digest or None, "transactionDir": transaction_dir, "environmentSnapshot": env_snapshot or None, "installerVersion": version, "startedAt": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), "terminalDirectoryDeletionIntent": None, } directory = os.path.dirname(target) info = os.lstat(directory) if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022): raise SystemExit(1) fd, temporary = tempfile.mkstemp(prefix=".active-uninstall.", dir=directory) try: os.fchmod(fd, 0o600) with os.fdopen(fd, "w", encoding="utf-8") as handle: json.dump(document, handle, indent=2, sort_keys=True) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, target) temporary = "" os.chmod(target, 0o600) directory_fd = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY UNINSTALL_TRANSACTION_ID="${transaction_id}" } read_uninstall_transaction_field() { local field="$1" [[ "${field}" =~ ^[A-Za-z][A-Za-z0-9]*$ ]] || return 1 python3 - "${UNINSTALL_ACTIVE_JOURNAL}" "${field}" <<'PY' import json import os import re import stat import sys path, field = sys.argv[1:] info = os.lstat(path) if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 or info.st_size <= 0 or info.st_size > 65536): raise SystemExit(1) with open(path, "r", encoding="utf-8") as handle: document = json.load(handle) expected_keys = { "schema", "transactionId", "mode", "phase", "residuePolicy", "residuePlan", "residuePlanDigest", "transactionDir", "environmentSnapshot", "installerVersion", "startedAt", "terminalDirectoryDeletionIntent", } if ( not isinstance(document, dict) or set(document) != expected_keys or document.get("schema") != "bridgesllm.uninstall-transaction.v1" or not re.fullmatch(r"[a-f0-9]{32}", str(document.get("transactionId", ""))) or document.get("mode") not in {"keep", "clean"} or document.get("residuePolicy") not in {"safe", "wipe"} or not isinstance(document.get("phase"), str) or not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", document["phase"]) or not isinstance(document.get("installerVersion"), str) or not document["installerVersion"] or not isinstance(document.get("startedAt"), str) or not document["startedAt"] ): raise SystemExit(1) transaction_id = document["transactionId"] transaction_root = os.path.join(os.path.dirname(path), "transactions") transaction_dir = document.get("transactionDir") if ( not isinstance(transaction_dir, str) or transaction_dir != os.path.join(transaction_root, transaction_id) or os.path.normpath(transaction_dir) != transaction_dir ): raise SystemExit(1) environment_snapshot = document.get("environmentSnapshot") if environment_snapshot not in { None, os.path.join(transaction_dir, "portal.env") }: raise SystemExit(1) expected_plan = os.path.join( transaction_dir, "residue-plan.json") if document.get("mode") == "clean": if ( document.get("residuePlan") != expected_plan or not re.fullmatch( r"[a-f0-9]{64}", str(document.get("residuePlanDigest", ""))) ): raise SystemExit(1) elif ( document.get("residuePlan") is not None or document.get("residuePlanDigest") is not None ): raise SystemExit(1) intent = document.get("terminalDirectoryDeletionIntent") if intent is not None: expected_intent_keys = { "schema", "transactionId", "path", "device", "inode", "mode", "uid", "gid", "residuePlanDigest", } expected_intent_digest = ( document["residuePlanDigest"] if document["mode"] == "clean" and document["residuePolicy"] == "wipe" else None ) if ( document["phase"] != "terminal" or not isinstance(intent, dict) or set(intent) != expected_intent_keys or intent.get("schema") != "bridgesllm.uninstall-terminal-directory-deletion-intent.v1" or intent.get("transactionId") != transaction_id or intent.get("path") != transaction_dir or not all( isinstance(intent.get(name), int) and not isinstance(intent.get(name), bool) for name in ("device", "inode", "mode", "uid", "gid") ) or intent["device"] < 0 or intent["inode"] <= 0 or not stat.S_ISDIR(intent["mode"]) or stat.S_IMODE(intent["mode"]) != 0o700 or intent["uid"] != 0 or intent["gid"] != 0 or intent.get("residuePlanDigest") != expected_intent_digest ): raise SystemExit(1) if field not in document: raise SystemExit(1) value = document.get(field) if value is None: print("") elif isinstance(value, (str, int, bool)): print(str(value).lower() if isinstance(value, bool) else value) else: raise SystemExit(1) PY } advance_uninstall_transaction_phase() { local expected="$1" local next="$2" [[ "${expected}" =~ ^[a-z][a-z0-9_]{0,63}$ \ && "${next}" =~ ^[a-z][a-z0-9_]{0,63}$ ]] || return 1 python3 - "${UNINSTALL_ACTIVE_JOURNAL}" "${expected}" "${next}" <<'PY' import json import os import re import stat import sys import tempfile path, expected, next_phase = sys.argv[1:] info = os.lstat(path) if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 or info.st_size <= 0 or info.st_size > 65536): raise SystemExit(1) with open(path, "r", encoding="utf-8") as handle: document = json.load(handle) expected_keys = { "schema", "transactionId", "mode", "phase", "residuePolicy", "residuePlan", "residuePlanDigest", "transactionDir", "environmentSnapshot", "installerVersion", "startedAt", "terminalDirectoryDeletionIntent", } transaction_id = document.get("transactionId") transaction_root = os.path.join(os.path.dirname(path), "transactions") transaction_dir = document.get("transactionDir") expected_plan = ( os.path.join(transaction_dir, "residue-plan.json") if isinstance(transaction_dir, str) else "" ) if ( not isinstance(document, dict) or set(document) != expected_keys or document.get("schema") != "bridgesllm.uninstall-transaction.v1" or not isinstance(transaction_id, str) or not re.fullmatch(r"[a-f0-9]{32}", transaction_id) or transaction_dir != os.path.join(transaction_root, transaction_id) or os.path.normpath(transaction_dir) != transaction_dir or document.get("mode") not in {"keep", "clean"} or document.get("residuePolicy") not in {"safe", "wipe"} or document.get("phase") != expected or not isinstance(document.get("installerVersion"), str) or not document["installerVersion"] or not isinstance(document.get("startedAt"), str) or not document["startedAt"] or document.get("environmentSnapshot") not in { None, os.path.join(transaction_dir, "portal.env") } or document.get("terminalDirectoryDeletionIntent") is not None ): raise SystemExit(1) if document["mode"] == "clean": if ( document.get("residuePlan") != expected_plan or not re.fullmatch( r"[a-f0-9]{64}", str(document.get("residuePlanDigest", ""))) ): raise SystemExit(1) elif ( document.get("residuePlan") is not None or document.get("residuePlanDigest") is not None ): raise SystemExit(1) document["phase"] = next_phase directory = os.path.dirname(path) fd, temporary = tempfile.mkstemp(prefix=".active-uninstall.", dir=directory) try: os.fchmod(fd, 0o600) with os.fdopen(fd, "w", encoding="utf-8") as handle: json.dump(document, handle, indent=2, sort_keys=True) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) temporary = "" directory_fd = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY } install_portal_uninstall_boot_fence() { local expected="[Unit] ConditionPathExists=!${UNINSTALL_ACTIVE_JOURNAL} " install -d -m 755 -o root -g root "${UPDATE_BOOT_FENCE_DROPIN_DIR}" || return 1 python3 - "${UNINSTALL_BOOT_FENCE_DROPIN}" "${expected}" <<'PY' import os import stat import sys import tempfile target, expected = sys.argv[1:] directory = os.path.dirname(target) info = os.lstat(directory) if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022): raise SystemExit(1) try: current = os.lstat(target) except FileNotFoundError: current = None if current is not None: if (not stat.S_ISREG(current.st_mode) or stat.S_ISLNK(current.st_mode) or current.st_uid != 0 or current.st_gid != 0 or current.st_nlink != 1 or current.st_mode & 0o022 or current.st_size > 4096): raise SystemExit(1) with open(target, "r", encoding="utf-8") as handle: if handle.read() == expected: raise SystemExit(0) raise SystemExit(1) fd, temporary = tempfile.mkstemp(prefix=".30-uninstall-fence.", dir=directory) try: os.fchmod(fd, 0o644) with os.fdopen(fd, "w", encoding="utf-8") as handle: handle.write(expected) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, target) temporary = "" finally: if temporary: try: os.unlink(temporary) except FileNotFoundError: pass PY systemctl daemon-reload >/dev/null 2>&1 || return 1 } remove_exact_portal_tailnet_serve_mapping() { local env_file="${1:-}" [[ -n "${env_file}" && -f "${env_file}" && ! -L "${env_file}" ]] || return 0 assert_env_file_no_duplicate_keys "${env_file}" || return 1 local origin_mode tailnet_name origin_mode="$(read_env_value "${env_file}" ORIGIN_MODE 2>/dev/null || true)" [[ "${origin_mode}" == "tailnet" ]] || return 0 tailnet_name="$(read_env_value "${env_file}" TAILNET_DNS_NAME 2>/dev/null || true)" [[ "${tailnet_name}" =~ ^[A-Za-z0-9.-]{1,253}$ ]] || return 1 command -v tailscale >/dev/null 2>&1 || return 1 local before_file after_file target_port="" before_file="$(mktemp /tmp/bridgesllm-tailnet-serve-before.XXXXXX)" after_file="$(mktemp /tmp/bridgesllm-tailnet-serve-after.XXXXXX)" chmod 600 "${before_file}" "${after_file}" if ! tailscale serve status --json > "${before_file}" 2>/dev/null; then rm -f -- "${before_file}" "${after_file}" return 1 fi target_port="$(python3 - "${before_file}" "${tailnet_name}" <<'PY' import json import re import sys with open(sys.argv[1], "r", encoding="utf-8") as handle: document = json.load(handle) expected_name = sys.argv[2].rstrip(".").lower() matches = [] for endpoint, value in (document.get("Web") or {}).items(): if not isinstance(endpoint, str) or not isinstance(value, dict): continue host, separator, port = endpoint.rpartition(":") if not separator or host.rstrip(".").lower() != expected_name or not port.isdigit(): continue handlers = value.get("Handlers") if not isinstance(handlers, dict) or set(handlers) != {"/"}: continue handler = handlers.get("/") proxy = handler.get("Proxy") if isinstance(handler, dict) else None if proxy in {"http://127.0.0.1:4001", "http://localhost:4001"}: tcp = (document.get("TCP") or {}).get(port) if tcp != {"HTTPS": True}: raise SystemExit(2) matches.append(port) if len(matches) > 1: raise SystemExit(2) if matches: print(matches[0]) PY )" || { rm -f -- "${before_file}" "${after_file}"; return 1; } if [[ -z "${target_port}" ]]; then rm -f -- "${before_file}" "${after_file}" return 0 fi [[ "${target_port}" =~ ^[0-9]{1,5}$ \ && "${target_port}" -ge 1 && "${target_port}" -le 65535 ]] \ || { rm -f -- "${before_file}" "${after_file}"; return 1; } tailscale serve --https="${target_port}" off >/dev/null 2>&1 \ || { rm -f -- "${before_file}" "${after_file}"; return 1; } tailscale serve status --json > "${after_file}" 2>/dev/null \ || { rm -f -- "${before_file}" "${after_file}"; return 1; } python3 - "${before_file}" "${after_file}" "${tailnet_name}" "${target_port}" <<'PY' import copy import json import sys before_path, after_path, tailnet_name, port = sys.argv[1:] with open(before_path, "r", encoding="utf-8") as handle: before = json.load(handle) with open(after_path, "r", encoding="utf-8") as handle: after = json.load(handle) expected = copy.deepcopy(before) endpoint = f"{tailnet_name.rstrip('.')}:{port}" web = expected.get("Web") tcp = expected.get("TCP") if not isinstance(web, dict) or not isinstance(tcp, dict): raise SystemExit(1) removed = False for key in list(web): if key.rstrip(".").lower() == endpoint.lower(): del web[key] removed = True if port in tcp: del tcp[port] removed = True if not removed: raise SystemExit(1) if web == {}: expected.pop("Web", None) if tcp == {}: expected.pop("TCP", None) if after != expected: raise SystemExit(1) PY local status=$? rm -f -- "${before_file}" "${after_file}" return "${status}" } remove_portal_service_unit_for_uninstall() { systemctl disable bridgesllm-product >/dev/null 2>&1 || true rm -f -- \ /etc/systemd/system/bridgesllm-product.service \ /etc/systemd/system/multi-user.target.wants/bridgesllm-product.service systemctl daemon-reload >/dev/null 2>&1 || return 1 local active_state active_state="$(systemctl show --property=ActiveState --value bridgesllm-product 2>/dev/null)" \ || return 1 [[ "${active_state}" == "inactive" || "${active_state}" == "failed" ]] } # Persist a deletion before the uninstall journal advances past it. These # helpers are deliberately narrower than rm: only the transaction directory # or one exact root-owned regular file may be retired, and the containing # directory is fsynced after the directory entry changes. fsync_uninstall_directory() { local directory="$1" python3 - "${directory}" <<'PY' import os import stat import sys path = sys.argv[1] info = os.lstat(path) if ( not os.path.isabs(path) or os.path.normpath(path) != path or not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_mode & 0o022 ): raise SystemExit(1) descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(descriptor) finally: os.close(descriptor) PY } seal_terminal_uninstall_directory_deletion_intent() { local journal_path="$1" local transaction_dir="$2" python3 - "${journal_path}" "${transaction_dir}" <<'PY' import json import os import re import stat import sys import tempfile journal_path, transaction_dir = sys.argv[1:] expected_keys = { "schema", "transactionId", "mode", "phase", "residuePolicy", "residuePlan", "residuePlanDigest", "transactionDir", "environmentSnapshot", "installerVersion", "startedAt", "terminalDirectoryDeletionIntent", } expected_intent_keys = { "schema", "transactionId", "path", "device", "inode", "mode", "uid", "gid", "residuePlanDigest", } def read_pinned_journal(path): before = os.lstat(path) if ( not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or before.st_uid != 0 or before.st_gid != 0 or before.st_nlink != 1 or stat.S_IMODE(before.st_mode) != 0o600 or not 0 < before.st_size <= 65536 ): raise SystemExit("the terminal uninstall journal is unsafe") flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW descriptor = os.open(path, flags) try: opened = os.fstat(descriptor) current = os.lstat(path) identity = ( before.st_dev, before.st_ino, before.st_mode, before.st_uid, before.st_gid, before.st_nlink, ) if ( ( opened.st_dev, opened.st_ino, opened.st_mode, opened.st_uid, opened.st_gid, opened.st_nlink, ) != identity or ( current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_gid, current.st_nlink, ) != identity ): raise SystemExit( "the terminal uninstall journal raced its sealed identity") with os.fdopen( os.dup(descriptor), "r", encoding="utf-8" ) as handle: return json.load(handle) finally: os.close(descriptor) def validate_document(document): if ( not isinstance(document, dict) or set(document) != expected_keys or document.get("schema") != "bridgesllm.uninstall-transaction.v1" or not isinstance(document.get("transactionId"), str) or not re.fullmatch( r"[a-f0-9]{32}", document["transactionId"]) or document.get("mode") not in {"keep", "clean"} or document.get("phase") != "terminal" or document.get("residuePolicy") not in {"safe", "wipe"} or not isinstance(document.get("installerVersion"), str) or not document["installerVersion"] or not isinstance(document.get("startedAt"), str) or not document["startedAt"] ): raise SystemExit( "the terminal uninstall journal schema is invalid") transaction_id = document["transactionId"] transaction_root = os.path.join( os.path.dirname(journal_path), "transactions") expected_transaction_dir = os.path.join( transaction_root, transaction_id) if ( transaction_dir != expected_transaction_dir or document.get("transactionDir") != expected_transaction_dir or os.path.normpath(transaction_dir) != transaction_dir or document.get("environmentSnapshot") not in { None, os.path.join(transaction_dir, "portal.env") } ): raise SystemExit( "the terminal uninstall journal path is invalid") expected_plan = os.path.join( transaction_dir, "residue-plan.json") if document["mode"] == "clean": if ( document.get("residuePlan") != expected_plan or not re.fullmatch( r"[a-f0-9]{64}", str(document.get("residuePlanDigest", "")), ) ): raise SystemExit( "the terminal uninstall journal plan is invalid") elif ( document.get("residuePlan") is not None or document.get("residuePlanDigest") is not None ): raise SystemExit( "the terminal uninstall journal plan is invalid") expected_digest = ( document["residuePlanDigest"] if document["mode"] == "clean" and document["residuePolicy"] == "wipe" else None ) intent = document.get("terminalDirectoryDeletionIntent") if intent is not None and ( not isinstance(intent, dict) or set(intent) != expected_intent_keys or intent.get("schema") != "bridgesllm.uninstall-terminal-directory-deletion-intent.v1" or intent.get("transactionId") != transaction_id or intent.get("path") != transaction_dir or not all( isinstance(intent.get(name), int) and not isinstance(intent.get(name), bool) for name in ("device", "inode", "mode", "uid", "gid") ) or intent["device"] < 0 or intent["inode"] <= 0 or not stat.S_ISDIR(intent["mode"]) or stat.S_IMODE(intent["mode"]) != 0o700 or intent["uid"] != 0 or intent["gid"] != 0 or intent.get("residuePlanDigest") != expected_digest ): raise SystemExit( "the terminal uninstall deletion intent is invalid") return transaction_id, expected_digest, intent if ( not os.path.isabs(journal_path) or os.path.normpath(journal_path) != journal_path ): raise SystemExit("the terminal uninstall journal is unsafe") document = read_pinned_journal(journal_path) transaction_id, expected_digest, intent = validate_document(document) parent = os.path.dirname(transaction_dir) parent_info = os.lstat(parent) if ( not stat.S_ISDIR(parent_info.st_mode) or stat.S_ISLNK(parent_info.st_mode) or parent_info.st_uid != 0 or parent_info.st_gid != 0 or parent_info.st_mode & 0o022 or os.path.realpath(parent) != parent ): raise SystemExit( "the terminal uninstall transaction parent is unsafe") parent_flags = ( os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) ) parent_fd = os.open(parent, parent_flags) try: opened_parent = os.fstat(parent_fd) current_parent = os.lstat(parent) sealed_parent = ( parent_info.st_dev, parent_info.st_ino, parent_info.st_mode, parent_info.st_uid, parent_info.st_gid, ) if ( ( opened_parent.st_dev, opened_parent.st_ino, opened_parent.st_mode, opened_parent.st_uid, opened_parent.st_gid, ) != sealed_parent or ( current_parent.st_dev, current_parent.st_ino, current_parent.st_mode, current_parent.st_uid, current_parent.st_gid, ) != sealed_parent ): raise SystemExit( "the terminal uninstall transaction parent raced sealing") name = os.path.basename(transaction_dir) try: before = os.stat( name, dir_fd=parent_fd, follow_symlinks=False) except FileNotFoundError: if intent is None: raise SystemExit( "the uninstall transaction disappeared before deletion " "intent was sealed") raise SystemExit(0) if ( not stat.S_ISDIR(before.st_mode) or stat.S_ISLNK(before.st_mode) or before.st_uid != 0 or before.st_gid != 0 or stat.S_IMODE(before.st_mode) != 0o700 or before.st_dev != opened_parent.st_dev ): raise SystemExit( "the terminal uninstall transaction directory is unsafe") target_fd = os.open(name, parent_flags, dir_fd=parent_fd) try: opened = os.fstat(target_fd) current = os.stat( name, dir_fd=parent_fd, follow_symlinks=False) identity = ( before.st_dev, before.st_ino, before.st_mode, before.st_uid, before.st_gid, ) if ( ( opened.st_dev, opened.st_ino, opened.st_mode, opened.st_uid, opened.st_gid, ) != identity or ( current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_gid, ) != identity ): raise SystemExit( "the terminal uninstall transaction directory raced sealing") expected_intent = { "schema": "bridgesllm.uninstall-terminal-directory-deletion-intent.v1", "transactionId": transaction_id, "path": transaction_dir, "device": before.st_dev, "inode": before.st_ino, "mode": before.st_mode, "uid": before.st_uid, "gid": before.st_gid, "residuePlanDigest": expected_digest, } if intent is not None: if intent != expected_intent: raise SystemExit( "the terminal uninstall transaction no longer matches " "its deletion intent") raise SystemExit(0) rebound = os.stat( name, dir_fd=parent_fd, follow_symlinks=False) if ( rebound.st_dev, rebound.st_ino, rebound.st_mode, rebound.st_uid, rebound.st_gid, ) != identity: raise SystemExit( "the terminal uninstall transaction raced intent publication") finally: os.close(target_fd) finally: os.close(parent_fd) document["terminalDirectoryDeletionIntent"] = expected_intent validate_document(document) journal_parent = os.path.dirname(journal_path) journal_parent_info = os.lstat(journal_parent) if ( not stat.S_ISDIR(journal_parent_info.st_mode) or stat.S_ISLNK(journal_parent_info.st_mode) or journal_parent_info.st_uid != 0 or journal_parent_info.st_gid != 0 or journal_parent_info.st_mode & 0o022 ): raise SystemExit("the terminal uninstall journal parent is unsafe") descriptor, temporary = tempfile.mkstemp( prefix=".active-uninstall-terminal-intent.", dir=journal_parent, ) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "w", encoding="utf-8") as handle: descriptor = -1 json.dump(document, handle, indent=2, sort_keys=True) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, journal_path) temporary = "" directory_fd = os.open( journal_parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), ) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if descriptor >= 0: os.close(descriptor) if temporary: try: os.unlink(temporary) except FileNotFoundError: pass sealed_document = read_pinned_journal(journal_path) _transaction_id, _digest, sealed_intent = validate_document( sealed_document) if sealed_intent != expected_intent: raise SystemExit( "the terminal uninstall deletion intent was not committed") PY } durably_remove_uninstall_transaction_directory() { local transaction_dir="$1" local transactions_root="$2" local journal_path="$3" local self_mountinfo="${4:-/proc/self/mountinfo}" local pid1_mountinfo="${5:-/proc/1/mountinfo}" local fixture_crash_control="${6:-}" local fixture_trace="${7:-}" python3 - \ "${transaction_dir}" "${transactions_root}" "${journal_path}" \ "${self_mountinfo}" "${pid1_mountinfo}" \ "${fixture_crash_control}" "${fixture_trace}" <<'PY' import json import os import re import stat import sys ( target, expected_parent, journal_path, self_mountinfo, pid1_mountinfo, fixture_crash_control, fixture_trace, ) = sys.argv[1:] if ( not os.path.isabs(target) or os.path.normpath(target) != target or not os.path.isabs(expected_parent) or os.path.normpath(expected_parent) != expected_parent or os.path.dirname(target) != expected_parent or not re.fullmatch(r"[a-f0-9]{32}", os.path.basename(target)) ): raise SystemExit("the uninstall transaction path is invalid") expected_journal_keys = { "schema", "transactionId", "mode", "phase", "residuePolicy", "residuePlan", "residuePlanDigest", "transactionDir", "environmentSnapshot", "installerVersion", "startedAt", "terminalDirectoryDeletionIntent", } expected_intent_keys = { "schema", "transactionId", "path", "device", "inode", "mode", "uid", "gid", "residuePlanDigest", } if ( not os.path.isabs(journal_path) or os.path.normpath(journal_path) != journal_path or expected_parent != os.path.join(os.path.dirname(journal_path), "transactions") ): raise SystemExit( "the terminal uninstall deletion-intent journal is unsafe") journal_info = os.lstat(journal_path) if ( not stat.S_ISREG(journal_info.st_mode) or stat.S_ISLNK(journal_info.st_mode) or journal_info.st_uid != 0 or journal_info.st_gid != 0 or journal_info.st_nlink != 1 or stat.S_IMODE(journal_info.st_mode) != 0o600 or journal_info.st_size <= 0 or journal_info.st_size > 65536 ): raise SystemExit( "the terminal uninstall deletion-intent journal is unsafe") journal_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) if hasattr(os, "O_NOFOLLOW"): journal_flags |= os.O_NOFOLLOW journal_fd = os.open(journal_path, journal_flags) try: journal_opened = os.fstat(journal_fd) journal_current = os.lstat(journal_path) journal_identity = ( journal_info.st_dev, journal_info.st_ino, journal_info.st_mode, journal_info.st_uid, journal_info.st_gid, journal_info.st_nlink, ) if ( ( journal_opened.st_dev, journal_opened.st_ino, journal_opened.st_mode, journal_opened.st_uid, journal_opened.st_gid, journal_opened.st_nlink, ) != journal_identity or ( journal_current.st_dev, journal_current.st_ino, journal_current.st_mode, journal_current.st_uid, journal_current.st_gid, journal_current.st_nlink, ) != journal_identity ): raise SystemExit( "the terminal uninstall deletion-intent journal raced") with os.fdopen( os.dup(journal_fd), "r", encoding="utf-8" ) as handle: journal = json.load(handle) finally: os.close(journal_fd) transaction_id = os.path.basename(target) if ( not isinstance(journal, dict) or set(journal) != expected_journal_keys or journal.get("schema") != "bridgesllm.uninstall-transaction.v1" or journal.get("transactionId") != transaction_id or journal.get("transactionDir") != target or journal.get("mode") not in {"keep", "clean"} or journal.get("phase") != "terminal" or journal.get("residuePolicy") not in {"safe", "wipe"} or not isinstance(journal.get("installerVersion"), str) or not journal["installerVersion"] or not isinstance(journal.get("startedAt"), str) or not journal["startedAt"] or journal.get("environmentSnapshot") not in { None, os.path.join(target, "portal.env") } ): raise SystemExit( "the terminal uninstall deletion-intent journal is invalid") if journal["mode"] == "clean": if ( journal.get("residuePlan") != os.path.join(target, "residue-plan.json") or not re.fullmatch( r"[a-f0-9]{64}", str(journal.get("residuePlanDigest", "")), ) ): raise SystemExit( "the terminal uninstall deletion-intent plan is invalid") elif ( journal.get("residuePlan") is not None or journal.get("residuePlanDigest") is not None ): raise SystemExit( "the terminal uninstall deletion-intent plan is invalid") expected_intent_digest = ( journal["residuePlanDigest"] if journal["mode"] == "clean" and journal["residuePolicy"] == "wipe" else None ) intent = journal.get("terminalDirectoryDeletionIntent") if ( not isinstance(intent, dict) or set(intent) != expected_intent_keys or intent.get("schema") != "bridgesllm.uninstall-terminal-directory-deletion-intent.v1" or intent.get("transactionId") != transaction_id or intent.get("path") != target or not all( isinstance(intent.get(name), int) and not isinstance(intent.get(name), bool) for name in ("device", "inode", "mode", "uid", "gid") ) or intent["device"] < 0 or intent["inode"] <= 0 or not stat.S_ISDIR(intent["mode"]) or stat.S_IMODE(intent["mode"]) != 0o700 or intent["uid"] != 0 or intent["gid"] != 0 or intent.get("residuePlanDigest") != expected_intent_digest ): raise SystemExit( "the terminal uninstall directory deletion intent is invalid") intent_identity = ( intent["device"], intent["inode"], intent["mode"], intent["uid"], intent["gid"], ) fixture_root = os.environ.get("BRIDGESLLM_RESIDUE_TEST_ROOT", "") if fixture_crash_control or fixture_trace: if ( not fixture_root or not re.fullmatch( r"/tmp/bridgesllm-residue-fixture-[A-Za-z0-9._-]+", fixture_root, ) or os.path.realpath(fixture_root) != fixture_root or not target.startswith(fixture_root + os.sep) or fixture_crash_control != os.path.join( fixture_root, "state", "terminal-delete-crash-point") or fixture_trace != os.path.join( fixture_root, "state", "terminal-delete-fsync.log") ): raise SystemExit("the uninstall transaction fixture controls are unsafe") def decode_mount_path(value): escapes = { "040": " ", "011": "\t", "012": "\n", "134": "\\", } return re.sub( r"\\(040|011|012|134)", lambda match: escapes[match.group(1)], value, ) def relevant_mounts(path): try: with open(path, "r", encoding="utf-8") as handle: payload = handle.read(16 * 1024 * 1024 + 1) except OSError: raise SystemExit("uninstall transaction mount topology is unreadable") if len(payload.encode("utf-8", "replace")) > 16 * 1024 * 1024: raise SystemExit("uninstall transaction mount topology is oversized") result = set() for line in payload.splitlines(): columns = line.split() if len(columns) < 10 or "-" not in columns[6:]: raise SystemExit("uninstall transaction mount topology is invalid") mountpoint = decode_mount_path(columns[4]) if not os.path.isabs(mountpoint): raise SystemExit("uninstall transaction mount topology is invalid") if mountpoint == target or mountpoint.startswith(target + os.sep): result.add(mountpoint) return result def assert_mount_boundary_empty(): self_mounts = relevant_mounts(self_mountinfo) pid1_mounts = relevant_mounts(pid1_mountinfo) if self_mounts != pid1_mounts: raise SystemExit( "PID 1 and installer mount topology disagree below " "the uninstall transaction") if self_mounts: raise SystemExit( "the uninstall transaction contains a mount boundary") def descriptor_mount_id(descriptor): try: with open( f"/proc/self/fdinfo/{descriptor}", "r", encoding="utf-8", ) as handle: matches = [ line.split(":", 1)[1].strip() for line in handle if line.startswith("mnt_id:") ] except OSError: raise SystemExit( "the uninstall transaction descriptor mount identity " "could not be read") if ( len(matches) != 1 or not re.fullmatch(r"[0-9]+", matches[0]) or int(matches[0]) <= 0 ): raise SystemExit( "the uninstall transaction descriptor mount identity is invalid") return int(matches[0]) parent_before = os.lstat(expected_parent) if ( not stat.S_ISDIR(parent_before.st_mode) or stat.S_ISLNK(parent_before.st_mode) or parent_before.st_uid != 0 or parent_before.st_gid != 0 or parent_before.st_mode & 0o022 or os.path.realpath(expected_parent) != expected_parent ): raise SystemExit("the uninstall transaction parent is unsafe") parent_flags = ( os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) ) parent_fd = os.open(expected_parent, parent_flags) try: parent_opened = os.fstat(parent_fd) parent_current = os.lstat(expected_parent) sealed_parent = ( parent_before.st_dev, parent_before.st_ino, parent_before.st_mode, parent_before.st_uid, parent_before.st_gid, ) if ( ( parent_opened.st_dev, parent_opened.st_ino, parent_opened.st_mode, parent_opened.st_uid, parent_opened.st_gid, ) != sealed_parent or ( parent_current.st_dev, parent_current.st_ino, parent_current.st_mode, parent_current.st_uid, parent_current.st_gid, ) != sealed_parent ): raise SystemExit( "the uninstall transaction parent raced its sealed identity") root_mount_id = descriptor_mount_id(parent_fd) assert_mount_boundary_empty() name = os.path.basename(target) try: before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) except FileNotFoundError: before = None if before is not None: if ( not stat.S_ISDIR(before.st_mode) or stat.S_ISLNK(before.st_mode) or before.st_uid != 0 or before.st_gid != 0 or stat.S_IMODE(before.st_mode) != 0o700 or before.st_dev != parent_opened.st_dev or ( before.st_dev, before.st_ino, before.st_mode, before.st_uid, before.st_gid, ) != intent_identity ): raise SystemExit( "the uninstall transaction directory does not match " "its sealed deletion intent") target_fd = os.open(name, parent_flags, dir_fd=parent_fd) try: opened = os.fstat(target_fd) current = os.stat( name, dir_fd=parent_fd, follow_symlinks=False) sealed_target = ( before.st_dev, before.st_ino, before.st_mode, before.st_uid, before.st_gid, ) if ( ( opened.st_dev, opened.st_ino, opened.st_mode, opened.st_uid, opened.st_gid, ) != sealed_target or ( current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_gid, ) != sealed_target or descriptor_mount_id(target_fd) != root_mount_id ): raise SystemExit( "the uninstall transaction directory raced its " "sealed identity or crossed a mount") assert_mount_boundary_empty() def remove_contents( directory_fd, directory_path, root_device, root_mount_id, ): if descriptor_mount_id(directory_fd) != root_mount_id: raise SystemExit( "the uninstall transaction crossed a directory " "mount identity") assert_mount_boundary_empty() for entry in list(os.scandir(directory_fd)): child_path = os.path.join(directory_path, entry.name) child = os.stat( entry.name, dir_fd=directory_fd, follow_symlinks=False, ) if stat.S_ISDIR(child.st_mode): if ( child.st_dev != root_device or child.st_uid != 0 or child.st_gid != 0 or child.st_mode & 0o022 ): raise SystemExit( "the uninstall transaction crossed an " "unsafe directory boundary") child_fd = os.open( entry.name, parent_flags, dir_fd=directory_fd) try: child_opened = os.fstat(child_fd) if ( child_opened.st_dev, child_opened.st_ino, child_opened.st_mode, child_opened.st_uid, child_opened.st_gid, ) != ( child.st_dev, child.st_ino, child.st_mode, child.st_uid, child.st_gid, ) or descriptor_mount_id( child_fd ) != root_mount_id: raise SystemExit( "the uninstall transaction directory " "raced deletion or crossed a mount") remove_contents( child_fd, child_path, root_device, root_mount_id, ) os.fsync(child_fd) finally: os.close(child_fd) rebound = os.stat( entry.name, dir_fd=directory_fd, follow_symlinks=False, ) if ( rebound.st_dev, rebound.st_ino, rebound.st_mode, ) != ( child.st_dev, child.st_ino, child.st_mode, ): raise SystemExit( "the uninstall transaction directory " "raced deletion") assert_mount_boundary_empty() if descriptor_mount_id( directory_fd ) != root_mount_id: raise SystemExit( "the uninstall transaction parent crossed " "a mount before directory removal") os.rmdir(entry.name, dir_fd=directory_fd) continue if ( not stat.S_ISREG(child.st_mode) or child.st_dev != root_device or child.st_uid != 0 or child.st_gid != 0 or child.st_nlink != 1 or child.st_mode & 0o022 ): raise SystemExit( "the uninstall transaction contains an " "unsafe non-directory entry") rebound = os.stat( entry.name, dir_fd=directory_fd, follow_symlinks=False, ) if ( rebound.st_dev, rebound.st_ino, rebound.st_mode, rebound.st_nlink, ) != ( child.st_dev, child.st_ino, child.st_mode, child.st_nlink, ): raise SystemExit( "the uninstall transaction file raced deletion") assert_mount_boundary_empty() if descriptor_mount_id( directory_fd ) != root_mount_id: raise SystemExit( "the uninstall transaction parent crossed " "a mount before file removal") os.unlink(entry.name, dir_fd=directory_fd) remove_contents( target_fd, target, opened.st_dev, root_mount_id, ) os.fsync(target_fd) rebound = os.stat( name, dir_fd=parent_fd, follow_symlinks=False) if ( rebound.st_dev, rebound.st_ino, rebound.st_mode, ) != ( before.st_dev, before.st_ino, before.st_mode, ): raise SystemExit( "the uninstall transaction root raced deletion") os.rmdir(name, dir_fd=parent_fd) finally: os.close(target_fd) if fixture_crash_control: try: with open( fixture_crash_control, "r", encoding="utf-8" ) as handle: crash_point = handle.read(128).strip() except FileNotFoundError: crash_point = "" if crash_point == "after-rmdir-before-parent-fsync": os.unlink(fixture_crash_control) raise SystemExit( "fixture crash after transaction rmdir") # This fsync is intentionally unconditional. A resumed invocation that # observes the target already absent may be closing the kill window after # a prior rmdir but before that prior invocation's parent fsync. os.fsync(parent_fd) try: os.stat(name, dir_fd=parent_fd, follow_symlinks=False) except FileNotFoundError: pass else: raise SystemExit( "the uninstall transaction directory survived durable removal") if fixture_trace: with open(fixture_trace, "a", encoding="utf-8") as handle: handle.write("parent-fsync-after-absence\n") finally: os.close(parent_fd) PY } durably_unlink_uninstall_file() { local path="$1" local expected_parent="$2" python3 - "${path}" "${expected_parent}" <<'PY' import os import stat import sys path, expected_parent = sys.argv[1:] if ( not os.path.isabs(path) or os.path.normpath(path) != path or os.path.dirname(path) != expected_parent ): raise SystemExit(1) parent = os.lstat(expected_parent) if ( not stat.S_ISDIR(parent.st_mode) or stat.S_ISLNK(parent.st_mode) or parent.st_uid != 0 or parent.st_gid != 0 or parent.st_mode & 0o022 ): raise SystemExit(1) try: info = os.lstat(path) except FileNotFoundError: info = None if info is not None: if ( not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 ): raise SystemExit(1) os.unlink(path) descriptor = os.open(expected_parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(descriptor) finally: os.close(descriptor) PY } # Terminal cleanup must stay re-runnable: a crash between the terminal phase # advance and these removals leaves the journal on disk, and a surviving # journal keeps the boot fence blocking every future Portal start and refuses # any new uninstall transaction. The transaction directory (which holds the # environment snapshot) may already be gone on a re-run only when the journal # first sealed its exact deletion intent; the journal is removed last as the # commit point so recovery can always finish the rest. cleanup_terminal_uninstall_transaction() { local transaction_dir transaction_id mode residue_policy residue_plan transaction_dir="$(read_uninstall_transaction_field transactionDir)" || return 1 transaction_id="$(read_uninstall_transaction_field transactionId)" || return 1 mode="$(read_uninstall_transaction_field mode)" || return 1 residue_policy="$(read_uninstall_transaction_field residuePolicy)" || return 1 [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ \ && "${transaction_dir}" == "${UNINSTALL_TRANSACTIONS_ROOT}/${transaction_id}" ]] \ || return 1 if [[ "${mode}" == "clean" && "${residue_policy}" == "wipe" ]]; then # This is the final commit guard, after every phase reports success and # before the consent plan or journal can disappear. While the transaction # directory exists, prove every sealed full Docker identity absent too; # after that directory is durably removed, container/network identities # cannot be recreated and exact volume names remain covered by the global # name proof. Any resurrected filesystem path or Portal firewall rule also # keeps recovery active. if [[ -e "${transaction_dir}" || -L "${transaction_dir}" ]]; then residue_plan="$(read_uninstall_transaction_field residuePlan)" || return 1 [[ "${residue_plan}" == "${transaction_dir}/residue-plan.json" ]] \ || return 1 managed_runtime_residue_tool \ prove-absent all "" "${residue_plan}" >/dev/null \ || return 1 fi assert_no_managed_project_runtime_residuals || return 1 assert_no_managed_agent_zero_runtime_residuals || return 1 fi if [[ "${mode}" == "clean" ]]; then # Clean slate has already retired the Portal runtime, authorization # database, and managed provider resources. Remove only the exact # installer/backend-owned OpenClaw fence pair. Keep Data deliberately # preserves both so an unresolved transition cannot be reopened by # uninstalling the Portal runtime around it. remove_openclaw_gateway_authorization_fence_for_clean_uninstall \ || return 1 fi # Claim the exact root-owned transaction inode in the active journal only # after every applicable sealed/global absence proof succeeds. A resumed # cleanup may accept lexical directory absence only when this durable intent # already exists; otherwise an out-of-band pre-cleanup rename would erase # the full-identity consent plan and bypass prove-absent. seal_terminal_uninstall_directory_deletion_intent \ "${UNINSTALL_ACTIVE_JOURNAL}" "${transaction_dir}" \ || return 1 durably_remove_uninstall_transaction_directory \ "${transaction_dir}" "${UNINSTALL_TRANSACTIONS_ROOT}" \ "${UNINSTALL_ACTIVE_JOURNAL}" \ || return 1 durably_unlink_uninstall_file \ "${UNINSTALL_BOOT_FENCE_DROPIN}" "${UPDATE_BOOT_FENCE_DROPIN_DIR}" \ || return 1 systemctl daemon-reload >/dev/null 2>&1 || return 1 # The journal is the final commit point. If any earlier cleanup or # daemon-reload fails, it remains and the next installer run resumes the # exact terminal phase instead of marooning a loaded boot fence. durably_unlink_uninstall_file \ "${UNINSTALL_ACTIVE_JOURNAL}" "${UNINSTALL_STATE_ROOT}" \ || return 1 [[ ! -e "${UNINSTALL_ACTIVE_JOURNAL}" && ! -L "${UNINSTALL_ACTIVE_JOURNAL}" ]] } complete_uninstall_transaction() { advance_uninstall_transaction_phase terminal_pending terminal || return 1 cleanup_terminal_uninstall_transaction } resume_active_uninstall_transaction() { [[ -f "${UNINSTALL_ACTIVE_JOURNAL}" && ! -L "${UNINSTALL_ACTIVE_JOURNAL}" ]] \ || return 1 local mode phase env_snapshot transaction_id mode="$(read_uninstall_transaction_field mode)" || return 1 env_snapshot="$(read_uninstall_transaction_field environmentSnapshot)" || return 1 transaction_id="$(read_uninstall_transaction_field transactionId)" || return 1 [[ "${mode}" == "keep" || "${mode}" == "clean" ]] || return 1 [[ "${transaction_id}" =~ ^[a-f0-9]{32}$ ]] || return 1 phase="$(read_uninstall_transaction_field phase)" || return 1 if [[ "${phase}" == "terminal" ]]; then # The prior run crashed after its terminal advance; the transaction # directory and its environment snapshot may already be gone, so finish # the terminal cleanup without revalidating them. cleanup_terminal_uninstall_transaction return fi if [[ -n "${env_snapshot}" ]]; then [[ "${env_snapshot}" == "${UNINSTALL_TRANSACTIONS_ROOT}/${transaction_id}/portal.env" \ && -f "${env_snapshot}" && ! -L "${env_snapshot}" \ && "$(stat -c '%u:%g:%a' "${env_snapshot}")" == '0:0:600' ]] || return 1 assert_env_file_no_duplicate_keys "${env_snapshot}" || return 1 fi while :; do phase="$(read_uninstall_transaction_field phase)" || return 1 case "${phase}" in prepared) install_portal_uninstall_boot_fence || return 1 advance_uninstall_transaction_phase prepared boot_fenced || return 1 ;; boot_fenced) stop_portal_service_for_uninstall advance_uninstall_transaction_phase boot_fenced portal_quiesced || return 1 ;; portal_quiesced) remove_portal_remote_desktop_runtime "$([[ "${mode}" == "clean" ]] && printf 2 || printf 1)" advance_uninstall_transaction_phase portal_quiesced remote_desktop_removed || return 1 ;; remote_desktop_removed) remove_portal_backup_automation advance_uninstall_transaction_phase remote_desktop_removed backup_automation_removed || return 1 ;; backup_automation_removed) if [[ "${mode}" == "clean" ]]; then run_project_runtime_clean_slate_preflight else record_retained_runtime_intent quiesce_project_runtimes_preserving_data fi advance_uninstall_transaction_phase backup_automation_removed project_runtimes_removed || return 1 ;; project_runtimes_removed) local bridge_helper="${PORTAL_DIR}/installer/agent-zero-project-model-bridge.sh" if [[ -f "${bridge_helper}" && ! -L "${bridge_helper}" ]]; then local bridge_command='uninstall' [[ "${mode}" == "clean" ]] && bridge_command='uninstall-clean-slate' bash "${bridge_helper}" "${bridge_command}" \ || fail "Agent Zero Project model bridge could not revoke its upstream token safely." elif [[ -e "${bridge_helper}" || -L "${bridge_helper}" ]]; then fail "Agent Zero Project model bridge lifecycle helper is linked or non-regular; uninstall was aborted." else assert_no_agent_zero_project_bridge_residuals fi advance_uninstall_transaction_phase project_runtimes_removed agent_zero_bridge_removed || return 1 ;; agent_zero_bridge_removed) remove_exact_portal_tailnet_serve_mapping "${env_snapshot}" \ || fail "The Portal's exact Tailscale Serve mapping could not be removed without changing unrelated tailnet routes." advance_uninstall_transaction_phase agent_zero_bridge_removed tailnet_mapping_removed || return 1 ;; tailnet_mapping_removed) remove_portal_service_unit_for_uninstall \ || fail "The Portal service unit could not be removed and re-inspected safely." advance_uninstall_transaction_phase tailnet_mapping_removed portal_service_removed || return 1 ;; portal_service_removed) if [[ "${mode}" == "clean" ]]; then remove_managed_agent_zero_for_clean_uninstall advance_uninstall_transaction_phase portal_service_removed agent_zero_removed || return 1 else remove_portal_runtime_preserving_data "${PORTAL_DIR}" \ || fail "Could not remove the Portal runtime without risking retained data." advance_uninstall_transaction_phase portal_service_removed runtime_removed || return 1 fi ;; runtime_removed) [[ "${mode}" == "keep" ]] || return 1 write_retained_install_receipt \ || fail "The retained Portal tree could not be sealed with an exact root-only reconnect receipt." advance_uninstall_transaction_phase runtime_removed retained_receipt_written || return 1 ;; retained_receipt_written) [[ "${mode}" == "keep" ]] || return 1 advance_uninstall_transaction_phase retained_receipt_written terminal_pending || return 1 ;; agent_zero_removed) [[ "${mode}" == "clean" ]] || return 1 stop_managed_stalwart_for_clean_uninstall "${INSTALL_ROOT}/stalwart" advance_uninstall_transaction_phase agent_zero_removed stalwart_removed || return 1 ;; stalwart_removed) [[ "${mode}" == "clean" ]] || return 1 local configured_url="" if [[ -n "${env_snapshot}" ]]; then configured_url="$(read_env_value "${env_snapshot}" DATABASE_URL 2>/dev/null || true)" fi remove_configured_default_database_for_clean_uninstall "${configured_url}" advance_uninstall_transaction_phase stalwart_removed database_removed || return 1 ;; database_removed) [[ "${mode}" == "clean" ]] || return 1 remove_legacy_portal_data_for_clean_uninstall advance_uninstall_transaction_phase database_removed legacy_data_removed || return 1 ;; legacy_data_removed) [[ "${mode}" == "clean" ]] || return 1 remove_install_root_for_clean_uninstall advance_uninstall_transaction_phase legacy_data_removed install_root_removed || return 1 ;; install_root_removed) [[ "${mode}" == "clean" ]] || return 1 advance_uninstall_transaction_phase install_root_removed terminal_pending || return 1 ;; terminal_pending) complete_uninstall_transaction || return 1 return 0 ;; terminal) cleanup_terminal_uninstall_transaction || return 1 return 0 ;; *) return 1 ;; esac done } recover_pending_uninstall_transaction() { [[ -e "${UNINSTALL_ACTIVE_JOURNAL}" || -L "${UNINSTALL_ACTIVE_JOURNAL}" ]] \ || return 0 [[ -f "${UNINSTALL_ACTIVE_JOURNAL}" && ! -L "${UNINSTALL_ACTIVE_JOURNAL}" ]] \ || return 1 warn "An interrupted Portal uninstall exists; resuming its exact previously confirmed data policy." resume_active_uninstall_transaction || return 1 UNINSTALL_RECOVERED_THIS_RUN=true ok "Interrupted Portal uninstall converged" } stop_portal_service_for_uninstall() { # Stop errors are not authoritative: a service that was already stopped can # legitimately make the stop command fail. The readback is authoritative, # and any query failure is unsafe because the API could still recreate # Remote Desktop or backup state while uninstall removes it. systemctl stop bridgesllm-product >/dev/null 2>&1 || true local load_state active_state load_state="$(systemctl show --property=LoadState --value bridgesllm-product 2>/dev/null)" \ || fail "Portal service load state could not be verified; uninstall was aborted before data removal." active_state="$(systemctl show --property=ActiveState --value bridgesllm-product 2>/dev/null)" \ || fail "Portal service active state could not be verified; uninstall was aborted before data removal." [[ -n "${load_state}" && -n "${active_state}" ]] \ || fail "Portal service returned an incomplete state; uninstall was aborted before data removal." case "${active_state}" in inactive|failed) ;; *) fail "Portal service is still ${active_state}; uninstall was aborted before data removal." ;; esac } do_uninstall() { banner echo "" echo -e " ${BOLD}${RED}Uninstalling BridgesLLM Portal${NC}" echo "" warn "This will remove:" echo " ${BULLET} Portal at ${PORTAL_DIR}" echo " ${BULLET} Portal and Remote Desktop services and launchers" echo " ${BULLET} Local Portal data only if you explicitly choose Clean slate" echo "" # The database may be remote/custom and therefore absent from the local # PostgreSQL catalog. Always offer preservation; do not infer that no data # exists merely because the default local database is not present. local configured_database_url="" configured_database_url="$(read_env_value "${PORTAL_DIR}/backend/.env.production" DATABASE_URL 2>/dev/null || true)" echo -e " ${CYAN}Choose whether Portal-owned data should remain on this server.${NC}" echo "" echo -e " 1) ${BOLD}Keep my data${NC} — Preserve the configured database, projects, apps," echo -e " uploads, mail, Portal state, backups, OpenClaw state, and Remote Desktop profile" echo -e " ${DIM}(Reinstall later using the same owner credentials.)${NC}" echo -e "" echo -e " 2) ${BOLD}Clean slate${NC} — Remove the local Portal installation, default database," echo -e " and Portal-owned Remote Desktop account/profile" echo -e " ${DIM}(Backup archives inside ${INSTALL_ROOT} and managed Agent Zero recovery snapshots${NC}" echo -e " ${DIM} under /var/backups/bridgesllm/agent-zero are removed. Other external archives remain and may${NC}" echo -e " ${DIM} contain Portal, OpenClaw, mail, project, credential, and uploaded-file data.)${NC}" echo -e " ${DIM}(Irreversible. A separately hosted/custom database is not dropped.)${NC}" echo "" read -rp " Choose [1/2] (default: 1): " data_choice data_choice="${data_choice:-1}" [[ "${data_choice}" == "1" || "${data_choice}" == "2" ]] \ || fail "Invalid uninstall data choice; expected 1 or 2." # Leftover residue from an earlier partial cleanup is only decidable by the # user: the signed helper is gone, so either the leftovers stay (safe) or # the installer edits the host firewall (wipe). Ask with the findings and an # honest size estimate in front of them, before the final confirmation. local residue_policy="safe" if [[ "${data_choice}" == "2" ]]; then local residue_helper="${PORTAL_DIR}/backend/dist/cli/projectRuntimeUninstallPreflight.js" local residue_report="" residue_first="" residue_report="$(managed_runtime_residue_tool report all)" \ || fail "Managed runtime residue could not be authoritatively inventoried before confirmation." residue_first="$(head -n 1 <<<"${residue_report}")" [[ "${residue_first}" =~ planDigest=([a-f0-9]{64}) ]] \ || fail "Managed runtime residue inventory did not include a valid consent digest." UNINSTALL_RESIDUE_PLAN_DIGEST="${BASH_REMATCH[1]}" if [[ ( ! -f "${residue_helper}" || -L "${residue_helper}" ) \ && "${residue_first}" == present=true* ]]; then local residue_bytes residue_mb residue_bytes="${residue_first##*bytes=}" [[ "${residue_bytes}" =~ ^[0-9]+$ ]] || residue_bytes=0 residue_mb=$(( (residue_bytes + 1048575) / 1048576 )) echo "" warn "Leftovers from an earlier partial cleanup were found (their cleanup helper is gone):" tail -n +2 <<<"${residue_report}" | sed 's/^/ /' echo "" echo -e " ${CYAN}Choose how to handle these leftovers.${NC}" echo "" echo " 1) ${BOLD}Safe cleanup${NC} — Stop leftover managed containers, then finish" echo " uninstalling while leaving their Docker objects and firewall entries in place" echo " ${DIM}(~${residue_mb} MB stays on disk. The firewall entries use no disk space —${NC}" echo " ${DIM} they may still affect matching traffic. The host firewall is never edited.)${NC}" echo "" echo " 2) ${BOLD}Complete wipe${NC} — Also delete the leftover Docker resources and the" echo " Portal-created firewall entries" echo " ${DIM}(A full firewall backup is saved first, but editing the host firewall${NC}" echo " ${DIM} carries a small risk of breaking this server's networking.)${NC}" echo " ${DIM}(Removal is confirmed at each recorded location. Anything copied or${NC}" echo " ${DIM} moved elsewhere beforehand is not tracked and will remain.)${NC}" echo "" if [[ -n "${RESIDUE_POLICY}" ]]; then residue_policy="${RESIDUE_POLICY}" info "Using --residue-policy ${residue_policy} for these leftovers." else local residue_choice read -rp " Choose [1/2] (default: 1): " residue_choice residue_choice="${residue_choice:-1}" [[ "${residue_choice}" == "1" || "${residue_choice}" == "2" ]] \ || fail "Invalid leftover-cleanup choice; expected 1 or 2." [[ "${residue_choice}" == "2" ]] && residue_policy="wipe" fi elif [[ -n "${RESIDUE_POLICY}" ]]; then residue_policy="${RESIDUE_POLICY}" fi fi echo "" read -rp " Type 'yes' to confirm uninstall: " yn [[ "$yn" == "yes" ]] || { echo " Cancelled."; exit 0; } local uninstall_mode="keep" [[ "${data_choice}" == "2" ]] && uninstall_mode="clean" # Clean slate destroys the database and runtimes before it reaches legacy # attribution. Prove attribution first, while the system is still intact. if [[ "${uninstall_mode}" == "clean" ]]; then assert_clean_slate_legacy_attribution fi prepare_uninstall_transaction \ "${uninstall_mode}" "${PORTAL_DIR}/backend/.env.production" "${residue_policy}" \ || fail "The durable uninstall transaction could not be created before teardown." resume_active_uninstall_transaction \ || fail "The uninstall transaction did not converge. Its root-only journal and boot fence were preserved for the next installer run." if [[ "${data_choice}" == "2" ]]; then ok "BridgesLLM Portal and local Portal data removed" else ok "BridgesLLM Portal runtime removed; retained data was left in place" fi echo "" if [[ "${data_choice:-1}" == "1" ]]; then echo -e " ${GREEN}Your data has been preserved.${NC}" echo -e " ${DIM}Reinstall to reconnect the retained database and files.${NC}" echo -e " ${DIM}Your existing owner password remains unchanged.${NC}" echo -e " ${DIM}Stalwart mail and OpenClaw were not stopped or modified.${NC}" [[ -n "${configured_database_url}" ]] \ && echo -e " ${DIM}The retained DATABASE_URL will be reused exactly.${NC}" else echo -e " ${DIM}Not removed (clean up manually if needed):${NC}" echo " ${BULLET} Caddy (/etc/caddy/Caddyfile)" echo " ${BULLET} Node.js, Docker, ClamAV, Ollama, OpenClaw" echo " ${BULLET} Backup archives stored outside ${INSTALL_ROOT}" echo " ${BULLET} Tailscale, including this machine's tailnet membership" echo "" # Tailscale surviving while the Portal's record of the pairing does not is # the confusing part: the tailnet still looks healthy, so a working Remote # GPU appears to have broken for no reason. Say it plainly. echo -e " ${DIM}Removed with the database (re-add after reinstalling):${NC}" echo " ${BULLET} Remote GPU / tailnet peer configuration held by Portal" echo " ${BULLET} Connected provider accounts and per-project settings" echo -e " ${DIM}Tailscale itself stays installed and joined, so the peer remains${NC}" echo -e " ${DIM}reachable — only Portal's side of the pairing is gone.${NC}" if [[ -n "${UNINSTALL_RESIDUE_WIPE_BACKUP_DIR}" ]]; then echo "" echo -e " ${DIM}A firewall backup from the Complete wipe was kept at:${NC}" echo " ${BULLET} ${UNINSTALL_RESIDUE_WIPE_BACKUP_DIR}" echo -e " ${DIM}(Restore with iptables-restore / ip6tables-restore if networking misbehaves.)${NC}" fi fi echo "" } # ═══════════════════════════════════════════════════════════════ # Main # ═══════════════════════════════════════════════════════════════ print_dry_run_plan() { local existing_install=false action="install" if [[ -f "${PORTAL_DIR}/backend/package.json" \ && -f "${PORTAL_DIR}/backend/.env.production" ]]; then existing_install=true fi if $UNINSTALL_MODE; then action="uninstall" elif $UPDATE_MODE; then action="update" elif $FORCE_FRESH && $existing_install; then action="reinstall" elif $existing_install; then action="update" fi banner echo "" echo -e " ${BOLD}${WHITE}Dry-run plan${NC}" echo -e " ${GREEN}No changes will be made.${NC}" echo -e " ${DIM}This command performs no filesystem writes, lock creation, downloads,${NC}" echo -e " ${DIM}telemetry, package work, service changes, firewall changes, Tailscale${NC}" echo -e " ${DIM}changes, database work, or interactive confirmation prompts.${NC}" echo "" print_kv "Action" "${action}" print_kv "Portal path" "${PORTAL_DIR}" print_kv "Existing install" "${existing_install}" if use_tailnet_profile; then print_kv "Origin" "experimental private Tailscale (${TS_HOSTNAME})" elif use_local_profile; then print_kv "Origin" "experimental local/loopback" elif [[ -n "${DOMAIN}" ]]; then print_kv "Origin" "public HTTPS (${DOMAIN})" else print_kv "Origin" "loopback until browser setup" fi echo "" case "${action}" in uninstall) echo " A real run would:" echo " ${BULLET} Ask whether to keep Portal data or remove the local clean slate" echo " ${BULLET} On a clean slate with helperless leftovers, ask for Safe cleanup or Complete wipe" echo " ${BULLET} Require an exact uninstall confirmation" echo " ${BULLET} Stop Portal, then remove its service, Remote Desktop, and backup automation" echo " ${BULLET} Preserve or remove only the data selected at confirmation time" ;; update) echo " A real run would:" echo " ${BULLET} Stage and cryptographically verify the signed ${VERSION} release" echo " ${BULLET} Stop Portal and capture runtime, database, dependency, config, and provenance recovery state" echo " ${BULLET} Replace the runtime, migrate the database, and converge managed services" echo " ${BULLET} Verify readiness before committing release provenance; recover on failure" if $MAINTAIN_TOOLS; then echo " ${BULLET} Explicitly maintain optional AI tools" fi ;; reinstall) echo " A real run would repair the existing Portal through the protected update transaction:" echo " ${BULLET} Preserve its database, secrets, origin identities, projects, apps, uploads, and runtime data" echo " ${BULLET} Quiesce Portal, snapshot every rollback layer, then reinstall the signed runtime" echo " ${BULLET} Run migrations and converge managed services with durable interruption recovery" echo " ${BULLET} Verify the new process and exact version before committing" ;; install) echo " A real run would:" echo " ${BULLET} Validate host resources, OS support, and required port boundaries" if use_tailnet_profile; then echo " ${BULLET} Install/connect Tailscale and configure private HTTPS Serve" fi echo " ${BULLET} Install pinned host tools and provision the Portal database" echo " ${BULLET} Verify and install the signed Portal release" echo " ${BULLET} Configure managed services, backups, and Remote Desktop" echo " ${BULLET} Start Portal and verify readiness before reporting completion" ;; esac echo "" echo -e " ${DIM}Dry-run is intentionally read-only; it does not prove network or dependency readiness.${NC}" echo "" } main() { parse_args "$@" if ${REPAIR_PROJECT_RUNTIME_IMAGE}; then [[ "${EUID:-$(id -u)}" -eq 0 ]] \ || fail "Must run Project runtime image repair as root." acquire_project_runtime_image_repair_lock repair_project_runtime_image exit 0 fi detect_runtime_profile detect_retained_install_reconnect load_existing_origin_for_forced_reinstall validate_selected_origin # A dry-run is a zero-side-effect planning command. Keep it above the root # check and operation lock: even creating the lock inode would violate the # advertised contract, and routing into update/uninstall would be dangerous. if $DRY_RUN; then print_dry_run_plan exit 0 fi [[ "${EUID:-$(id -u)}" -eq 0 ]] || fail "Must run as root. Use: sudo bash ${SCRIPT_NAME}" acquire_portal_operation_lock if ${UNINSTALL_RECOVERED_THIS_RUN}; then info "The interrupted uninstall is complete. Run the installer again only if you now want to install Portal." exit 0 fi # Uninstall mode if $UNINSTALL_MODE; then do_uninstall exit 0 fi # Update mode if $UPDATE_MODE; then mkdir -p "$LOG_DIR" touch "$LOG_FILE" chmod 600 "$LOG_FILE" do_update exit 0 fi # A forced reinstall over an attested installation is a repair operation, # not a fresh install. It must use the same quiesce/snapshot/rollback path as # an update so a live old process can never race file or schema replacement. if $FORCE_FRESH \ && [[ -f "${PORTAL_DIR}/backend/package.json" ]] \ && [[ -f "${PORTAL_DIR}/backend/.env.production" ]]; then mkdir -p "$LOG_DIR" touch "$LOG_FILE" chmod 600 "$LOG_FILE" REPAIR_REINSTALL=true UPDATE_MODE=true do_update exit 0 fi # Auto-detect an existing installation and update it instead of reinstalling. # The advertised one-liner (curl ... | sudo bash) carries no --update flag, so # a re-run on an existing box would otherwise take the fresh-install path — # which rsyncs the live dir onto itself (RELEASE_FALLBACK_DIR == PORTAL_DIR), # silently no-ops the deploy, and still reports "Installation complete!". It # would also rewrite a working HTTPS Caddyfile to HTTP-only when DOMAIN is # blank in .env.production. Routing to do_update fixes both. --reinstall uses # that same protected transaction with repair semantics. if ! $FORCE_FRESH \ && [[ -f "${PORTAL_DIR}/backend/package.json" ]] \ && [[ -f "${PORTAL_DIR}/backend/.env.production" ]]; then mkdir -p "$LOG_DIR" touch "$LOG_FILE" chmod 600 "$LOG_FILE" info "Existing installation detected at ${PORTAL_DIR} — updating instead of reinstalling." info "(Use --reinstall to force a fresh install.)" UPDATE_MODE=true do_update exit 0 fi assert_fresh_install_target_available # Fresh install mkdir -p "$LOG_DIR" touch "$LOG_FILE" chmod 600 "$LOG_FILE" INSTALL_START_TIME=$(date +%s) banner preflight converge_unsafe_docker_prune_automation \ || fail "Unsafe scheduled Docker cleanup remains active. Review the guard details above, disable the unknown job or repair the legacy-file drift, and retry." ensure_telemetry_install_id telemetry_event "install_start" if use_tailnet_profile; then setup_tailnet_origin fi install_system_packages install_ai_tools setup_database build_portal install_native_provider_tools configure_services configure_backup_timers setup_remote_desktop start_portal print_success } if [[ "${BRIDGESLLM_INSTALLER_SOURCE_ONLY:-0}" != "1" ]]; then main "$@" fi