fix(lifecycle): retain Docker start diagnostics

docs/PROJECT-STATE.md

@@ -99,3 +99,4 @@ Read this compact operational baseline before starting a milestone. Open detaile
 - Update this file after every merged milestone or durable architectural change; keep it compact and remove stale statements.
 
 - Web access and i18n are stored in the initial SQLite schema. HTTP defaults to working session/CSRF cookies without Secure and without HSTS; HTTPS enforcement is explicit.
+- Lifecycle failures retain a bounded diagnostic record separate from Audit. The restricted agent captures Docker inspection state (including exit/OOM/timestamps/health) and a bounded log tail after a failed start; the application persists this record under the operation ID and writes a stable `DGM-*` error category. Existing SQLite stores receive the additive diagnostic table at open time.

internal/agent/agent_docker.go

@@ -31,15 +31,25 @@ type DockerRuntime interface {
 	Restart(context.Context, string, int) error
 	Delete(context.Context, string) error
 	Inspect(context.Context, string) (DockerInspection, error)
+	Logs(context.Context, string, int) (string, error)
 	Stats(context.Context, string) (agentwire.InstanceStats, error)
 }
 
 type DockerInspection struct {
-	ContainerID string
-	Running     bool
-	Health      string
-	ExitCode    int
-	Labels      map[string]string
+	ContainerID  string
+	Running      bool
+	Restarting   bool
+	Paused       bool
+	OOMKilled    bool
+	Dead         bool
+	Status       string
+	Error        string
+	StartedAt    string
+	FinishedAt   string
+	RestartCount int
+	Health       string
+	ExitCode     int
+	Labels       map[string]string
 }
 
 type AssetMount struct {
@@ -291,9 +301,18 @@ func (d *dockerRuntime) Inspect(ctx context.Context, id string) (DockerInspectio
 			Labels map[string]string `json:"Labels"`
 		} `json:"Config"`
 		State struct {
-			Running  bool `json:"Running"`
-			ExitCode int  `json:"ExitCode"`
-			Health   *struct {
+			Running      bool   `json:"Running"`
+			Restarting   bool   `json:"Restarting"`
+			Paused       bool   `json:"Paused"`
+			OOMKilled    bool   `json:"OOMKilled"`
+			Dead         bool   `json:"Dead"`
+			Status       string `json:"Status"`
+			Error        string `json:"Error"`
+			StartedAt    string `json:"StartedAt"`
+			FinishedAt   string `json:"FinishedAt"`
+			RestartCount int    `json:"RestartCount"`
+			ExitCode     int    `json:"ExitCode"`
+			Health       *struct {
 				Status string `json:"Status"`
 			} `json:"Health"`
 		} `json:"State"`
@@ -305,7 +324,18 @@ func (d *dockerRuntime) Inspect(ctx context.Context, id string) (DockerInspectio
 	if payload.State.Health != nil {
 		health = payload.State.Health.Status
 	}
-	return DockerInspection{ContainerID: payload.ID, Running: payload.State.Running, Health: health, ExitCode: payload.State.ExitCode, Labels: payload.Config.Labels}, nil
+	return DockerInspection{ContainerID: payload.ID, Running: payload.State.Running, Restarting: payload.State.Restarting, Paused: payload.State.Paused, OOMKilled: payload.State.OOMKilled, Dead: payload.State.Dead, Status: payload.State.Status, Error: payload.State.Error, StartedAt: payload.State.StartedAt, FinishedAt: payload.State.FinishedAt, RestartCount: payload.State.RestartCount, Health: health, ExitCode: payload.State.ExitCode, Labels: payload.Config.Labels}, nil
+}
+
+// Logs returns a bounded tail. Docker multiplexes stream frames only when TTY
+// is enabled; game templates do not enable TTY, so the raw tail is still useful
+// even if Docker returns no stdout/stderr at all.
+func (d *dockerRuntime) Logs(ctx context.Context, id string, tail int) (string, error) {
+	response, err := d.call(ctx, http.MethodGet, dockerAPIVersion+"/containers/"+url.PathEscape(id)+"/logs?stdout=true&stderr=true&tail="+strconv.Itoa(tail), nil, "", 64<<10)
+	if err != nil || response.status != http.StatusOK {
+		return "", errors.New("container log retrieval failed")
+	}
+	return strings.TrimSpace(string(response.body)), nil
 }
 
 func (d *dockerRuntime) Stats(ctx context.Context, id string) (agentwire.InstanceStats, error) {

internal/agent/agent_server.go

@@ -8,6 +8,8 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
+	"regexp"
+	"strings"
 	"syscall"
 	"time"
 
@@ -16,6 +18,8 @@ import (
 
 const maxDiskPaths = 16
 
+var diagnosticSecretPattern = regexp.MustCompile(`(?i)(password|token|secret|api[_-]?key)\s*[:=]\s*[^\s,;]+`)
+
 type service struct {
 	paths    *PathPolicy
 	registry *Registry
@@ -288,10 +292,25 @@ func (s *service) startInstance(w http.ResponseWriter, r *http.Request) {
 	}
 	if !state.Running {
 		if err := s.docker.Start(r.Context(), entry.ContainerID); err != nil {
-			writeProblem(w, http.StatusBadGateway, "start_failed", "The registered container could not be started.")
+			s.logger.Error("instance start failed", "event", "instance.start.failed", "instance_id", entry.InstanceID, "container_id", entry.ContainerID, "error", err)
+			writeDiagnosticProblem(w, http.StatusBadGateway, "start_failed", "The registered container could not be started.", s.containerDiagnostic(r.Context(), entry.ContainerID, err))
 			return
 		}
-		state.Running, state.Health = true, "starting"
+		inspection, inspectErr := s.docker.Inspect(r.Context(), entry.ContainerID)
+		if inspectErr != nil || (!inspection.Running && (inspection.Status != "" || inspection.FinishedAt != "")) {
+			s.logger.Error("instance exited after start", "event", "container.exited", "instance_id", entry.InstanceID, "container_id", entry.ContainerID, "error", err)
+			writeDiagnosticProblem(w, http.StatusBadGateway, "start_exited", "The registered container exited immediately after start.", s.containerDiagnostic(r.Context(), entry.ContainerID, inspectErr))
+			return
+		}
+		if inspection.Running {
+			state, err = s.boundState(r.Context(), entry)
+			if err != nil {
+				writeDiagnosticProblem(w, http.StatusBadGateway, "start_inspect_failed", "The started container could not be inspected.", s.containerDiagnostic(r.Context(), entry.ContainerID, err))
+				return
+			}
+		} else {
+			state.Running, state.Health = true, "starting"
+		}
 	}
 	writeJSON(w, http.StatusOK, state)
 }
@@ -403,6 +422,50 @@ func (s *service) bindingProblem(w http.ResponseWriter) {
 	writeProblem(w, http.StatusConflict, "registration_mismatch", "The registered container binding is invalid.")
 }
 
+// containerDiagnostic deliberately contains Docker state rather than request
+// payloads: it is safe to return over the authenticated private agent link and
+// remains useful when stdout/stderr is empty.
+func (s *service) containerDiagnostic(ctx context.Context, containerID string, cause error) map[string]any {
+	diagnostic := map[string]any{"container_id": containerID}
+	if cause != nil {
+		diagnostic["operation_error"] = redactDiagnostic(cause.Error())
+	}
+	inspection, err := s.docker.Inspect(ctx, containerID)
+	if err != nil {
+		diagnostic["inspect_error"] = redactDiagnostic(err.Error())
+		return diagnostic
+	}
+	diagnostic["state"] = inspection.Status
+	diagnostic["running"] = inspection.Running
+	diagnostic["restarting"] = inspection.Restarting
+	diagnostic["paused"] = inspection.Paused
+	diagnostic["oom_killed"] = inspection.OOMKilled
+	diagnostic["dead"] = inspection.Dead
+	diagnostic["exit_code"] = inspection.ExitCode
+	diagnostic["error"] = redactDiagnostic(inspection.Error)
+	diagnostic["started_at"] = inspection.StartedAt
+	diagnostic["finished_at"] = inspection.FinishedAt
+	diagnostic["health"] = inspection.Health
+	diagnostic["restart_count"] = inspection.RestartCount
+	if logs, logsErr := s.docker.Logs(ctx, containerID, 100); logsErr == nil {
+		diagnostic["logs_tail"] = redactDiagnostic(logs)
+	} else {
+		diagnostic["logs_error"] = redactDiagnostic(logsErr.Error())
+	}
+	return diagnostic
+}
+
+func redactDiagnostic(value string) string {
+	return diagnosticSecretPattern.ReplaceAllStringFunc(value, func(match string) string {
+		separator := "="
+		if strings.Contains(match, ":") {
+			separator = ":"
+		}
+		parts := strings.SplitN(match, separator, 2)
+		return parts[0] + separator + "[REDACTED]"
+	})
+}
+
 func (s *service) headers(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Cache-Control", "no-store")
@@ -431,8 +494,13 @@ func writeJSON(w http.ResponseWriter, status int, value any) {
 }
 
 func writeProblem(w http.ResponseWriter, status int, code, message string) {
+	writeDiagnosticProblem(w, status, code, message, nil)
+}
+
+func writeDiagnosticProblem(w http.ResponseWriter, status int, code, message string, details map[string]any) {
 	writeJSON(w, status, struct {
-		Code    string `json:"code"`
-		Message string `json:"message"`
-	}{Code: code, Message: message})
+		Code    string         `json:"code"`
+		Message string         `json:"message"`
+		Details map[string]any `json:"details,omitempty"`
+	}{Code: code, Message: message, Details: details})
 }

internal/agent/agent_server_test.go

@@ -53,6 +53,7 @@ func (d fakeDocker) Inspect(_ context.Context, id string) (agent.DockerInspectio
 	}
 	return inspection, nil
 }
+func (d fakeDocker) Logs(context.Context, string, int) (string, error) { return "", d.err }
 func (d fakeDocker) Stats(context.Context, string) (agentwire.InstanceStats, error) {
 	return agentwire.InstanceStats{MemoryBytes: 42}, d.err
 }

internal/agentclient/client.go

@@ -48,12 +48,17 @@ type ProblemError struct {
 	Status  int
 	Code    string
 	Message string
+	Details map[string]any `json:"details,omitempty"`
 }
 
 func (e *ProblemError) Error() string {
 	return fmt.Sprintf("agent request failed: %s (HTTP %d)", e.Code, e.Status)
 }
 
+// DiagnosticDetails lets the lifecycle service persist the agent's bounded
+// Docker inspection without exposing it to unprivileged HTTP clients.
+func (e *ProblemError) DiagnosticDetails() map[string]any { return e.Details }
+
 // New constructs a client. The base URL must not contain credentials, a query
 // or a path beyond an optional trailing slash.
 func New(baseURL string, secret []byte, httpClient *http.Client) (*Client, error) {

internal/instance/lifecycle.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"sync"
@@ -38,6 +39,23 @@ type OperationResult struct {
 	AgentState             agentwire.InstanceState `json:"agent_state,omitempty"`
 }
 
+// Diagnostic is a bounded, redacted technical record for an operation. It is
+// intentionally separate from audit: audit records who did something;
+// diagnostics retain why it failed.
+type Diagnostic struct {
+	OperationID string
+	InstanceID  string
+	Step        string
+	ErrorCode   string
+	Details     string
+	CreatedAt   string
+}
+
+type DiagnosticRepository interface {
+	RecordDiagnostic(context.Context, Diagnostic) error
+	ListDiagnostics(context.Context, string, int) ([]Diagnostic, error)
+}
+
 type LifecycleRepository interface {
 	GetInstance(context.Context, string) (StoredInstance, error)
 	ListLifecycleInstances(context.Context) ([]StoredInstance, error)
@@ -335,10 +353,42 @@ func (s *LifecycleService) inspectAndPersist(ctx context.Context, current Stored
 }
 
 func (s *LifecycleService) fail(ctx context.Context, operationID, instanceID, code string, cause error) (OperationResult, error) {
-	if err := s.repository.FailOperation(ctx, operationID, "error", code); err != nil {
+	stableCode := stableErrorCode(code)
+	if diagnostics, ok := s.repository.(DiagnosticRepository); ok {
+		_ = diagnostics.RecordDiagnostic(ctx, Diagnostic{OperationID: operationID, InstanceID: instanceID, Step: code, ErrorCode: stableCode, Details: diagnosticDetails(cause)})
+	}
+	if err := s.repository.FailOperation(ctx, operationID, "error", stableCode); err != nil {
 		return OperationResult{}, err
 	}
-	return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "error", Observed: "unknown"}, fmt.Errorf("%s: %w", code, cause)
+	return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "error", Observed: "unknown"}, fmt.Errorf("%s: %w", stableCode, cause)
+}
+
+func diagnosticDetails(cause error) string {
+	if cause == nil {
+		return ""
+	}
+	type detailed interface{ DiagnosticDetails() map[string]any }
+	var value detailed
+	if errors.As(cause, &value) {
+		encoded, err := json.Marshal(value.DiagnosticDetails())
+		if err == nil {
+			return string(encoded)
+		}
+	}
+	return cause.Error()
+}
+
+func stableErrorCode(step string) string {
+	switch step {
+	case "agent_start_failed", "agent_restart_failed":
+		return "DGM-START-001"
+	case "agent_create_failed":
+		return "DGM-DEPLOY-001"
+	case "invalid_plan":
+		return "DGM-DEPLOY-002"
+	default:
+		return "DGM-DOCKER-001"
+	}
 }
 
 func (s *LifecycleService) exclusive(instanceID string, action func() (OperationResult, error)) (OperationResult, error) {

internal/persistence/sqlite/catalog.go

@@ -350,6 +350,37 @@ func (r *Repository) FailOperation(ctx context.Context, operationID, lifecycleSt
 	return nil
 }
 
+func (r *Repository) RecordDiagnostic(ctx context.Context, value instance.Diagnostic) error {
+	_, err := r.db.ExecContext(ctx, `INSERT INTO operation_diagnostics(operation_id, instance_id, step, error_code, details, created_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(operation_id) DO UPDATE SET step=excluded.step, error_code=excluded.error_code, details=excluded.details, created_at=excluded.created_at`, value.OperationID, value.InstanceID, value.Step, value.ErrorCode, value.Details, r.now().UTC().Format(time.RFC3339Nano))
+	if err != nil {
+		return fmt.Errorf("record operation diagnostic: %w", err)
+	}
+	return nil
+}
+
+func (r *Repository) ListDiagnostics(ctx context.Context, instanceID string, limit int) ([]instance.Diagnostic, error) {
+	if limit < 1 || limit > 50 {
+		limit = 20
+	}
+	rows, err := r.db.QueryContext(ctx, `SELECT operation_id, instance_id, step, error_code, details, created_at FROM operation_diagnostics WHERE instance_id=? ORDER BY created_at DESC LIMIT ?`, instanceID, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list operation diagnostics: %w", err)
+	}
+	defer rows.Close()
+	var result []instance.Diagnostic
+	for rows.Next() {
+		var value instance.Diagnostic
+		if err := rows.Scan(&value.OperationID, &value.InstanceID, &value.Step, &value.ErrorCode, &value.Details, &value.CreatedAt); err != nil {
+			return nil, fmt.Errorf("scan operation diagnostic: %w", err)
+		}
+		result = append(result, value)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate operation diagnostics: %w", err)
+	}
+	return result, nil
+}
+
 func (r *Repository) UpdateObservation(ctx context.Context, instanceID, lifecycleState, observedState, containerID string, desiredRunning bool, errorCode string) error {
 	desired := 0
 	if desiredRunning {

internal/persistence/sqlite/store.go

@@ -57,10 +57,27 @@ func initialize(ctx context.Context, db *sql.DB) error {
 		return fmt.Errorf("inspect sqlite schema: %w", err)
 	}
 	if existing == 1 {
-		return nil
+		return ensureDiagnosticSchema(ctx, db)
 	}
 	if _, err := db.ExecContext(ctx, schema); err != nil {
 		return fmt.Errorf("initialize sqlite schema: %w", err)
 	}
+	return ensureDiagnosticSchema(ctx, db)
+}
+
+// This additive migration is safe for existing V1 databases and keeps the
+// original embedded schema immutable for fresh installs.
+func ensureDiagnosticSchema(ctx context.Context, db *sql.DB) error {
+	_, err := db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS operation_diagnostics (
+        operation_id TEXT PRIMARY KEY REFERENCES instance_operations(id) ON DELETE CASCADE,
+        instance_id TEXT NOT NULL REFERENCES instances(id) ON DELETE CASCADE,
+        step TEXT NOT NULL,
+        error_code TEXT NOT NULL,
+        details TEXT NOT NULL,
+        created_at TEXT NOT NULL
+    ); CREATE INDEX IF NOT EXISTS operation_diagnostics_instance_idx ON operation_diagnostics(instance_id, created_at DESC);`)
+	if err != nil {
+		return fmt.Errorf("migrate diagnostic schema: %w", err)
+	}
 	return nil
 }

internal/web/server.go

@@ -1025,6 +1025,9 @@ func (s *server) instanceDeleteContainer(w http.ResponseWriter, r *http.Request)
 
 func (s *server) lifecycleProblem(w http.ResponseWriter, err error) {
 	status, code := http.StatusBadGateway, "lifecycle_failed"
+	if strings.HasPrefix(err.Error(), "DGM-") {
+		code = strings.SplitN(err.Error(), ":", 2)[0]
+	}
 	switch {
 	case errors.Is(err, instance.ErrInstanceNotFound):
 		status, code = http.StatusNotFound, "instance_not_found"
@@ -1034,6 +1037,7 @@ func (s *server) lifecycleProblem(w http.ResponseWriter, err error) {
 		status, code = http.StatusConflict, "invalid_instance_state"
 	}
 	s.apiProblem(w, status, code, "The instance operation could not be completed.")
+	s.logger.Error("instance lifecycle failed", "event", "instance.lifecycle.failed", "error_code", code, "error", err)
 }
 
 func (s *server) buildAPIPreview(w http.ResponseWriter, r *http.Request) (previewAPIRequest, instance.Preview, bool) {