602 lines
23 KiB
Go
602 lines
23 KiB
Go
package instance
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire"
|
|
)
|
|
|
|
var (
|
|
ErrInstanceNotFound = errors.New("instance not found")
|
|
ErrOperationConflict = errors.New("instance operation conflict")
|
|
ErrInvalidState = errors.New("invalid instance lifecycle state")
|
|
)
|
|
|
|
const startupVerificationDelay = 3 * time.Second
|
|
|
|
type StoredInstance struct {
|
|
ID string
|
|
Preview Preview
|
|
LifecycleState string
|
|
ObservedState string
|
|
ContainerID string
|
|
PlanDigest string
|
|
DesiredRunning bool
|
|
ContainerConfigPending bool
|
|
}
|
|
|
|
type OperationResult struct {
|
|
OperationID string `json:"operation_id,omitempty"`
|
|
InstanceID string `json:"instance_id"`
|
|
State string `json:"state"`
|
|
Observed string `json:"observed_state"`
|
|
ContainerID string `json:"container_id,omitempty"`
|
|
ContainerConfigPending bool `json:"container_config_pending"`
|
|
AgentState agentwire.InstanceState `json:"agent_state,omitempty"`
|
|
}
|
|
|
|
// Diagnostic is a bounded, redacted technical record for an operation. It is
|
|
// intentionally separate from audit: audit records who did something;
|
|
// diagnostics retain why it failed.
|
|
type Diagnostic struct {
|
|
OperationID string
|
|
InstanceID string
|
|
Step string
|
|
ErrorCode string
|
|
Details string
|
|
CreatedAt string
|
|
}
|
|
|
|
type OperationHistory struct {
|
|
OperationID string `json:"operation_id"`
|
|
Kind string `json:"kind"`
|
|
State string `json:"state"`
|
|
Phase string `json:"phase"`
|
|
ErrorCode string `json:"error_code,omitempty"`
|
|
CreatedAt string `json:"created_at"`
|
|
CompletedAt string `json:"completed_at,omitempty"`
|
|
Diagnostic *Diagnostic `json:"diagnostic,omitempty"`
|
|
}
|
|
|
|
// OperationProgress is the non-technical, persisted view consumed by the UI.
|
|
// Diagnostics deliberately remain a separate administrator-only resource.
|
|
type OperationProgress struct {
|
|
OperationID string `json:"operation_id"`
|
|
OperationType string `json:"operation_type"`
|
|
Game string `json:"game"`
|
|
InstanceID string `json:"instance_id,omitempty"`
|
|
GlobalStatus string `json:"global_status"`
|
|
CurrentStep string `json:"current_step,omitempty"`
|
|
Steps []OperationStep `json:"steps"`
|
|
ErrorCode string `json:"error_code,omitempty"`
|
|
StartedAt string `json:"started_at"`
|
|
FinishedAt string `json:"finished_at,omitempty"`
|
|
}
|
|
type OperationStep struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
type ProgressRepository interface {
|
|
SetOperationStep(context.Context, string, string, string) error
|
|
GetOperationProgress(context.Context, string) (OperationProgress, error)
|
|
}
|
|
|
|
type DiagnosticRepository interface {
|
|
RecordDiagnostic(context.Context, Diagnostic) error
|
|
ListDiagnostics(context.Context, string, int) ([]Diagnostic, error)
|
|
ListOperationHistory(context.Context, string, int) ([]OperationHistory, error)
|
|
}
|
|
|
|
type LifecycleRepository interface {
|
|
GetInstance(context.Context, string) (StoredInstance, error)
|
|
ListLifecycleInstances(context.Context) ([]StoredInstance, error)
|
|
BeginOperation(context.Context, string, string, string, string) (StoredInstance, error)
|
|
FinishOperation(context.Context, string, string, string, string, string, bool, string) error
|
|
FailOperation(context.Context, string, string, string) error
|
|
UpdateObservation(context.Context, string, string, string, string, bool, string) error
|
|
RecoverInterruptedOperations(context.Context) error
|
|
}
|
|
|
|
type LifecycleAgent interface {
|
|
CreateInstance(context.Context, agentwire.DeploymentPlan) (agentwire.InstanceState, error)
|
|
InspectInstance(context.Context, string) (agentwire.InstanceState, error)
|
|
StartInstance(context.Context, string) (agentwire.InstanceState, error)
|
|
StopInstance(context.Context, string, int) (agentwire.InstanceState, error)
|
|
RestartInstance(context.Context, string, int) (agentwire.InstanceState, error)
|
|
DeleteContainer(context.Context, string) error
|
|
GetInstanceStats(context.Context, string) (agentwire.InstanceStats, error)
|
|
}
|
|
|
|
type replacementAgent interface {
|
|
ReplaceInstance(context.Context, agentwire.DeploymentPlan) (agentwire.InstanceState, error)
|
|
}
|
|
|
|
type LifecycleService struct {
|
|
repository LifecycleRepository
|
|
agent LifecycleAgent
|
|
locks sync.Map
|
|
}
|
|
|
|
// AgentHealthy checks the already-configured restricted agent when it exposes
|
|
// its health endpoint. Older or test-only agents are reported as unavailable.
|
|
func (s *LifecycleService) AgentHealthy(ctx context.Context) error {
|
|
agent, ok := s.agent.(interface{ Health(context.Context) error })
|
|
if !ok {
|
|
return errors.New("agent health unavailable")
|
|
}
|
|
return agent.Health(ctx)
|
|
}
|
|
|
|
func NewLifecycleService(repository LifecycleRepository, agent LifecycleAgent) *LifecycleService {
|
|
return &LifecycleService{repository: repository, agent: agent}
|
|
}
|
|
|
|
func (s *LifecycleService) Install(ctx context.Context, instanceID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if current.LifecycleState != "draft" && current.ContainerID != "" {
|
|
return resultFrom(current, ""), nil
|
|
}
|
|
operationID, err := operationToken()
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "install", "installing")
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
plan, err := deploymentPlan(ctx, s.repository, current)
|
|
if err != nil {
|
|
return s.fail(ctx, operationID, instanceID, "invalid_plan", err)
|
|
}
|
|
state, err := s.agent.CreateInstance(ctx, plan)
|
|
if err != nil {
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "installation", "failed")
|
|
}
|
|
return s.fail(ctx, operationID, instanceID, "agent_create_failed", err)
|
|
}
|
|
if err := s.repository.FinishOperation(ctx, operationID, "stopped", "stopped", state.ContainerID, plan.PlanDigest, false, ""); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "stopped", Observed: "stopped", ContainerID: state.ContainerID, AgentState: state}, nil
|
|
})
|
|
}
|
|
|
|
// BeginInstall allocates the existing operation identifier before any long
|
|
// running work. Callers may safely return it to HTTP clients and run InstallOperation later.
|
|
func (s *LifecycleService) BeginInstall(ctx context.Context, instanceID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
operationID, err := operationToken()
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
current, err := s.repository.BeginOperation(ctx, operationID, instanceID, "install", "installing")
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return resultFrom(current, operationID), nil
|
|
})
|
|
}
|
|
|
|
// InstallOperation continues an operation already created by BeginInstall.
|
|
func (s *LifecycleService) InstallOperation(ctx context.Context, instanceID, operationID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "installation", "running")
|
|
}
|
|
plan, err := deploymentPlan(ctx, s.repository, current)
|
|
if err != nil {
|
|
return s.fail(ctx, operationID, instanceID, "invalid_plan", err)
|
|
}
|
|
state, err := s.agent.CreateInstance(ctx, plan)
|
|
if err != nil {
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "installation", "failed")
|
|
}
|
|
return s.fail(ctx, operationID, instanceID, "agent_create_failed", err)
|
|
}
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "installation", "success")
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "stopped", Observed: "stopped", ContainerID: state.ContainerID, AgentState: state}, nil
|
|
})
|
|
}
|
|
|
|
// CompleteInstall records the final state after the deployment worker has run
|
|
// imports, configuration and the start action under the same operation id.
|
|
func (s *LifecycleService) CompleteInstall(ctx context.Context, result OperationResult) error {
|
|
current, err := s.repository.GetInstance(ctx, result.InstanceID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.repository.FinishOperation(ctx, result.OperationID, result.State, result.Observed, result.ContainerID, current.PlanDigest, true, "")
|
|
}
|
|
|
|
// StartInstall starts the container without allocating a second operation.
|
|
func (s *LifecycleService) StartInstall(ctx context.Context, instanceID, operationID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
if _, err := s.repository.GetInstance(ctx, instanceID); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "starting", "running")
|
|
}
|
|
state, err := s.agent.StartInstance(ctx, instanceID)
|
|
if err != nil {
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "starting", "failed")
|
|
}
|
|
return s.fail(ctx, operationID, instanceID, "agent_start_failed", err)
|
|
}
|
|
lifecycle, observed := stateToLifecycle(state)
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "starting", "success")
|
|
_ = progress.SetOperationStep(ctx, operationID, "verification", "running")
|
|
}
|
|
verified, verifyErr := s.verifyStarted(ctx, instanceID)
|
|
if verifyErr != nil {
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "verification", "failed")
|
|
}
|
|
return s.fail(ctx, operationID, instanceID, "verification_failed", verifyErr)
|
|
}
|
|
lifecycle, observed = stateToLifecycle(verified)
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, "verification", "success")
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: lifecycle, Observed: observed, ContainerID: verified.ContainerID, AgentState: verified}, nil
|
|
})
|
|
}
|
|
|
|
func (s *LifecycleService) verifyStarted(ctx context.Context, instanceID string) (agentwire.InstanceState, error) {
|
|
timer := time.NewTimer(startupVerificationDelay)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
return agentwire.InstanceState{}, ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
state, err := s.agent.InspectInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return agentwire.InstanceState{}, err
|
|
}
|
|
if !state.Running || state.Health == "unhealthy" {
|
|
return agentwire.InstanceState{}, errors.New("container did not remain running during startup verification")
|
|
}
|
|
return state, nil
|
|
}
|
|
|
|
func (s *LifecycleService) FailInstall(ctx context.Context, operationID, instanceID, step string, cause error) (OperationResult, error) {
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, step, "failed")
|
|
}
|
|
return s.fail(ctx, operationID, instanceID, step, cause)
|
|
}
|
|
|
|
func (s *LifecycleService) MarkStep(ctx context.Context, operationID, step, status string) {
|
|
if p, ok := s.repository.(ProgressRepository); ok {
|
|
_ = p.SetOperationStep(ctx, operationID, step, status)
|
|
}
|
|
}
|
|
|
|
func deploymentPlan(ctx context.Context, repository LifecycleRepository, current StoredInstance) (agentwire.DeploymentPlan, error) {
|
|
if secrets, ok := repository.(SecretRepository); ok && (len(current.Preview.ResolvedConfiguration.SecretEnvironment) != 0 || len(current.Preview.ResolvedConfiguration.SecretArguments) != 0) {
|
|
values, err := secrets.LoadInstanceSecrets(ctx, current.ID)
|
|
if err != nil {
|
|
return agentwire.DeploymentPlan{}, err
|
|
}
|
|
return current.Preview.DeploymentPlanWithSecrets(current.ID, values)
|
|
}
|
|
return current.Preview.DeploymentPlan(current.ID)
|
|
}
|
|
|
|
func (s *LifecycleService) Start(ctx context.Context, instanceID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if current.ContainerID == "" {
|
|
return OperationResult{}, ErrInvalidState
|
|
}
|
|
if current.DesiredRunning && (current.LifecycleState == "online" || current.LifecycleState == "starting" || current.LifecycleState == "degraded") {
|
|
return s.inspectAndPersist(ctx, current, "")
|
|
}
|
|
operationID, err := operationToken()
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "start", "starting")
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
var state agentwire.InstanceState
|
|
if current.ContainerConfigPending {
|
|
replacement, ok := s.agent.(replacementAgent)
|
|
if !ok {
|
|
return s.fail(ctx, operationID, instanceID, "container_replace_unavailable", errors.New("container replacement is unavailable"))
|
|
}
|
|
plan, planErr := current.Preview.DeploymentPlan(instanceID)
|
|
if planErr != nil {
|
|
return s.fail(ctx, operationID, instanceID, "invalid_plan", planErr)
|
|
}
|
|
state, err = replacement.ReplaceInstance(ctx, plan)
|
|
if err == nil {
|
|
err = s.repository.(ConfigurationRepository).ClearContainerConfigPending(ctx, instanceID, plan.PlanDigest)
|
|
}
|
|
}
|
|
if err == nil {
|
|
state, err = s.agent.StartInstance(ctx, instanceID)
|
|
}
|
|
if err != nil {
|
|
return s.fail(ctx, operationID, instanceID, "agent_start_failed", err)
|
|
}
|
|
lifecycle, observed := stateToLifecycle(state)
|
|
if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, true, ""); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: lifecycle, Observed: observed, ContainerID: state.ContainerID, AgentState: state}, nil
|
|
})
|
|
}
|
|
|
|
func (s *LifecycleService) Stop(ctx context.Context, instanceID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if current.ContainerID == "" {
|
|
return OperationResult{}, ErrInvalidState
|
|
}
|
|
if !current.DesiredRunning && current.LifecycleState == "stopped" {
|
|
return resultFrom(current, ""), nil
|
|
}
|
|
operationID, err := operationToken()
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "stop", "stopping")
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
state, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds)
|
|
if err != nil {
|
|
return s.fail(ctx, operationID, instanceID, "agent_stop_failed", err)
|
|
}
|
|
if err := s.repository.FinishOperation(ctx, operationID, "stopped", "stopped", state.ContainerID, current.PlanDigest, false, ""); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "stopped", Observed: "stopped", ContainerID: state.ContainerID, AgentState: state}, nil
|
|
})
|
|
}
|
|
|
|
func (s *LifecycleService) Restart(ctx context.Context, instanceID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if current.ContainerID == "" {
|
|
return OperationResult{}, ErrInvalidState
|
|
}
|
|
operationID, err := operationToken()
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "restart", "starting")
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
state, err := s.agent.RestartInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds)
|
|
if err != nil {
|
|
return s.fail(ctx, operationID, instanceID, "agent_restart_failed", err)
|
|
}
|
|
lifecycle, observed := stateToLifecycle(state)
|
|
if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, true, ""); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: lifecycle, Observed: observed, ContainerID: state.ContainerID, AgentState: state}, nil
|
|
})
|
|
}
|
|
|
|
func (s *LifecycleService) DeleteContainer(ctx context.Context, instanceID string) (OperationResult, error) {
|
|
return s.exclusive(instanceID, func() (OperationResult, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if current.ContainerID == "" {
|
|
return resultFrom(current, ""), nil
|
|
}
|
|
if current.DesiredRunning || (current.LifecycleState != "stopped" && current.LifecycleState != "deleting" && current.LifecycleState != "intervention_required") {
|
|
return OperationResult{}, ErrInvalidState
|
|
}
|
|
operationID, err := operationToken()
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
_, err = s.repository.BeginOperation(ctx, operationID, instanceID, "delete_container", "deleting")
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if err := s.agent.DeleteContainer(ctx, instanceID); err != nil {
|
|
return s.fail(ctx, operationID, instanceID, "agent_delete_failed", err)
|
|
}
|
|
if err := s.repository.FinishOperation(ctx, operationID, "unknown", "missing", "", current.PlanDigest, false, ""); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "unknown", Observed: "missing"}, nil
|
|
})
|
|
}
|
|
|
|
func (s *LifecycleService) Inspect(ctx context.Context, instanceID string) (OperationResult, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
if current.ContainerID == "" {
|
|
return resultFrom(current, ""), nil
|
|
}
|
|
return s.inspectAndPersist(ctx, current, "")
|
|
}
|
|
|
|
func (s *LifecycleService) Stats(ctx context.Context, instanceID string) (agentwire.InstanceStats, error) {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return agentwire.InstanceStats{}, err
|
|
}
|
|
if current.ContainerID == "" {
|
|
return agentwire.InstanceStats{}, ErrInvalidState
|
|
}
|
|
return s.agent.GetInstanceStats(ctx, instanceID)
|
|
}
|
|
|
|
func (s *LifecycleService) ReconcileAll(ctx context.Context) error {
|
|
instances, err := s.repository.ListLifecycleInstances(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, current := range instances {
|
|
if current.ContainerID == "" {
|
|
continue
|
|
}
|
|
if _, err := s.inspectAndPersist(ctx, current, ""); err != nil {
|
|
operationID, tokenErr := operationToken()
|
|
if tokenErr != nil {
|
|
return tokenErr
|
|
}
|
|
if _, beginErr := s.repository.BeginOperation(ctx, operationID, current.ID, "reconcile", "unknown"); beginErr == nil {
|
|
_ = s.repository.FailOperation(ctx, operationID, "unknown", "agent_reconcile_failed")
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *LifecycleService) RecoverInterruptedOperations(ctx context.Context) error {
|
|
return s.repository.RecoverInterruptedOperations(ctx)
|
|
}
|
|
|
|
func (s *LifecycleService) inspectAndPersist(ctx context.Context, current StoredInstance, operationID string) (OperationResult, error) {
|
|
state, err := s.agent.InspectInstance(ctx, current.ID)
|
|
if err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
lifecycle, observed := stateToLifecycle(state)
|
|
if current.LifecycleState == "intervention_required" {
|
|
lifecycle = "intervention_required"
|
|
}
|
|
if !current.DesiredRunning && !state.Running {
|
|
lifecycle = "stopped"
|
|
}
|
|
if operationID != "" {
|
|
if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, current.DesiredRunning, ""); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
} else if err := s.repository.UpdateObservation(ctx, current.ID, lifecycle, observed, state.ContainerID, current.DesiredRunning, ""); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: current.ID, State: lifecycle, Observed: observed, ContainerID: state.ContainerID, AgentState: state}, nil
|
|
}
|
|
|
|
func (s *LifecycleService) fail(ctx context.Context, operationID, instanceID, code string, cause error) (OperationResult, error) {
|
|
stableCode := stableErrorCode(code)
|
|
if progress, ok := s.repository.(ProgressRepository); ok {
|
|
_ = progress.SetOperationStep(ctx, operationID, code, "failed")
|
|
}
|
|
if diagnostics, ok := s.repository.(DiagnosticRepository); ok {
|
|
_ = diagnostics.RecordDiagnostic(ctx, Diagnostic{OperationID: operationID, InstanceID: instanceID, Step: code, ErrorCode: stableCode, Details: diagnosticDetails(operationID, code, stableCode, cause)})
|
|
}
|
|
if err := s.repository.FailOperation(ctx, operationID, "error", stableCode); err != nil {
|
|
return OperationResult{}, err
|
|
}
|
|
return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "error", Observed: "unknown"}, fmt.Errorf("%s: %w", stableCode, cause)
|
|
}
|
|
|
|
func diagnosticDetails(operationID, step, stableCode string, cause error) string {
|
|
details := map[string]any{
|
|
"operation_id": operationID,
|
|
"step": step,
|
|
"code": stableCode,
|
|
}
|
|
type detailed interface{ DiagnosticDetails() map[string]any }
|
|
var value detailed
|
|
if errors.As(cause, &value) {
|
|
for key, detail := range value.DiagnosticDetails() {
|
|
details[key] = detail
|
|
}
|
|
} else if cause != nil {
|
|
details["operation_error"] = cause.Error()
|
|
}
|
|
if _, exists := details["phase"]; !exists {
|
|
details["phase"] = step
|
|
}
|
|
encoded, err := json.Marshal(details)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return string(encoded)
|
|
}
|
|
|
|
func stableErrorCode(step string) string {
|
|
switch step {
|
|
case "agent_start_failed", "agent_restart_failed", "verification_failed":
|
|
return "DGM-START-001"
|
|
case "agent_create_failed":
|
|
return "DGM-DEPLOY-001"
|
|
case "invalid_plan":
|
|
return "DGM-DEPLOY-002"
|
|
default:
|
|
return "DGM-DOCKER-001"
|
|
}
|
|
}
|
|
|
|
func (s *LifecycleService) exclusive(instanceID string, action func() (OperationResult, error)) (OperationResult, error) {
|
|
lockValue, _ := s.locks.LoadOrStore(instanceID, &sync.Mutex{})
|
|
lock := lockValue.(*sync.Mutex)
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
return action()
|
|
}
|
|
|
|
func stateToLifecycle(state agentwire.InstanceState) (string, string) {
|
|
if !state.Running {
|
|
return "stopped", "stopped"
|
|
}
|
|
if state.Ready {
|
|
return "online", "ready"
|
|
}
|
|
if state.Health == "unhealthy" || state.Health == "none" {
|
|
return "degraded", "degraded"
|
|
}
|
|
return "starting", "running"
|
|
}
|
|
|
|
func resultFrom(current StoredInstance, operationID string) OperationResult {
|
|
return OperationResult{OperationID: operationID, InstanceID: current.ID, State: current.LifecycleState, Observed: current.ObservedState, ContainerID: current.ContainerID, ContainerConfigPending: current.ContainerConfigPending}
|
|
}
|
|
|
|
func operationToken() (string, error) {
|
|
buffer := make([]byte, 24)
|
|
if _, err := rand.Read(buffer); err != nil {
|
|
return "", fmt.Errorf("generate operation ID: %w", err)
|
|
}
|
|
return base64.RawURLEncoding.EncodeToString(buffer), nil
|
|
}
|