DoGaMa-serv/internal/instance/lifecycle.go

package instance

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"fmt"
	"sync"

	"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire"
)

var (
	ErrInstanceNotFound  = errors.New("instance not found")
	ErrOperationConflict = errors.New("instance operation conflict")
	ErrInvalidState      = errors.New("invalid instance lifecycle state")
)

type StoredInstance struct {
	ID             string
	Preview        Preview
	LifecycleState string
	ObservedState  string
	ContainerID    string
	PlanDigest     string
	DesiredRunning bool
	ContainerConfigPending bool
}

type OperationResult struct {
	OperationID string                  `json:"operation_id,omitempty"`
	InstanceID  string                  `json:"instance_id"`
	State       string                  `json:"state"`
	Observed    string                  `json:"observed_state"`
	ContainerID string                  `json:"container_id,omitempty"`
	ContainerConfigPending bool         `json:"container_config_pending"`
	AgentState  agentwire.InstanceState `json:"agent_state,omitempty"`
}

type LifecycleRepository interface {
	GetInstance(context.Context, string) (StoredInstance, error)
	ListLifecycleInstances(context.Context) ([]StoredInstance, error)
	BeginOperation(context.Context, string, string, string, string) (StoredInstance, error)
	FinishOperation(context.Context, string, string, string, string, string, bool, string) error
	FailOperation(context.Context, string, string, string) error
	UpdateObservation(context.Context, string, string, string, string, bool, string) error
	RecoverInterruptedOperations(context.Context) error
}

type LifecycleAgent interface {
	CreateInstance(context.Context, agentwire.DeploymentPlan) (agentwire.InstanceState, error)
	InspectInstance(context.Context, string) (agentwire.InstanceState, error)
	StartInstance(context.Context, string) (agentwire.InstanceState, error)
	StopInstance(context.Context, string, int) (agentwire.InstanceState, error)
	RestartInstance(context.Context, string, int) (agentwire.InstanceState, error)
	DeleteContainer(context.Context, string) error
	GetInstanceStats(context.Context, string) (agentwire.InstanceStats, error)
}

type replacementAgent interface { ReplaceInstance(context.Context, agentwire.DeploymentPlan) (agentwire.InstanceState, error) }

type LifecycleService struct {
	repository LifecycleRepository
	agent      LifecycleAgent
	locks      sync.Map
}

func NewLifecycleService(repository LifecycleRepository, agent LifecycleAgent) *LifecycleService {
	return &LifecycleService{repository: repository, agent: agent}
}

func (s *LifecycleService) Install(ctx context.Context, instanceID string) (OperationResult, error) {
	return s.exclusive(instanceID, func() (OperationResult, error) {
		current, err := s.repository.GetInstance(ctx, instanceID)
		if err != nil {
			return OperationResult{}, err
		}
		if current.LifecycleState != "draft" && current.ContainerID != "" {
			return resultFrom(current, ""), nil
		}
		operationID, err := operationToken()
		if err != nil {
			return OperationResult{}, err
		}
		current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "install", "installing")
		if err != nil {
			return OperationResult{}, err
		}
		plan, err := current.Preview.DeploymentPlan(instanceID)
		if err != nil {
			return s.fail(ctx, operationID, instanceID, "invalid_plan", err)
		}
		state, err := s.agent.CreateInstance(ctx, plan)
		if err != nil {
			return s.fail(ctx, operationID, instanceID, "agent_create_failed", err)
		}
		if err := s.repository.FinishOperation(ctx, operationID, "stopped", "stopped", state.ContainerID, plan.PlanDigest, false, ""); err != nil {
			return OperationResult{}, err
		}
		return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "stopped", Observed: "stopped", ContainerID: state.ContainerID, AgentState: state}, nil
	})
}

func (s *LifecycleService) Start(ctx context.Context, instanceID string) (OperationResult, error) {
	return s.exclusive(instanceID, func() (OperationResult, error) {
		current, err := s.repository.GetInstance(ctx, instanceID)
		if err != nil {
			return OperationResult{}, err
		}
		if current.ContainerID == "" {
			return OperationResult{}, ErrInvalidState
		}
		if current.DesiredRunning && (current.LifecycleState == "online" || current.LifecycleState == "starting" || current.LifecycleState == "degraded") {
			return s.inspectAndPersist(ctx, current, "")
		}
		operationID, err := operationToken()
		if err != nil {
			return OperationResult{}, err
		}
		current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "start", "starting")
		if err != nil {
			return OperationResult{}, err
		}
		var state agentwire.InstanceState
		if current.ContainerConfigPending {
			replacement, ok := s.agent.(replacementAgent)
			if !ok { return s.fail(ctx, operationID, instanceID, "container_replace_unavailable", errors.New("container replacement is unavailable")) }
			plan, planErr := current.Preview.DeploymentPlan(instanceID)
			if planErr != nil { return s.fail(ctx, operationID, instanceID, "invalid_plan", planErr) }
			state, err = replacement.ReplaceInstance(ctx, plan)
			if err == nil { err = s.repository.(ConfigurationRepository).ClearContainerConfigPending(ctx, instanceID, plan.PlanDigest) }
		}
		if err == nil { state, err = s.agent.StartInstance(ctx, instanceID) }
		if err != nil {
			return s.fail(ctx, operationID, instanceID, "agent_start_failed", err)
		}
		lifecycle, observed := stateToLifecycle(state)
		if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, true, ""); err != nil {
			return OperationResult{}, err
		}
		return OperationResult{OperationID: operationID, InstanceID: instanceID, State: lifecycle, Observed: observed, ContainerID: state.ContainerID, AgentState: state}, nil
	})
}

func (s *LifecycleService) Stop(ctx context.Context, instanceID string) (OperationResult, error) {
	return s.exclusive(instanceID, func() (OperationResult, error) {
		current, err := s.repository.GetInstance(ctx, instanceID)
		if err != nil {
			return OperationResult{}, err
		}
		if current.ContainerID == "" {
			return OperationResult{}, ErrInvalidState
		}
		if !current.DesiredRunning && current.LifecycleState == "stopped" {
			return resultFrom(current, ""), nil
		}
		operationID, err := operationToken()
		if err != nil {
			return OperationResult{}, err
		}
		current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "stop", "stopping")
		if err != nil {
			return OperationResult{}, err
		}
		state, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds)
		if err != nil {
			return s.fail(ctx, operationID, instanceID, "agent_stop_failed", err)
		}
		if err := s.repository.FinishOperation(ctx, operationID, "stopped", "stopped", state.ContainerID, current.PlanDigest, false, ""); err != nil {
			return OperationResult{}, err
		}
		return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "stopped", Observed: "stopped", ContainerID: state.ContainerID, AgentState: state}, nil
	})
}

func (s *LifecycleService) Restart(ctx context.Context, instanceID string) (OperationResult, error) {
	return s.exclusive(instanceID, func() (OperationResult, error) {
		current, err := s.repository.GetInstance(ctx, instanceID)
		if err != nil {
			return OperationResult{}, err
		}
		if current.ContainerID == "" {
			return OperationResult{}, ErrInvalidState
		}
		operationID, err := operationToken()
		if err != nil {
			return OperationResult{}, err
		}
		current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "restart", "starting")
		if err != nil {
			return OperationResult{}, err
		}
		state, err := s.agent.RestartInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds)
		if err != nil {
			return s.fail(ctx, operationID, instanceID, "agent_restart_failed", err)
		}
		lifecycle, observed := stateToLifecycle(state)
		if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, true, ""); err != nil {
			return OperationResult{}, err
		}
		return OperationResult{OperationID: operationID, InstanceID: instanceID, State: lifecycle, Observed: observed, ContainerID: state.ContainerID, AgentState: state}, nil
	})
}

func (s *LifecycleService) DeleteContainer(ctx context.Context, instanceID string) (OperationResult, error) {
	return s.exclusive(instanceID, func() (OperationResult, error) {
		current, err := s.repository.GetInstance(ctx, instanceID)
		if err != nil {
			return OperationResult{}, err
		}
		if current.ContainerID == "" {
			return resultFrom(current, ""), nil
		}
		if current.DesiredRunning || (current.LifecycleState != "stopped" && current.LifecycleState != "deleting" && current.LifecycleState != "intervention_required") {
			return OperationResult{}, ErrInvalidState
		}
		operationID, err := operationToken()
		if err != nil {
			return OperationResult{}, err
		}
		_, err = s.repository.BeginOperation(ctx, operationID, instanceID, "delete_container", "deleting")
		if err != nil {
			return OperationResult{}, err
		}
		if err := s.agent.DeleteContainer(ctx, instanceID); err != nil {
			return s.fail(ctx, operationID, instanceID, "agent_delete_failed", err)
		}
		if err := s.repository.FinishOperation(ctx, operationID, "unknown", "missing", "", current.PlanDigest, false, ""); err != nil {
			return OperationResult{}, err
		}
		return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "unknown", Observed: "missing"}, nil
	})
}

func (s *LifecycleService) Inspect(ctx context.Context, instanceID string) (OperationResult, error) {
	current, err := s.repository.GetInstance(ctx, instanceID)
	if err != nil {
		return OperationResult{}, err
	}
	if current.ContainerID == "" {
		return resultFrom(current, ""), nil
	}
	return s.inspectAndPersist(ctx, current, "")
}

func (s *LifecycleService) Stats(ctx context.Context, instanceID string) (agentwire.InstanceStats, error) {
	current, err := s.repository.GetInstance(ctx, instanceID)
	if err != nil {
		return agentwire.InstanceStats{}, err
	}
	if current.ContainerID == "" {
		return agentwire.InstanceStats{}, ErrInvalidState
	}
	return s.agent.GetInstanceStats(ctx, instanceID)
}

func (s *LifecycleService) ReconcileAll(ctx context.Context) error {
	instances, err := s.repository.ListLifecycleInstances(ctx)
	if err != nil {
		return err
	}
	for _, current := range instances {
		if current.ContainerID == "" {
			continue
		}
		if _, err := s.inspectAndPersist(ctx, current, ""); err != nil {
			operationID, tokenErr := operationToken()
			if tokenErr != nil {
				return tokenErr
			}
			if _, beginErr := s.repository.BeginOperation(ctx, operationID, current.ID, "reconcile", "unknown"); beginErr == nil {
				_ = s.repository.FailOperation(ctx, operationID, "unknown", "agent_reconcile_failed")
			}
		}
	}
	return nil
}

func (s *LifecycleService) RecoverInterruptedOperations(ctx context.Context) error {
	return s.repository.RecoverInterruptedOperations(ctx)
}

func (s *LifecycleService) inspectAndPersist(ctx context.Context, current StoredInstance, operationID string) (OperationResult, error) {
	state, err := s.agent.InspectInstance(ctx, current.ID)
	if err != nil {
		return OperationResult{}, err
	}
	lifecycle, observed := stateToLifecycle(state)
	if current.LifecycleState == "intervention_required" {
		lifecycle = "intervention_required"
	}
	if !current.DesiredRunning && !state.Running {
		lifecycle = "stopped"
	}
	if operationID != "" {
		if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, current.DesiredRunning, ""); err != nil {
			return OperationResult{}, err
		}
	} else if err := s.repository.UpdateObservation(ctx, current.ID, lifecycle, observed, state.ContainerID, current.DesiredRunning, ""); err != nil {
		return OperationResult{}, err
	}
	return OperationResult{OperationID: operationID, InstanceID: current.ID, State: lifecycle, Observed: observed, ContainerID: state.ContainerID, AgentState: state}, nil
}

func (s *LifecycleService) fail(ctx context.Context, operationID, instanceID, code string, cause error) (OperationResult, error) {
	if err := s.repository.FailOperation(ctx, operationID, "error", code); err != nil {
		return OperationResult{}, err
	}
	return OperationResult{OperationID: operationID, InstanceID: instanceID, State: "error", Observed: "unknown"}, fmt.Errorf("%s: %w", code, cause)
}

func (s *LifecycleService) exclusive(instanceID string, action func() (OperationResult, error)) (OperationResult, error) {
	lockValue, _ := s.locks.LoadOrStore(instanceID, &sync.Mutex{})
	lock := lockValue.(*sync.Mutex)
	lock.Lock()
	defer lock.Unlock()
	return action()
}

func stateToLifecycle(state agentwire.InstanceState) (string, string) {
	if !state.Running {
		return "stopped", "stopped"
	}
	if state.Ready {
		return "online", "ready"
	}
	if state.Health == "unhealthy" || state.Health == "none" {
		return "degraded", "degraded"
	}
	return "starting", "running"
}

func resultFrom(current StoredInstance, operationID string) OperationResult {
	return OperationResult{OperationID: operationID, InstanceID: current.ID, State: current.LifecycleState, Observed: current.ObservedState, ContainerID: current.ContainerID, ContainerConfigPending: current.ContainerConfigPending}
}

func operationToken() (string, error) {
	buffer := make([]byte, 24)
	if _, err := rand.Read(buffer); err != nil {
		return "", fmt.Errorf("generate operation ID: %w", err)
	}
	return base64.RawURLEncoding.EncodeToString(buffer), nil
}