Files
DoGaMa-serv/internal/backup/service.go
T

746 lines
23 KiB
Go

// Package backup owns game-data archives, retention and recoverable restores.
package backup
import (
"archive/tar"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
"github.com/klauspost/compress/zstd"
"github.com/robfig/cron/v3"
"golang.org/x/sys/unix"
)
var (
ErrNotFound = errors.New("backup not found")
ErrInvalidInput = errors.New("invalid backup input")
ErrInvalidState = errors.New("invalid backup state")
ErrUnsafePath = errors.New("unsafe backup path")
ErrIntegrity = errors.New("backup integrity check failed")
ErrIncompatible = errors.New("backup is incompatible")
)
const manifestSchemaVersion = 1
type Backup struct {
ID string `json:"id"`
InstanceID string `json:"instance_id"`
Origin string `json:"origin"`
Status string `json:"status"`
SizeBytes int64 `json:"size_bytes,omitempty"`
SHA256 string `json:"sha256,omitempty"`
CreatedAt string `json:"created_at"`
CompletedAt string `json:"completed_at,omitempty"`
ErrorCode string `json:"error_code,omitempty"`
RelativePath string `json:"-"`
}
type Policy struct {
InstanceID string `json:"instance_id"`
Enabled bool `json:"enabled"`
CronExpression string `json:"cron_expression,omitempty"`
Timezone string `json:"timezone"`
RetentionCount int `json:"retention_count"`
NextRunAt string `json:"next_run_at,omitempty"`
}
type Manifest struct {
SchemaVersion int `json:"schema_version"`
BackupID string `json:"backup_id"`
InstanceID string `json:"instance_id"`
TemplateID string `json:"template_id"`
TemplateVersion string `json:"template_version"`
Origin string `json:"origin"`
CreatedAt string `json:"created_at"`
MountIDs []string `json:"mount_ids"`
}
type Repository interface {
instance.LifecycleRepository
BeginBackup(context.Context, Backup, string, string) error
CompleteBackup(context.Context, string, string, int64, string, Manifest) error
FailBackup(context.Context, string, string) error
ListBackups(context.Context, string) ([]Backup, error)
GetBackup(context.Context, string, string) (Backup, Manifest, error)
RetentionCandidates(context.Context, string, int) ([]Backup, error)
MarkBackupDeleted(context.Context, string) error
GetBackupPolicy(context.Context, string) (Policy, error)
SetBackupPolicy(context.Context, Policy) error
ListDueBackupPolicies(context.Context, string) ([]Policy, error)
}
type Agent interface {
StartInstance(context.Context, string) (agentwire.InstanceState, error)
StopInstance(context.Context, string, int) (agentwire.InstanceState, error)
}
type Service struct {
repository Repository
agent Agent
serversRoot string
backupsRoot string
now func() time.Time
}
func New(repository Repository, agent Agent, serversRoot, backupsRoot string) (*Service, error) {
serversRoot, err := canonicalRoot(serversRoot)
if err != nil {
return nil, fmt.Errorf("server root: %w", err)
}
backupsRoot, err = canonicalRoot(backupsRoot)
if err != nil {
return nil, fmt.Errorf("backup root: %w", err)
}
return &Service{repository: repository, agent: agent, serversRoot: serversRoot, backupsRoot: backupsRoot, now: time.Now}, nil
}
func (s *Service) Create(ctx context.Context, actorID, instanceID, origin string) (Backup, error) {
if !validOrigin(origin) || instanceID == "" {
return Backup{}, ErrInvalidInput
}
current, err := s.repository.GetInstance(ctx, instanceID)
if err != nil {
return Backup{}, err
}
if current.ContainerID == "" {
return Backup{}, ErrInvalidState
}
operationID, backupID := token(), token()
if operationID == "" || backupID == "" {
return Backup{}, errors.New("generate backup identifiers")
}
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "backup", "backup")
if err != nil {
return Backup{}, err
}
backup := Backup{ID: backupID, InstanceID: instanceID, Origin: origin, Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)}
if err := s.repository.BeginBackup(ctx, backup, operationID, actorID); err != nil {
_ = s.repository.FailOperation(ctx, operationID, "error", "backup_metadata_failed")
return Backup{}, err
}
wasRunning := current.DesiredRunning
if wasRunning {
if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil {
return Backup{}, s.fail(ctx, operationID, backupID, "backup_stop_failed", err)
}
}
completed, manifest, err := s.writeArchive(ctx, current, backup)
if err != nil {
if wasRunning {
if _, restartErr := s.agent.StartInstance(ctx, instanceID); restartErr != nil {
_ = s.repository.FailBackup(ctx, backupID, "backup_restart_failed")
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed")
return Backup{}, fmt.Errorf("backup failed and restart failed: %v: %w", err, restartErr)
}
}
return Backup{}, s.fail(ctx, operationID, backupID, "backup_archive_failed", err)
}
if err := s.repository.CompleteBackup(ctx, backupID, completed.RelativePath, completed.SizeBytes, completed.SHA256, manifest); err != nil {
if full, pathErr := s.backupPath(completed.RelativePath); pathErr == nil {
_ = os.Remove(full)
}
if wasRunning {
_, _ = s.agent.StartInstance(ctx, instanceID)
}
return Backup{}, s.fail(ctx, operationID, backupID, "backup_persist_failed", err)
}
state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped"
if wasRunning {
state, err = s.agent.StartInstance(ctx, instanceID)
if err != nil {
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed")
return Backup{}, fmt.Errorf("restart after backup: %w", err)
}
lifecycle, observed = lifecycleFromAgent(state)
}
if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, ""); err != nil {
return Backup{}, err
}
completed.Status, completed.CompletedAt = "available", s.now().UTC().Format(time.RFC3339Nano)
if err := s.applyRetention(ctx, instanceID, current.Preview.Backup.RetentionCount); err != nil {
return Backup{}, fmt.Errorf("apply retention: %w", err)
}
return completed, nil
}
func (s *Service) List(ctx context.Context, instanceID string) ([]Backup, error) {
return s.repository.ListBackups(ctx, instanceID)
}
func (s *Service) GetPolicy(ctx context.Context, instanceID string) (Policy, error) {
return s.repository.GetBackupPolicy(ctx, instanceID)
}
func (s *Service) SetPolicy(ctx context.Context, value Policy) (Policy, error) {
if value.InstanceID == "" || value.RetentionCount < 1 || value.RetentionCount > 1000 {
return Policy{}, ErrInvalidInput
}
location, err := time.LoadLocation(value.Timezone)
if err != nil {
return Policy{}, ErrInvalidInput
}
if !value.Enabled {
value.CronExpression, value.NextRunAt = "", ""
} else {
schedule, err := cronParser().Parse(value.CronExpression)
if err != nil {
return Policy{}, ErrInvalidInput
}
value.NextRunAt = schedule.Next(s.now().In(location)).UTC().Format(time.RFC3339Nano)
}
if err := s.repository.SetBackupPolicy(ctx, value); err != nil {
return Policy{}, err
}
return value, nil
}
func (s *Service) RunDue(ctx context.Context) error {
now := s.now().UTC()
policies, err := s.repository.ListDueBackupPolicies(ctx, now.Format(time.RFC3339Nano))
if err != nil {
return err
}
for _, policy := range policies {
location, locationErr := time.LoadLocation(policy.Timezone)
schedule, parseErr := cronParser().Parse(policy.CronExpression)
if locationErr != nil || parseErr != nil {
continue
}
policy.NextRunAt = schedule.Next(now.In(location)).UTC().Format(time.RFC3339Nano)
if err := s.repository.SetBackupPolicy(ctx, policy); err != nil {
return err
}
if _, err := s.Create(ctx, "", policy.InstanceID, "scheduled"); err != nil && !errors.Is(err, instance.ErrOperationConflict) {
return err
}
}
return nil
}
func (s *Service) Export(ctx context.Context, instanceID, backupID string) (*os.File, Backup, error) {
backup, _, err := s.repository.GetBackup(ctx, instanceID, backupID)
if err != nil {
return nil, Backup{}, err
}
if backup.Status != "available" {
return nil, Backup{}, ErrInvalidState
}
full, err := s.backupPath(backup.RelativePath)
if err != nil {
return nil, Backup{}, err
}
if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil {
return nil, Backup{}, err
}
file, err := os.Open(full)
return file, backup, err
}
func (s *Service) Restore(ctx context.Context, actorID, instanceID, backupID string) error {
current, err := s.repository.GetInstance(ctx, instanceID)
if err != nil {
return err
}
backup, manifest, err := s.repository.GetBackup(ctx, instanceID, backupID)
if err != nil {
return err
}
if backup.Status != "available" {
return ErrInvalidState
}
if manifest.InstanceID != instanceID || manifest.TemplateID != current.Preview.Template.ID || manifest.TemplateVersion != current.Preview.Template.Version {
return ErrIncompatible
}
full, err := s.backupPath(backup.RelativePath)
if err != nil {
return err
}
if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil {
return err
}
operationID := token()
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "restore", "restore")
if err != nil {
return err
}
wasRunning := current.DesiredRunning
if wasRunning {
if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil {
return s.restoreFail(ctx, operationID, "restore_stop_failed", err)
}
}
safety := Backup{ID: token(), InstanceID: instanceID, Origin: "pre_restore", Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)}
if err := s.repository.BeginBackup(ctx, safety, operationID, actorID); err != nil {
return s.restoreFail(ctx, operationID, "restore_safety_metadata_failed", err)
}
safety, safetyManifest, err := s.writeArchive(ctx, current, safety)
if err != nil {
return s.restoreFail(ctx, operationID, "restore_safety_backup_failed", err)
}
if err := s.repository.CompleteBackup(ctx, safety.ID, safety.RelativePath, safety.SizeBytes, safety.SHA256, safetyManifest); err != nil {
return s.restoreFail(ctx, operationID, "restore_safety_persist_failed", err)
}
if err := s.restoreArchive(full, current, manifest); err != nil {
return s.restoreFail(ctx, operationID, "restore_extract_failed", err)
}
state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped"
if wasRunning {
state, err = s.agent.StartInstance(ctx, instanceID)
if err != nil {
return s.restoreFail(ctx, operationID, "restore_restart_failed", err)
}
lifecycle, observed = lifecycleFromAgent(state)
}
return s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, "")
}
func (s *Service) writeArchive(ctx context.Context, current instance.StoredInstance, backup Backup) (Backup, Manifest, error) {
mounts := make(map[string]string)
for _, mount := range current.Preview.Mounts {
mounts[mount.ID] = mount.HostPath
}
manifest := Manifest{SchemaVersion: manifestSchemaVersion, BackupID: backup.ID, InstanceID: backup.InstanceID, TemplateID: current.Preview.Template.ID, TemplateVersion: current.Preview.Template.Version, Origin: backup.Origin, CreatedAt: backup.CreatedAt, MountIDs: append([]string(nil), current.Preview.Backup.SourceMounts...)}
sort.Strings(manifest.MountIDs)
directory, err := s.instanceBackupDirectory(backup.InstanceID)
if err != nil {
return Backup{}, Manifest{}, err
}
if err := os.MkdirAll(directory, 0o750); err != nil {
return Backup{}, Manifest{}, err
}
var estimated int64
for _, mountID := range manifest.MountIDs {
source := mounts[mountID]
if source == "" {
return Backup{}, Manifest{}, ErrUnsafePath
}
if _, err := s.serverPath(source); err != nil {
return Backup{}, Manifest{}, err
}
size, err := estimateTree(ctx, source)
if err != nil {
return Backup{}, Manifest{}, err
}
estimated += size
}
if err := ensureFreeSpace(directory, estimated+(64<<20)); err != nil {
return Backup{}, Manifest{}, err
}
temporary, err := os.CreateTemp(directory, ".creating-*.tar.zst")
if err != nil {
return Backup{}, Manifest{}, err
}
temporaryName := temporary.Name()
defer func() { _ = temporary.Close(); _ = os.Remove(temporaryName) }()
hasher := sha256.New()
zstdWriter, err := zstd.NewWriter(io.MultiWriter(temporary, hasher), zstd.WithEncoderConcurrency(1))
if err != nil {
return Backup{}, Manifest{}, err
}
tarWriter := tar.NewWriter(zstdWriter)
manifestJSON, _ := json.Marshal(manifest)
err = tarWriter.WriteHeader(&tar.Header{Name: "manifest.json", Mode: 0o600, Size: int64(len(manifestJSON)), ModTime: s.now().UTC(), Typeflag: tar.TypeReg})
if err == nil {
_, err = tarWriter.Write(manifestJSON)
}
if err == nil {
for _, mountID := range manifest.MountIDs {
source := mounts[mountID]
if source == "" {
err = ErrUnsafePath
break
}
if _, pathErr := s.serverPath(source); pathErr != nil {
err = pathErr
break
}
if walkErr := addTree(ctx, tarWriter, source, path.Join("data", mountID)); walkErr != nil {
err = walkErr
break
}
}
}
if closeErr := tarWriter.Close(); err == nil {
err = closeErr
}
if closeErr := zstdWriter.Close(); err == nil {
err = closeErr
}
if syncErr := temporary.Sync(); err == nil {
err = syncErr
}
if closeErr := temporary.Close(); err == nil {
err = closeErr
}
if err != nil {
return Backup{}, Manifest{}, err
}
info, err := os.Stat(temporaryName)
if err != nil {
return Backup{}, Manifest{}, err
}
finalName := backup.ID + ".tar.zst"
finalPath := filepath.Join(directory, finalName)
if err := os.Rename(temporaryName, finalPath); err != nil {
return Backup{}, Manifest{}, err
}
backup.RelativePath = filepath.ToSlash(filepath.Join(backup.InstanceID, finalName))
backup.SizeBytes, backup.SHA256 = info.Size(), hex.EncodeToString(hasher.Sum(nil))
return backup, manifest, nil
}
func addTree(ctx context.Context, writer *tar.Writer, source, prefix string) error {
return filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) {
return ErrUnsafePath
}
relative, err := filepath.Rel(source, current)
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return ErrUnsafePath
}
name := prefix
if relative != "." {
name = path.Join(prefix, filepath.ToSlash(relative))
}
header, err := tar.FileInfoHeader(info, "")
if err != nil {
return err
}
header.Name, header.Uid, header.Gid, header.Uname, header.Gname = name, 0, 0, "", ""
if err := writer.WriteHeader(header); err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
file, err := os.Open(current)
if err != nil {
return err
}
_, copyErr := io.Copy(writer, file)
closeErr := file.Close()
if copyErr != nil {
return copyErr
}
return closeErr
})
}
func estimateTree(ctx context.Context, source string) (int64, error) {
var total int64
err := filepath.WalkDir(source, func(_ string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) {
return ErrUnsafePath
}
if info.Mode().IsRegular() {
total += info.Size()
}
return nil
})
return total, err
}
func ensureFreeSpace(directory string, required int64) error {
var stats unix.Statfs_t
if err := unix.Statfs(directory, &stats); err != nil {
return err
}
available := int64(stats.Bavail) * int64(stats.Bsize)
if available < required {
return ErrInvalidState
}
return nil
}
func (s *Service) restoreArchive(archive string, current instance.StoredInstance, manifest Manifest) error {
stage, err := os.MkdirTemp(s.serversRoot, ".dogama-restore-")
if err != nil {
return err
}
defer os.RemoveAll(stage)
file, err := os.Open(archive)
if err != nil {
return err
}
defer file.Close()
decoder, err := zstd.NewReader(file, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(2<<30))
if err != nil {
return err
}
defer decoder.Close()
reader := tar.NewReader(decoder)
var total int64
sawManifest := false
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
if header.Name == "manifest.json" {
if sawManifest || header.Size < 1 || header.Size > 64<<10 {
return ErrIntegrity
}
var archived Manifest
if err := json.NewDecoder(io.LimitReader(reader, header.Size)).Decode(&archived); err != nil {
return ErrIntegrity
}
if archived.SchemaVersion != manifestSchemaVersion || archived.BackupID != manifest.BackupID || archived.InstanceID != manifest.InstanceID || archived.TemplateID != manifest.TemplateID || archived.TemplateVersion != manifest.TemplateVersion || strings.Join(archived.MountIDs, "\x00") != strings.Join(manifest.MountIDs, "\x00") {
return ErrIntegrity
}
sawManifest = true
continue
}
if !fs.ValidPath(header.Name) || !strings.HasPrefix(header.Name, "data/") || header.Linkname != "" {
return ErrUnsafePath
}
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir {
return ErrUnsafePath
}
total += header.Size
if total > 1<<40 {
return ErrInvalidInput
}
target := filepath.Join(stage, filepath.FromSlash(header.Name))
if !within(stage, target) {
return ErrUnsafePath
}
if header.Typeflag == tar.TypeDir {
if err := os.MkdirAll(target, 0o750); err != nil {
return err
}
continue
}
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
if err != nil {
return err
}
_, copyErr := io.CopyN(out, reader, header.Size)
closeErr := out.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
}
if !sawManifest {
return ErrIntegrity
}
mountPaths := make(map[string]string)
for _, mount := range current.Preview.Mounts {
mountPaths[mount.ID] = mount.HostPath
}
var swapped []struct{ live, previous string }
rollback := func() {
for index := len(swapped) - 1; index >= 0; index-- {
_ = os.RemoveAll(swapped[index].live)
_ = os.Rename(swapped[index].previous, swapped[index].live)
}
}
for _, mountID := range manifest.MountIDs {
live := mountPaths[mountID]
if _, err := s.serverPath(live); err != nil {
rollback()
return err
}
staged := filepath.Join(stage, "data", mountID)
if _, err := os.Stat(staged); err != nil {
rollback()
return ErrIntegrity
}
previous := live + ".dogama-previous-" + manifest.BackupID
if err := os.Rename(live, previous); err != nil {
rollback()
return err
}
if err := os.Rename(staged, live); err != nil {
_ = os.Rename(previous, live)
rollback()
return err
}
swapped = append(swapped, struct{ live, previous string }{live, previous})
}
for _, item := range swapped {
if err := os.RemoveAll(item.previous); err != nil {
return err
}
}
return nil
}
func (s *Service) applyRetention(ctx context.Context, instanceID string, count int) error {
candidates, err := s.repository.RetentionCandidates(ctx, instanceID, count)
if err != nil {
return err
}
for _, candidate := range candidates {
full, err := s.backupPath(candidate.RelativePath)
if err != nil {
return err
}
if err := os.Remove(full); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err := s.repository.MarkBackupDeleted(ctx, candidate.ID); err != nil {
return err
}
}
return nil
}
func (s *Service) fail(ctx context.Context, operationID, backupID, code string, cause error) error {
_ = s.repository.FailBackup(ctx, backupID, code)
_ = s.repository.FailOperation(ctx, operationID, "error", code)
return fmt.Errorf("%s: %w", code, cause)
}
func (s *Service) restoreFail(ctx context.Context, operationID, code string, cause error) error {
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", code)
return fmt.Errorf("%s: %w", code, cause)
}
func (s *Service) instanceBackupDirectory(instanceID string) (string, error) {
if instanceID == "" || strings.ContainsAny(instanceID, `/\\`) {
return "", ErrUnsafePath
}
result := filepath.Join(s.backupsRoot, instanceID)
if !within(s.backupsRoot, result) {
return "", ErrUnsafePath
}
return result, nil
}
func (s *Service) backupPath(relative string) (string, error) {
if !fs.ValidPath(relative) {
return "", ErrUnsafePath
}
result := filepath.Join(s.backupsRoot, filepath.FromSlash(relative))
if !within(s.backupsRoot, result) {
return "", ErrUnsafePath
}
return result, nil
}
func (s *Service) serverPath(value string) (string, error) {
clean := filepath.Clean(value)
if !filepath.IsAbs(clean) || !within(s.serversRoot, clean) {
return "", ErrUnsafePath
}
return clean, nil
}
func canonicalRoot(value string) (string, error) {
if value == "" {
return "", ErrUnsafePath
}
absolute, err := filepath.Abs(value)
if err != nil {
return "", err
}
if err := os.MkdirAll(absolute, 0o750); err != nil {
return "", err
}
return filepath.EvalSymlinks(absolute)
}
func within(root, candidate string) bool {
relative, err := filepath.Rel(root, candidate)
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}
func verifyFile(name string, expectedSize int64, expectedSHA string) error {
file, err := os.Open(name)
if err != nil {
return err
}
defer file.Close()
hasher := sha256.New()
size, err := io.Copy(hasher, file)
if err != nil {
return err
}
if size != expectedSize || !strings.EqualFold(hex.EncodeToString(hasher.Sum(nil)), expectedSHA) {
return ErrIntegrity
}
return nil
}
func validOrigin(value string) bool {
switch value {
case "manual", "scheduled", "pre_update", "pre_restore", "idle_shutdown", "imported", "system":
return true
}
return false
}
func token() string {
buffer := make([]byte, 24)
if _, err := rand.Read(buffer); err != nil {
return ""
}
return base64.RawURLEncoding.EncodeToString(buffer)
}
func lifecycleFromAgent(state agentwire.InstanceState) (string, string) {
if !state.Running {
return "stopped", "stopped"
}
if state.Ready {
return "online", "ready"
}
if state.Health == "unhealthy" || state.Health == "none" {
return "degraded", "degraded"
}
return "starting", "running"
}
func cronParser() cron.Parser {
return cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
}