746 lines
23 KiB
Go
746 lines
23 KiB
Go
// Package backup owns game-data archives, retention and recoverable restores.
|
|
package backup
|
|
|
|
import (
|
|
"archive/tar"
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire"
|
|
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/robfig/cron/v3"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
var (
|
|
ErrNotFound = errors.New("backup not found")
|
|
ErrInvalidInput = errors.New("invalid backup input")
|
|
ErrInvalidState = errors.New("invalid backup state")
|
|
ErrUnsafePath = errors.New("unsafe backup path")
|
|
ErrIntegrity = errors.New("backup integrity check failed")
|
|
ErrIncompatible = errors.New("backup is incompatible")
|
|
)
|
|
|
|
const manifestSchemaVersion = 1
|
|
|
|
type Backup struct {
|
|
ID string `json:"id"`
|
|
InstanceID string `json:"instance_id"`
|
|
Origin string `json:"origin"`
|
|
Status string `json:"status"`
|
|
SizeBytes int64 `json:"size_bytes,omitempty"`
|
|
SHA256 string `json:"sha256,omitempty"`
|
|
CreatedAt string `json:"created_at"`
|
|
CompletedAt string `json:"completed_at,omitempty"`
|
|
ErrorCode string `json:"error_code,omitempty"`
|
|
RelativePath string `json:"-"`
|
|
}
|
|
|
|
type Policy struct {
|
|
InstanceID string `json:"instance_id"`
|
|
Enabled bool `json:"enabled"`
|
|
CronExpression string `json:"cron_expression,omitempty"`
|
|
Timezone string `json:"timezone"`
|
|
RetentionCount int `json:"retention_count"`
|
|
NextRunAt string `json:"next_run_at,omitempty"`
|
|
}
|
|
|
|
type Manifest struct {
|
|
SchemaVersion int `json:"schema_version"`
|
|
BackupID string `json:"backup_id"`
|
|
InstanceID string `json:"instance_id"`
|
|
TemplateID string `json:"template_id"`
|
|
TemplateVersion string `json:"template_version"`
|
|
Origin string `json:"origin"`
|
|
CreatedAt string `json:"created_at"`
|
|
MountIDs []string `json:"mount_ids"`
|
|
}
|
|
|
|
type Repository interface {
|
|
instance.LifecycleRepository
|
|
BeginBackup(context.Context, Backup, string, string) error
|
|
CompleteBackup(context.Context, string, string, int64, string, Manifest) error
|
|
FailBackup(context.Context, string, string) error
|
|
ListBackups(context.Context, string) ([]Backup, error)
|
|
GetBackup(context.Context, string, string) (Backup, Manifest, error)
|
|
RetentionCandidates(context.Context, string, int) ([]Backup, error)
|
|
MarkBackupDeleted(context.Context, string) error
|
|
GetBackupPolicy(context.Context, string) (Policy, error)
|
|
SetBackupPolicy(context.Context, Policy) error
|
|
ListDueBackupPolicies(context.Context, string) ([]Policy, error)
|
|
}
|
|
|
|
type Agent interface {
|
|
StartInstance(context.Context, string) (agentwire.InstanceState, error)
|
|
StopInstance(context.Context, string, int) (agentwire.InstanceState, error)
|
|
}
|
|
|
|
type Service struct {
|
|
repository Repository
|
|
agent Agent
|
|
serversRoot string
|
|
backupsRoot string
|
|
now func() time.Time
|
|
}
|
|
|
|
func New(repository Repository, agent Agent, serversRoot, backupsRoot string) (*Service, error) {
|
|
serversRoot, err := canonicalRoot(serversRoot)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("server root: %w", err)
|
|
}
|
|
backupsRoot, err = canonicalRoot(backupsRoot)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("backup root: %w", err)
|
|
}
|
|
return &Service{repository: repository, agent: agent, serversRoot: serversRoot, backupsRoot: backupsRoot, now: time.Now}, nil
|
|
}
|
|
|
|
func (s *Service) Create(ctx context.Context, actorID, instanceID, origin string) (Backup, error) {
|
|
if !validOrigin(origin) || instanceID == "" {
|
|
return Backup{}, ErrInvalidInput
|
|
}
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return Backup{}, err
|
|
}
|
|
if current.ContainerID == "" {
|
|
return Backup{}, ErrInvalidState
|
|
}
|
|
operationID, backupID := token(), token()
|
|
if operationID == "" || backupID == "" {
|
|
return Backup{}, errors.New("generate backup identifiers")
|
|
}
|
|
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "backup", "backup")
|
|
if err != nil {
|
|
return Backup{}, err
|
|
}
|
|
backup := Backup{ID: backupID, InstanceID: instanceID, Origin: origin, Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)}
|
|
if err := s.repository.BeginBackup(ctx, backup, operationID, actorID); err != nil {
|
|
_ = s.repository.FailOperation(ctx, operationID, "error", "backup_metadata_failed")
|
|
return Backup{}, err
|
|
}
|
|
wasRunning := current.DesiredRunning
|
|
if wasRunning {
|
|
if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil {
|
|
return Backup{}, s.fail(ctx, operationID, backupID, "backup_stop_failed", err)
|
|
}
|
|
}
|
|
completed, manifest, err := s.writeArchive(ctx, current, backup)
|
|
if err != nil {
|
|
if wasRunning {
|
|
if _, restartErr := s.agent.StartInstance(ctx, instanceID); restartErr != nil {
|
|
_ = s.repository.FailBackup(ctx, backupID, "backup_restart_failed")
|
|
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed")
|
|
return Backup{}, fmt.Errorf("backup failed and restart failed: %v: %w", err, restartErr)
|
|
}
|
|
}
|
|
return Backup{}, s.fail(ctx, operationID, backupID, "backup_archive_failed", err)
|
|
}
|
|
if err := s.repository.CompleteBackup(ctx, backupID, completed.RelativePath, completed.SizeBytes, completed.SHA256, manifest); err != nil {
|
|
if full, pathErr := s.backupPath(completed.RelativePath); pathErr == nil {
|
|
_ = os.Remove(full)
|
|
}
|
|
if wasRunning {
|
|
_, _ = s.agent.StartInstance(ctx, instanceID)
|
|
}
|
|
return Backup{}, s.fail(ctx, operationID, backupID, "backup_persist_failed", err)
|
|
}
|
|
state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped"
|
|
if wasRunning {
|
|
state, err = s.agent.StartInstance(ctx, instanceID)
|
|
if err != nil {
|
|
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed")
|
|
return Backup{}, fmt.Errorf("restart after backup: %w", err)
|
|
}
|
|
lifecycle, observed = lifecycleFromAgent(state)
|
|
}
|
|
if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, ""); err != nil {
|
|
return Backup{}, err
|
|
}
|
|
completed.Status, completed.CompletedAt = "available", s.now().UTC().Format(time.RFC3339Nano)
|
|
if err := s.applyRetention(ctx, instanceID, current.Preview.Backup.RetentionCount); err != nil {
|
|
return Backup{}, fmt.Errorf("apply retention: %w", err)
|
|
}
|
|
return completed, nil
|
|
}
|
|
|
|
func (s *Service) List(ctx context.Context, instanceID string) ([]Backup, error) {
|
|
return s.repository.ListBackups(ctx, instanceID)
|
|
}
|
|
|
|
func (s *Service) GetPolicy(ctx context.Context, instanceID string) (Policy, error) {
|
|
return s.repository.GetBackupPolicy(ctx, instanceID)
|
|
}
|
|
|
|
func (s *Service) SetPolicy(ctx context.Context, value Policy) (Policy, error) {
|
|
if value.InstanceID == "" || value.RetentionCount < 1 || value.RetentionCount > 1000 {
|
|
return Policy{}, ErrInvalidInput
|
|
}
|
|
location, err := time.LoadLocation(value.Timezone)
|
|
if err != nil {
|
|
return Policy{}, ErrInvalidInput
|
|
}
|
|
if !value.Enabled {
|
|
value.CronExpression, value.NextRunAt = "", ""
|
|
} else {
|
|
schedule, err := cronParser().Parse(value.CronExpression)
|
|
if err != nil {
|
|
return Policy{}, ErrInvalidInput
|
|
}
|
|
value.NextRunAt = schedule.Next(s.now().In(location)).UTC().Format(time.RFC3339Nano)
|
|
}
|
|
if err := s.repository.SetBackupPolicy(ctx, value); err != nil {
|
|
return Policy{}, err
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func (s *Service) RunDue(ctx context.Context) error {
|
|
now := s.now().UTC()
|
|
policies, err := s.repository.ListDueBackupPolicies(ctx, now.Format(time.RFC3339Nano))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, policy := range policies {
|
|
location, locationErr := time.LoadLocation(policy.Timezone)
|
|
schedule, parseErr := cronParser().Parse(policy.CronExpression)
|
|
if locationErr != nil || parseErr != nil {
|
|
continue
|
|
}
|
|
policy.NextRunAt = schedule.Next(now.In(location)).UTC().Format(time.RFC3339Nano)
|
|
if err := s.repository.SetBackupPolicy(ctx, policy); err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.Create(ctx, "", policy.InstanceID, "scheduled"); err != nil && !errors.Is(err, instance.ErrOperationConflict) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) Export(ctx context.Context, instanceID, backupID string) (*os.File, Backup, error) {
|
|
backup, _, err := s.repository.GetBackup(ctx, instanceID, backupID)
|
|
if err != nil {
|
|
return nil, Backup{}, err
|
|
}
|
|
if backup.Status != "available" {
|
|
return nil, Backup{}, ErrInvalidState
|
|
}
|
|
full, err := s.backupPath(backup.RelativePath)
|
|
if err != nil {
|
|
return nil, Backup{}, err
|
|
}
|
|
if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil {
|
|
return nil, Backup{}, err
|
|
}
|
|
file, err := os.Open(full)
|
|
return file, backup, err
|
|
}
|
|
|
|
func (s *Service) Restore(ctx context.Context, actorID, instanceID, backupID string) error {
|
|
current, err := s.repository.GetInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
backup, manifest, err := s.repository.GetBackup(ctx, instanceID, backupID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if backup.Status != "available" {
|
|
return ErrInvalidState
|
|
}
|
|
if manifest.InstanceID != instanceID || manifest.TemplateID != current.Preview.Template.ID || manifest.TemplateVersion != current.Preview.Template.Version {
|
|
return ErrIncompatible
|
|
}
|
|
full, err := s.backupPath(backup.RelativePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil {
|
|
return err
|
|
}
|
|
operationID := token()
|
|
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "restore", "restore")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
wasRunning := current.DesiredRunning
|
|
if wasRunning {
|
|
if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil {
|
|
return s.restoreFail(ctx, operationID, "restore_stop_failed", err)
|
|
}
|
|
}
|
|
safety := Backup{ID: token(), InstanceID: instanceID, Origin: "pre_restore", Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)}
|
|
if err := s.repository.BeginBackup(ctx, safety, operationID, actorID); err != nil {
|
|
return s.restoreFail(ctx, operationID, "restore_safety_metadata_failed", err)
|
|
}
|
|
safety, safetyManifest, err := s.writeArchive(ctx, current, safety)
|
|
if err != nil {
|
|
return s.restoreFail(ctx, operationID, "restore_safety_backup_failed", err)
|
|
}
|
|
if err := s.repository.CompleteBackup(ctx, safety.ID, safety.RelativePath, safety.SizeBytes, safety.SHA256, safetyManifest); err != nil {
|
|
return s.restoreFail(ctx, operationID, "restore_safety_persist_failed", err)
|
|
}
|
|
if err := s.restoreArchive(full, current, manifest); err != nil {
|
|
return s.restoreFail(ctx, operationID, "restore_extract_failed", err)
|
|
}
|
|
state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped"
|
|
if wasRunning {
|
|
state, err = s.agent.StartInstance(ctx, instanceID)
|
|
if err != nil {
|
|
return s.restoreFail(ctx, operationID, "restore_restart_failed", err)
|
|
}
|
|
lifecycle, observed = lifecycleFromAgent(state)
|
|
}
|
|
return s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, "")
|
|
}
|
|
|
|
func (s *Service) writeArchive(ctx context.Context, current instance.StoredInstance, backup Backup) (Backup, Manifest, error) {
|
|
mounts := make(map[string]string)
|
|
for _, mount := range current.Preview.Mounts {
|
|
mounts[mount.ID] = mount.HostPath
|
|
}
|
|
manifest := Manifest{SchemaVersion: manifestSchemaVersion, BackupID: backup.ID, InstanceID: backup.InstanceID, TemplateID: current.Preview.Template.ID, TemplateVersion: current.Preview.Template.Version, Origin: backup.Origin, CreatedAt: backup.CreatedAt, MountIDs: append([]string(nil), current.Preview.Backup.SourceMounts...)}
|
|
sort.Strings(manifest.MountIDs)
|
|
directory, err := s.instanceBackupDirectory(backup.InstanceID)
|
|
if err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
if err := os.MkdirAll(directory, 0o750); err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
var estimated int64
|
|
for _, mountID := range manifest.MountIDs {
|
|
source := mounts[mountID]
|
|
if source == "" {
|
|
return Backup{}, Manifest{}, ErrUnsafePath
|
|
}
|
|
if _, err := s.serverPath(source); err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
size, err := estimateTree(ctx, source)
|
|
if err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
estimated += size
|
|
}
|
|
if err := ensureFreeSpace(directory, estimated+(64<<20)); err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
temporary, err := os.CreateTemp(directory, ".creating-*.tar.zst")
|
|
if err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
temporaryName := temporary.Name()
|
|
defer func() { _ = temporary.Close(); _ = os.Remove(temporaryName) }()
|
|
hasher := sha256.New()
|
|
zstdWriter, err := zstd.NewWriter(io.MultiWriter(temporary, hasher), zstd.WithEncoderConcurrency(1))
|
|
if err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
tarWriter := tar.NewWriter(zstdWriter)
|
|
manifestJSON, _ := json.Marshal(manifest)
|
|
err = tarWriter.WriteHeader(&tar.Header{Name: "manifest.json", Mode: 0o600, Size: int64(len(manifestJSON)), ModTime: s.now().UTC(), Typeflag: tar.TypeReg})
|
|
if err == nil {
|
|
_, err = tarWriter.Write(manifestJSON)
|
|
}
|
|
if err == nil {
|
|
for _, mountID := range manifest.MountIDs {
|
|
source := mounts[mountID]
|
|
if source == "" {
|
|
err = ErrUnsafePath
|
|
break
|
|
}
|
|
if _, pathErr := s.serverPath(source); pathErr != nil {
|
|
err = pathErr
|
|
break
|
|
}
|
|
if walkErr := addTree(ctx, tarWriter, source, path.Join("data", mountID)); walkErr != nil {
|
|
err = walkErr
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if closeErr := tarWriter.Close(); err == nil {
|
|
err = closeErr
|
|
}
|
|
if closeErr := zstdWriter.Close(); err == nil {
|
|
err = closeErr
|
|
}
|
|
if syncErr := temporary.Sync(); err == nil {
|
|
err = syncErr
|
|
}
|
|
if closeErr := temporary.Close(); err == nil {
|
|
err = closeErr
|
|
}
|
|
if err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
info, err := os.Stat(temporaryName)
|
|
if err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
finalName := backup.ID + ".tar.zst"
|
|
finalPath := filepath.Join(directory, finalName)
|
|
if err := os.Rename(temporaryName, finalPath); err != nil {
|
|
return Backup{}, Manifest{}, err
|
|
}
|
|
backup.RelativePath = filepath.ToSlash(filepath.Join(backup.InstanceID, finalName))
|
|
backup.SizeBytes, backup.SHA256 = info.Size(), hex.EncodeToString(hasher.Sum(nil))
|
|
return backup, manifest, nil
|
|
}
|
|
|
|
func addTree(ctx context.Context, writer *tar.Writer, source, prefix string) error {
|
|
return filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
default:
|
|
}
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) {
|
|
return ErrUnsafePath
|
|
}
|
|
relative, err := filepath.Rel(source, current)
|
|
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
|
return ErrUnsafePath
|
|
}
|
|
name := prefix
|
|
if relative != "." {
|
|
name = path.Join(prefix, filepath.ToSlash(relative))
|
|
}
|
|
header, err := tar.FileInfoHeader(info, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
header.Name, header.Uid, header.Gid, header.Uname, header.Gname = name, 0, 0, "", ""
|
|
if err := writer.WriteHeader(header); err != nil {
|
|
return err
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return nil
|
|
}
|
|
file, err := os.Open(current)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, copyErr := io.Copy(writer, file)
|
|
closeErr := file.Close()
|
|
if copyErr != nil {
|
|
return copyErr
|
|
}
|
|
return closeErr
|
|
})
|
|
}
|
|
|
|
func estimateTree(ctx context.Context, source string) (int64, error) {
|
|
var total int64
|
|
err := filepath.WalkDir(source, func(_ string, entry fs.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
default:
|
|
}
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) {
|
|
return ErrUnsafePath
|
|
}
|
|
if info.Mode().IsRegular() {
|
|
total += info.Size()
|
|
}
|
|
return nil
|
|
})
|
|
return total, err
|
|
}
|
|
|
|
func ensureFreeSpace(directory string, required int64) error {
|
|
var stats unix.Statfs_t
|
|
if err := unix.Statfs(directory, &stats); err != nil {
|
|
return err
|
|
}
|
|
available := int64(stats.Bavail) * int64(stats.Bsize)
|
|
if available < required {
|
|
return ErrInvalidState
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) restoreArchive(archive string, current instance.StoredInstance, manifest Manifest) error {
|
|
stage, err := os.MkdirTemp(s.serversRoot, ".dogama-restore-")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.RemoveAll(stage)
|
|
file, err := os.Open(archive)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
decoder, err := zstd.NewReader(file, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(2<<30))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer decoder.Close()
|
|
reader := tar.NewReader(decoder)
|
|
var total int64
|
|
sawManifest := false
|
|
for {
|
|
header, err := reader.Next()
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if header.Name == "manifest.json" {
|
|
if sawManifest || header.Size < 1 || header.Size > 64<<10 {
|
|
return ErrIntegrity
|
|
}
|
|
var archived Manifest
|
|
if err := json.NewDecoder(io.LimitReader(reader, header.Size)).Decode(&archived); err != nil {
|
|
return ErrIntegrity
|
|
}
|
|
if archived.SchemaVersion != manifestSchemaVersion || archived.BackupID != manifest.BackupID || archived.InstanceID != manifest.InstanceID || archived.TemplateID != manifest.TemplateID || archived.TemplateVersion != manifest.TemplateVersion || strings.Join(archived.MountIDs, "\x00") != strings.Join(manifest.MountIDs, "\x00") {
|
|
return ErrIntegrity
|
|
}
|
|
sawManifest = true
|
|
continue
|
|
}
|
|
if !fs.ValidPath(header.Name) || !strings.HasPrefix(header.Name, "data/") || header.Linkname != "" {
|
|
return ErrUnsafePath
|
|
}
|
|
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir {
|
|
return ErrUnsafePath
|
|
}
|
|
total += header.Size
|
|
if total > 1<<40 {
|
|
return ErrInvalidInput
|
|
}
|
|
target := filepath.Join(stage, filepath.FromSlash(header.Name))
|
|
if !within(stage, target) {
|
|
return ErrUnsafePath
|
|
}
|
|
if header.Typeflag == tar.TypeDir {
|
|
if err := os.MkdirAll(target, 0o750); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
|
|
return err
|
|
}
|
|
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, copyErr := io.CopyN(out, reader, header.Size)
|
|
closeErr := out.Close()
|
|
if copyErr != nil {
|
|
return copyErr
|
|
}
|
|
if closeErr != nil {
|
|
return closeErr
|
|
}
|
|
}
|
|
if !sawManifest {
|
|
return ErrIntegrity
|
|
}
|
|
mountPaths := make(map[string]string)
|
|
for _, mount := range current.Preview.Mounts {
|
|
mountPaths[mount.ID] = mount.HostPath
|
|
}
|
|
var swapped []struct{ live, previous string }
|
|
rollback := func() {
|
|
for index := len(swapped) - 1; index >= 0; index-- {
|
|
_ = os.RemoveAll(swapped[index].live)
|
|
_ = os.Rename(swapped[index].previous, swapped[index].live)
|
|
}
|
|
}
|
|
for _, mountID := range manifest.MountIDs {
|
|
live := mountPaths[mountID]
|
|
if _, err := s.serverPath(live); err != nil {
|
|
rollback()
|
|
return err
|
|
}
|
|
staged := filepath.Join(stage, "data", mountID)
|
|
if _, err := os.Stat(staged); err != nil {
|
|
rollback()
|
|
return ErrIntegrity
|
|
}
|
|
previous := live + ".dogama-previous-" + manifest.BackupID
|
|
if err := os.Rename(live, previous); err != nil {
|
|
rollback()
|
|
return err
|
|
}
|
|
if err := os.Rename(staged, live); err != nil {
|
|
_ = os.Rename(previous, live)
|
|
rollback()
|
|
return err
|
|
}
|
|
swapped = append(swapped, struct{ live, previous string }{live, previous})
|
|
}
|
|
for _, item := range swapped {
|
|
if err := os.RemoveAll(item.previous); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) applyRetention(ctx context.Context, instanceID string, count int) error {
|
|
candidates, err := s.repository.RetentionCandidates(ctx, instanceID, count)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, candidate := range candidates {
|
|
full, err := s.backupPath(candidate.RelativePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.Remove(full); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
if err := s.repository.MarkBackupDeleted(ctx, candidate.ID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) fail(ctx context.Context, operationID, backupID, code string, cause error) error {
|
|
_ = s.repository.FailBackup(ctx, backupID, code)
|
|
_ = s.repository.FailOperation(ctx, operationID, "error", code)
|
|
return fmt.Errorf("%s: %w", code, cause)
|
|
}
|
|
|
|
func (s *Service) restoreFail(ctx context.Context, operationID, code string, cause error) error {
|
|
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", code)
|
|
return fmt.Errorf("%s: %w", code, cause)
|
|
}
|
|
|
|
func (s *Service) instanceBackupDirectory(instanceID string) (string, error) {
|
|
if instanceID == "" || strings.ContainsAny(instanceID, `/\\`) {
|
|
return "", ErrUnsafePath
|
|
}
|
|
result := filepath.Join(s.backupsRoot, instanceID)
|
|
if !within(s.backupsRoot, result) {
|
|
return "", ErrUnsafePath
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Service) backupPath(relative string) (string, error) {
|
|
if !fs.ValidPath(relative) {
|
|
return "", ErrUnsafePath
|
|
}
|
|
result := filepath.Join(s.backupsRoot, filepath.FromSlash(relative))
|
|
if !within(s.backupsRoot, result) {
|
|
return "", ErrUnsafePath
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Service) serverPath(value string) (string, error) {
|
|
clean := filepath.Clean(value)
|
|
if !filepath.IsAbs(clean) || !within(s.serversRoot, clean) {
|
|
return "", ErrUnsafePath
|
|
}
|
|
return clean, nil
|
|
}
|
|
|
|
func canonicalRoot(value string) (string, error) {
|
|
if value == "" {
|
|
return "", ErrUnsafePath
|
|
}
|
|
absolute, err := filepath.Abs(value)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err := os.MkdirAll(absolute, 0o750); err != nil {
|
|
return "", err
|
|
}
|
|
return filepath.EvalSymlinks(absolute)
|
|
}
|
|
|
|
func within(root, candidate string) bool {
|
|
relative, err := filepath.Rel(root, candidate)
|
|
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
|
}
|
|
|
|
func verifyFile(name string, expectedSize int64, expectedSHA string) error {
|
|
file, err := os.Open(name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
hasher := sha256.New()
|
|
size, err := io.Copy(hasher, file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if size != expectedSize || !strings.EqualFold(hex.EncodeToString(hasher.Sum(nil)), expectedSHA) {
|
|
return ErrIntegrity
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validOrigin(value string) bool {
|
|
switch value {
|
|
case "manual", "scheduled", "pre_update", "pre_restore", "idle_shutdown", "imported", "system":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func token() string {
|
|
buffer := make([]byte, 24)
|
|
if _, err := rand.Read(buffer); err != nil {
|
|
return ""
|
|
}
|
|
return base64.RawURLEncoding.EncodeToString(buffer)
|
|
}
|
|
|
|
func lifecycleFromAgent(state agentwire.InstanceState) (string, string) {
|
|
if !state.Running {
|
|
return "stopped", "stopped"
|
|
}
|
|
if state.Ready {
|
|
return "online", "ready"
|
|
}
|
|
if state.Health == "unhealthy" || state.Health == "none" {
|
|
return "degraded", "degraded"
|
|
}
|
|
return "starting", "running"
|
|
}
|
|
|
|
func cronParser() cron.Parser {
|
|
return cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
|
|
}
|