DoGaMa/DoGaMa-serv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backup and import foundation #6

Merged
tony merged 2 commits from codex/backup-foundation AGit into main 2026-08-07 18:13:36 +02:00
Conversation 1 Commits 2 Files Changed 19
+2414 -26
19 changed files with 2414 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -72,7 +72,7 @@ Only the main application's HTTP port is published. The agent and game-managemen
## Status
The first five roadmap foundations are implemented: the main application and authentication, the restricted agent boundary, the validated embedded catalog, the registered instance lifecycle and backend-enforced per-instance authorization. Administrators can manage local users, assign user or manager memberships, apply explicit permission overrides and review installation requests without approval causing deployment. Assigned users receive the documented safe baseline; managers receive the operational baseline, while restore remains an explicit grant. SQLite records identities, access policy, requests, serialized operation phases and desired/observed instance state. Backups, updates and the WebAssembly runtime remain later roadmap work.
The first six roadmap foundations are implemented: application/authentication, the restricted agent boundary, the validated catalog, registered instance lifecycle, per-instance authorization, and recoverable game-data backups. DoGaMa creates atomic `tar.zst` archives with manifests and SHA-256 metadata, selectively retains scheduled backups, supports five-field cron policies with IANA timezones, stages hostile imports under strict limits, and restores through validated staging with a default `pre_restore` safety backup. Backup, export, restore and policy APIs enforce backend permissions and recent authentication for sensitive actions. Updates and the WebAssembly runtime remain later roadmap work.
## Validate the specification
cmd/dogama/main.go
+50 -2
View File
@@ -15,7 +15,9 @@ import (
catalogdata "git.zaynet.fr/DoGaMa/DoGaMa-serv/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentclient"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/web"
@@ -32,6 +34,7 @@ func main() {
func run(logger *slog.Logger) error {
listenAddress := environment("DOGAMA_LISTEN_ADDRESS", ":8080")
databasePath := environment("DOGAMA_DATABASE_PATH", "dogama.db")
serversRoot := environment("DOGAMA_SERVERS_ROOT", "/srv/game-servers")
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
@@ -51,10 +54,15 @@ func run(logger *slog.Logger) error {
logger.Info("local catalog synchronized", "event", "catalog.synchronized", "template_count", len(snapshots))
var handler http.Handler
var lifecycle *instance.LifecycleService
var backupService *backup.Service
importService, err := importexport.New(repository, environment("DOGAMA_IMPORTS_ROOT", "/var/lib/dogama/imports/staging"), serversRoot)
if err != nil {
return err
}
agentURL, tokenFile := os.Getenv("DOGAMA_AGENT_URL"), os.Getenv("DOGAMA_AGENT_TOKEN_FILE")
if agentURL == "" && tokenFile == "" {
logger.Warn("instance lifecycle disabled", "event", "lifecycle.disabled")
handler, err = web.NewHandlerWithRepository(auth.New(db), repository, logger)
handler, err = web.NewHandlerWithRepositoryAndImports(auth.New(db), repository, importService, logger)
} else {
if agentURL == "" || tokenFile == "" {
return errors.New("DOGAMA_AGENT_URL and DOGAMA_AGENT_TOKEN_FILE must be configured together")
@@ -69,6 +77,10 @@ func run(logger *slog.Logger) error {
return clientErr
}
lifecycle = instance.NewLifecycleService(repository, agent)
backupService, err = backup.New(repository, agent, serversRoot, environment("DOGAMA_BACKUPS_ROOT", "/srv/game-backups"))
if err != nil {
return err
}
reconcileCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
if recoverErr := lifecycle.RecoverInterruptedOperations(reconcileCtx); recoverErr != nil {
cancel()
@@ -78,7 +90,7 @@ func run(logger *slog.Logger) error {
logger.Warn("instance reconciliation incomplete", "event", "lifecycle.reconcile.failed")
}
cancel()
handler, err = web.NewHandlerWithLifecycle(auth.New(db), repository, agent, logger)
handler, err = web.NewHandlerWithLifecycleAndBackup(auth.New(db), repository, agent, backupService, importService, logger)
}
if err != nil {
return err
@@ -86,6 +98,10 @@ func run(logger *slog.Logger) error {
if lifecycle != nil {
go reconcileInstances(ctx, lifecycle, logger)
}
if backupService != nil {
go runBackupScheduler(ctx, backupService, logger)
}
go runImportCleanup(ctx, importService, logger)
server := &http.Server{
Addr: listenAddress,
Handler: handler,
@@ -112,6 +128,38 @@ func run(logger *slog.Logger) error {
}
}
func runImportCleanup(ctx context.Context, service *importexport.Service, logger *slog.Logger) {
ticker := time.NewTicker(time.Hour)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := service.CleanupExpired(ctx); err != nil {
logger.Warn("expired import cleanup incomplete", "event", "import.cleanup.failed")
}
}
}
}
func runBackupScheduler(ctx context.Context, service *backup.Service, logger *slog.Logger) {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
runCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
if err := service.RunDue(runCtx); err != nil {
logger.Warn("scheduled backup run incomplete", "event", "backup.scheduler.failed")
}
cancel()
}
}
}
func reconcileInstances(ctx context.Context, lifecycle *instance.LifecycleService, logger *slog.Logger) {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
compose.yaml
+3
View File
@@ -9,6 +9,9 @@ services:
DOGAMA_AGENT_URL: http://agent:8081
DOGAMA_AGENT_TOKEN_FILE: /run/secrets/agent_token
DOGAMA_MASTER_KEY_FILE: /run/secrets/master_key
DOGAMA_SERVERS_ROOT: /srv/game-servers
DOGAMA_BACKUPS_ROOT: /srv/game-backups
DOGAMA_IMPORTS_ROOT: /var/lib/dogama/imports/staging
secrets:
- agent_token
- master_key
docs/contributing/development.md
+23
View File
@@ -76,6 +76,29 @@ request review additionally require an administrator session authenticated in
the previous ten minutes. Approving a request only records the decision and the
requested values; it never creates a draft or contacts the restricted agent.
Game-data backups are written below `DOGAMA_BACKUPS_ROOT` (default
`/srv/game-backups`) and may only read instance mounts below
`DOGAMA_SERVERS_ROOT` (default `/srv/game-servers`). Untrusted uploads are
isolated below `DOGAMA_IMPORTS_ROOT` (default
`/var/lib/dogama/imports/staging`). These are bootstrap path boundaries, not
ordinary product settings. The same canonical server and backup roots are
mounted into the main application and restricted agent by `compose.yaml`.
The current backup engine conservatively stops a running instance before
archiving. Once the WebAssembly runtime is active, an enabled `online_save`
module can provide the documented flush-before-archive optimization without
moving traversal or archive ownership out of the main application. Archives
are finalized before SQLite marks them available; a metadata failure removes
the orphaned file. Scheduled retention considers only successful `scheduled`
backups. Restore verifies size, checksum, manifest and pinned template version,
creates a `pre_restore` backup, extracts into sibling staging and keeps the
instance stopped with `intervention_required` if readiness cannot be restored.
Validated imports are pinned to the selected template version. Import-backed
drafts require that opaque import ID, and installation atomically places the
normalized staged tree at the template-declared mount-relative destination
before the restricted agent creates the first container. Repeated installation
submission recognizes an already attached import instead of copying it twice.
At main-application startup, every embedded `catalog/*/template.yaml` is
validated against `specs/template.schema.json`, checked for cross-reference and
asset integrity, canonicalized deterministically and synchronized into SQLite.
docs/operations/backups-import-export.md
+21
View File
@@ -91,3 +91,24 @@ The preview reports detected game/type, file count, expanded size, world/player
The reference template recognizes dedicated-server world layouts with `Level.sav` and `Players/`. A local hosted-world import may require player identity conversion. V1 must preserve the upload, warn about this possibility and never perform undocumented silent conversion. A future game-specific data converter remains separate from the WebAssembly API adapter because modules have no filesystem access.
## Implemented foundation
The V1 backup foundation persists policies, archives and import-validation
records in SQLite. Manual backups, listing, integrity-checked export, restore
and five-field cron policy APIs are protected by the stable instance permission
identifiers. Restore and backup-policy mutation use the existing recent-session
safeguard.
Until an integration module is activated, the generic engine uses the safe
`stop_then_archive` behavior even when a template advertises `online_save`.
The later WebAssembly milestone supplies the capability call; modules will
still never traverse or archive files.
ZIP, tar, tar.gz and tar.zst imports are copied to a unique staging directory
before validation. Extraction uses create-new files and rejects absolute or
Windows paths, traversal, links, special files, excessive nesting, excessive
file counts and expanded-size overflow. A compatible validated import can be
selected in an administrator creation preview. Its normalized data is copied
through a create-new sibling directory into the template-declared destination
immediately before first container creation; validation itself never creates a
container or writes into live player data.
go.mod
+3 -1
View File
@@ -4,8 +4,11 @@ go 1.25.0
require (
github.com/dlclark/regexp2 v1.12.0
github.com/klauspost/compress v1.18.0
github.com/robfig/cron/v3 v3.0.1
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
golang.org/x/crypto v0.53.0
golang.org/x/sys v0.47.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.56.0
)
@@ -16,7 +19,6 @@ require (
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.38.0 // indirect
modernc.org/libc v1.74.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
go.sum
+4
View File
@@ -8,12 +8,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
internal/backup/service.go
+745
View File
@@ -0,0 +1,745 @@
// Package backup owns game-data archives, retention and recoverable restores.
package backup
import (
"archive/tar"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
"github.com/klauspost/compress/zstd"
"github.com/robfig/cron/v3"
"golang.org/x/sys/unix"
)
var (
ErrNotFound = errors.New("backup not found")
ErrInvalidInput = errors.New("invalid backup input")
ErrInvalidState = errors.New("invalid backup state")
ErrUnsafePath = errors.New("unsafe backup path")
ErrIntegrity = errors.New("backup integrity check failed")
ErrIncompatible = errors.New("backup is incompatible")
)
const manifestSchemaVersion = 1
type Backup struct {
ID string `json:"id"`
InstanceID string `json:"instance_id"`
Origin string `json:"origin"`
Status string `json:"status"`
SizeBytes int64 `json:"size_bytes,omitempty"`
SHA256 string `json:"sha256,omitempty"`
CreatedAt string `json:"created_at"`
CompletedAt string `json:"completed_at,omitempty"`
ErrorCode string `json:"error_code,omitempty"`
RelativePath string `json:"-"`
}
type Policy struct {
InstanceID string `json:"instance_id"`
Enabled bool `json:"enabled"`
CronExpression string `json:"cron_expression,omitempty"`
Timezone string `json:"timezone"`
RetentionCount int `json:"retention_count"`
NextRunAt string `json:"next_run_at,omitempty"`
}
type Manifest struct {
SchemaVersion int `json:"schema_version"`
BackupID string `json:"backup_id"`
InstanceID string `json:"instance_id"`
TemplateID string `json:"template_id"`
TemplateVersion string `json:"template_version"`
Origin string `json:"origin"`
CreatedAt string `json:"created_at"`
MountIDs []string `json:"mount_ids"`
}
type Repository interface {
instance.LifecycleRepository
BeginBackup(context.Context, Backup, string, string) error
CompleteBackup(context.Context, string, string, int64, string, Manifest) error
FailBackup(context.Context, string, string) error
ListBackups(context.Context, string) ([]Backup, error)
GetBackup(context.Context, string, string) (Backup, Manifest, error)
RetentionCandidates(context.Context, string, int) ([]Backup, error)
MarkBackupDeleted(context.Context, string) error
GetBackupPolicy(context.Context, string) (Policy, error)
SetBackupPolicy(context.Context, Policy) error
ListDueBackupPolicies(context.Context, string) ([]Policy, error)
}
type Agent interface {
StartInstance(context.Context, string) (agentwire.InstanceState, error)
StopInstance(context.Context, string, int) (agentwire.InstanceState, error)
}
type Service struct {
repository Repository
agent Agent
serversRoot string
backupsRoot string
now func() time.Time
}
func New(repository Repository, agent Agent, serversRoot, backupsRoot string) (*Service, error) {
serversRoot, err := canonicalRoot(serversRoot)
if err != nil {
return nil, fmt.Errorf("server root: %w", err)
}
backupsRoot, err = canonicalRoot(backupsRoot)
if err != nil {
return nil, fmt.Errorf("backup root: %w", err)
}
return &Service{repository: repository, agent: agent, serversRoot: serversRoot, backupsRoot: backupsRoot, now: time.Now}, nil
}
func (s *Service) Create(ctx context.Context, actorID, instanceID, origin string) (Backup, error) {
if !validOrigin(origin) || instanceID == "" {
return Backup{}, ErrInvalidInput
}
current, err := s.repository.GetInstance(ctx, instanceID)
if err != nil {
return Backup{}, err
}
if current.ContainerID == "" {
return Backup{}, ErrInvalidState
}
operationID, backupID := token(), token()
if operationID == "" || backupID == "" {
return Backup{}, errors.New("generate backup identifiers")
}
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "backup", "backup")
if err != nil {
return Backup{}, err
}
backup := Backup{ID: backupID, InstanceID: instanceID, Origin: origin, Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)}
if err := s.repository.BeginBackup(ctx, backup, operationID, actorID); err != nil {
_ = s.repository.FailOperation(ctx, operationID, "error", "backup_metadata_failed")
return Backup{}, err
}
wasRunning := current.DesiredRunning
if wasRunning {
if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil {
return Backup{}, s.fail(ctx, operationID, backupID, "backup_stop_failed", err)
}
}
completed, manifest, err := s.writeArchive(ctx, current, backup)
if err != nil {
if wasRunning {
if _, restartErr := s.agent.StartInstance(ctx, instanceID); restartErr != nil {
_ = s.repository.FailBackup(ctx, backupID, "backup_restart_failed")
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed")
return Backup{}, fmt.Errorf("backup failed and restart failed: %v: %w", err, restartErr)
}
}
return Backup{}, s.fail(ctx, operationID, backupID, "backup_archive_failed", err)
}
if err := s.repository.CompleteBackup(ctx, backupID, completed.RelativePath, completed.SizeBytes, completed.SHA256, manifest); err != nil {
if full, pathErr := s.backupPath(completed.RelativePath); pathErr == nil {
_ = os.Remove(full)
}
if wasRunning {
_, _ = s.agent.StartInstance(ctx, instanceID)
}
return Backup{}, s.fail(ctx, operationID, backupID, "backup_persist_failed", err)
}
state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped"
if wasRunning {
state, err = s.agent.StartInstance(ctx, instanceID)
if err != nil {
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed")
return Backup{}, fmt.Errorf("restart after backup: %w", err)
}
lifecycle, observed = lifecycleFromAgent(state)
}
if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, ""); err != nil {
return Backup{}, err
}
completed.Status, completed.CompletedAt = "available", s.now().UTC().Format(time.RFC3339Nano)
if err := s.applyRetention(ctx, instanceID, current.Preview.Backup.RetentionCount); err != nil {
return Backup{}, fmt.Errorf("apply retention: %w", err)
}
return completed, nil
}
func (s *Service) List(ctx context.Context, instanceID string) ([]Backup, error) {
return s.repository.ListBackups(ctx, instanceID)
}
func (s *Service) GetPolicy(ctx context.Context, instanceID string) (Policy, error) {
return s.repository.GetBackupPolicy(ctx, instanceID)
}
func (s *Service) SetPolicy(ctx context.Context, value Policy) (Policy, error) {
if value.InstanceID == "" || value.RetentionCount < 1 || value.RetentionCount > 1000 {
return Policy{}, ErrInvalidInput
}
location, err := time.LoadLocation(value.Timezone)
if err != nil {
return Policy{}, ErrInvalidInput
}
if !value.Enabled {
value.CronExpression, value.NextRunAt = "", ""
} else {
schedule, err := cronParser().Parse(value.CronExpression)
if err != nil {
return Policy{}, ErrInvalidInput
}
value.NextRunAt = schedule.Next(s.now().In(location)).UTC().Format(time.RFC3339Nano)
}
if err := s.repository.SetBackupPolicy(ctx, value); err != nil {
return Policy{}, err
}
return value, nil
}
func (s *Service) RunDue(ctx context.Context) error {
now := s.now().UTC()
policies, err := s.repository.ListDueBackupPolicies(ctx, now.Format(time.RFC3339Nano))
if err != nil {
return err
}
for _, policy := range policies {
location, locationErr := time.LoadLocation(policy.Timezone)
schedule, parseErr := cronParser().Parse(policy.CronExpression)
if locationErr != nil || parseErr != nil {
continue
}
policy.NextRunAt = schedule.Next(now.In(location)).UTC().Format(time.RFC3339Nano)
if err := s.repository.SetBackupPolicy(ctx, policy); err != nil {
return err
}
if _, err := s.Create(ctx, "", policy.InstanceID, "scheduled"); err != nil && !errors.Is(err, instance.ErrOperationConflict) {
return err
}
}
return nil
}
func (s *Service) Export(ctx context.Context, instanceID, backupID string) (*os.File, Backup, error) {
backup, _, err := s.repository.GetBackup(ctx, instanceID, backupID)
if err != nil {
return nil, Backup{}, err
}
if backup.Status != "available" {
return nil, Backup{}, ErrInvalidState
}
full, err := s.backupPath(backup.RelativePath)
if err != nil {
return nil, Backup{}, err
}
if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil {
return nil, Backup{}, err
}
file, err := os.Open(full)
return file, backup, err
}
func (s *Service) Restore(ctx context.Context, actorID, instanceID, backupID string) error {
current, err := s.repository.GetInstance(ctx, instanceID)
if err != nil {
return err
}
backup, manifest, err := s.repository.GetBackup(ctx, instanceID, backupID)
if err != nil {
return err
}
if backup.Status != "available" {
return ErrInvalidState
}
if manifest.InstanceID != instanceID || manifest.TemplateID != current.Preview.Template.ID || manifest.TemplateVersion != current.Preview.Template.Version {
return ErrIncompatible
}
full, err := s.backupPath(backup.RelativePath)
if err != nil {
return err
}
if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil {
return err
}
operationID := token()
current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "restore", "restore")
if err != nil {
return err
}
wasRunning := current.DesiredRunning
if wasRunning {
if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil {
return s.restoreFail(ctx, operationID, "restore_stop_failed", err)
}
}
safety := Backup{ID: token(), InstanceID: instanceID, Origin: "pre_restore", Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)}
if err := s.repository.BeginBackup(ctx, safety, operationID, actorID); err != nil {
return s.restoreFail(ctx, operationID, "restore_safety_metadata_failed", err)
}
safety, safetyManifest, err := s.writeArchive(ctx, current, safety)
if err != nil {
return s.restoreFail(ctx, operationID, "restore_safety_backup_failed", err)
}
if err := s.repository.CompleteBackup(ctx, safety.ID, safety.RelativePath, safety.SizeBytes, safety.SHA256, safetyManifest); err != nil {
return s.restoreFail(ctx, operationID, "restore_safety_persist_failed", err)
}
if err := s.restoreArchive(full, current, manifest); err != nil {
return s.restoreFail(ctx, operationID, "restore_extract_failed", err)
}
state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped"
if wasRunning {
state, err = s.agent.StartInstance(ctx, instanceID)
if err != nil {
return s.restoreFail(ctx, operationID, "restore_restart_failed", err)
}
lifecycle, observed = lifecycleFromAgent(state)
}
return s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, "")
}
func (s *Service) writeArchive(ctx context.Context, current instance.StoredInstance, backup Backup) (Backup, Manifest, error) {
mounts := make(map[string]string)
for _, mount := range current.Preview.Mounts {
mounts[mount.ID] = mount.HostPath
}
manifest := Manifest{SchemaVersion: manifestSchemaVersion, BackupID: backup.ID, InstanceID: backup.InstanceID, TemplateID: current.Preview.Template.ID, TemplateVersion: current.Preview.Template.Version, Origin: backup.Origin, CreatedAt: backup.CreatedAt, MountIDs: append([]string(nil), current.Preview.Backup.SourceMounts...)}
sort.Strings(manifest.MountIDs)
directory, err := s.instanceBackupDirectory(backup.InstanceID)
if err != nil {
return Backup{}, Manifest{}, err
}
if err := os.MkdirAll(directory, 0o750); err != nil {
return Backup{}, Manifest{}, err
}
var estimated int64
for _, mountID := range manifest.MountIDs {
source := mounts[mountID]
if source == "" {
return Backup{}, Manifest{}, ErrUnsafePath
}
if _, err := s.serverPath(source); err != nil {
return Backup{}, Manifest{}, err
}
size, err := estimateTree(ctx, source)
if err != nil {
return Backup{}, Manifest{}, err
}
estimated += size
}
if err := ensureFreeSpace(directory, estimated+(64<<20)); err != nil {
return Backup{}, Manifest{}, err
}
temporary, err := os.CreateTemp(directory, ".creating-*.tar.zst")
if err != nil {
return Backup{}, Manifest{}, err
}
temporaryName := temporary.Name()
defer func() { _ = temporary.Close(); _ = os.Remove(temporaryName) }()
hasher := sha256.New()
zstdWriter, err := zstd.NewWriter(io.MultiWriter(temporary, hasher), zstd.WithEncoderConcurrency(1))
if err != nil {
return Backup{}, Manifest{}, err
}
tarWriter := tar.NewWriter(zstdWriter)
manifestJSON, _ := json.Marshal(manifest)
err = tarWriter.WriteHeader(&tar.Header{Name: "manifest.json", Mode: 0o600, Size: int64(len(manifestJSON)), ModTime: s.now().UTC(), Typeflag: tar.TypeReg})
if err == nil {
_, err = tarWriter.Write(manifestJSON)
}
if err == nil {
for _, mountID := range manifest.MountIDs {
source := mounts[mountID]
if source == "" {
err = ErrUnsafePath
break
}
if _, pathErr := s.serverPath(source); pathErr != nil {
err = pathErr
break
}
if walkErr := addTree(ctx, tarWriter, source, path.Join("data", mountID)); walkErr != nil {
err = walkErr
break
}
}
}
if closeErr := tarWriter.Close(); err == nil {
err = closeErr
}
if closeErr := zstdWriter.Close(); err == nil {
err = closeErr
}
if syncErr := temporary.Sync(); err == nil {
err = syncErr
}
if closeErr := temporary.Close(); err == nil {
err = closeErr
}
if err != nil {
return Backup{}, Manifest{}, err
}
info, err := os.Stat(temporaryName)
if err != nil {
return Backup{}, Manifest{}, err
}
finalName := backup.ID + ".tar.zst"
finalPath := filepath.Join(directory, finalName)
if err := os.Rename(temporaryName, finalPath); err != nil {
return Backup{}, Manifest{}, err
}
backup.RelativePath = filepath.ToSlash(filepath.Join(backup.InstanceID, finalName))
backup.SizeBytes, backup.SHA256 = info.Size(), hex.EncodeToString(hasher.Sum(nil))
return backup, manifest, nil
}
func addTree(ctx context.Context, writer *tar.Writer, source, prefix string) error {
return filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) {
return ErrUnsafePath
}
relative, err := filepath.Rel(source, current)
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return ErrUnsafePath
}
name := prefix
if relative != "." {
name = path.Join(prefix, filepath.ToSlash(relative))
}
header, err := tar.FileInfoHeader(info, "")
if err != nil {
return err
}
header.Name, header.Uid, header.Gid, header.Uname, header.Gname = name, 0, 0, "", ""
if err := writer.WriteHeader(header); err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
file, err := os.Open(current)
if err != nil {
return err
}
_, copyErr := io.Copy(writer, file)
closeErr := file.Close()
if copyErr != nil {
return copyErr
}
return closeErr
})
}
func estimateTree(ctx context.Context, source string) (int64, error) {
var total int64
err := filepath.WalkDir(source, func(_ string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) {
return ErrUnsafePath
}
if info.Mode().IsRegular() {
total += info.Size()
}
return nil
})
return total, err
}
func ensureFreeSpace(directory string, required int64) error {
var stats unix.Statfs_t
if err := unix.Statfs(directory, &stats); err != nil {
return err
}
available := int64(stats.Bavail) * int64(stats.Bsize)
if available < required {
return ErrInvalidState
}
return nil
}
func (s *Service) restoreArchive(archive string, current instance.StoredInstance, manifest Manifest) error {
stage, err := os.MkdirTemp(s.serversRoot, ".dogama-restore-")
if err != nil {
return err
}
defer os.RemoveAll(stage)
file, err := os.Open(archive)
if err != nil {
return err
}
defer file.Close()
decoder, err := zstd.NewReader(file, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(2<<30))
if err != nil {
return err
}
defer decoder.Close()
reader := tar.NewReader(decoder)
var total int64
sawManifest := false
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
if header.Name == "manifest.json" {
if sawManifest || header.Size < 1 || header.Size > 64<<10 {
return ErrIntegrity
}
var archived Manifest
if err := json.NewDecoder(io.LimitReader(reader, header.Size)).Decode(&archived); err != nil {
return ErrIntegrity
}
if archived.SchemaVersion != manifestSchemaVersion || archived.BackupID != manifest.BackupID || archived.InstanceID != manifest.InstanceID || archived.TemplateID != manifest.TemplateID || archived.TemplateVersion != manifest.TemplateVersion || strings.Join(archived.MountIDs, "\x00") != strings.Join(manifest.MountIDs, "\x00") {
return ErrIntegrity
}
sawManifest = true
continue
}
if !fs.ValidPath(header.Name) || !strings.HasPrefix(header.Name, "data/") || header.Linkname != "" {
return ErrUnsafePath
}
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir {
return ErrUnsafePath
}
total += header.Size
if total > 1<<40 {
return ErrInvalidInput
}
target := filepath.Join(stage, filepath.FromSlash(header.Name))
if !within(stage, target) {
return ErrUnsafePath
}
if header.Typeflag == tar.TypeDir {
if err := os.MkdirAll(target, 0o750); err != nil {
return err
}
continue
}
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
if err != nil {
return err
}
_, copyErr := io.CopyN(out, reader, header.Size)
closeErr := out.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
}
if !sawManifest {
return ErrIntegrity
}
mountPaths := make(map[string]string)
for _, mount := range current.Preview.Mounts {
mountPaths[mount.ID] = mount.HostPath
}
var swapped []struct{ live, previous string }
rollback := func() {
for index := len(swapped) - 1; index >= 0; index-- {
_ = os.RemoveAll(swapped[index].live)
_ = os.Rename(swapped[index].previous, swapped[index].live)
}
}
for _, mountID := range manifest.MountIDs {
live := mountPaths[mountID]
if _, err := s.serverPath(live); err != nil {
rollback()
return err
}
staged := filepath.Join(stage, "data", mountID)
if _, err := os.Stat(staged); err != nil {
rollback()
return ErrIntegrity
}
previous := live + ".dogama-previous-" + manifest.BackupID
if err := os.Rename(live, previous); err != nil {
rollback()
return err
}
if err := os.Rename(staged, live); err != nil {
_ = os.Rename(previous, live)
rollback()
return err
}
swapped = append(swapped, struct{ live, previous string }{live, previous})
}
for _, item := range swapped {
if err := os.RemoveAll(item.previous); err != nil {
return err
}
}
return nil
}
func (s *Service) applyRetention(ctx context.Context, instanceID string, count int) error {
candidates, err := s.repository.RetentionCandidates(ctx, instanceID, count)
if err != nil {
return err
}
for _, candidate := range candidates {
full, err := s.backupPath(candidate.RelativePath)
if err != nil {
return err
}
if err := os.Remove(full); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err := s.repository.MarkBackupDeleted(ctx, candidate.ID); err != nil {
return err
}
}
return nil
}
func (s *Service) fail(ctx context.Context, operationID, backupID, code string, cause error) error {
_ = s.repository.FailBackup(ctx, backupID, code)
_ = s.repository.FailOperation(ctx, operationID, "error", code)
return fmt.Errorf("%s: %w", code, cause)
}
func (s *Service) restoreFail(ctx context.Context, operationID, code string, cause error) error {
_ = s.repository.FailOperation(ctx, operationID, "intervention_required", code)
return fmt.Errorf("%s: %w", code, cause)
}
func (s *Service) instanceBackupDirectory(instanceID string) (string, error) {
if instanceID == "" || strings.ContainsAny(instanceID, `/\\`) {
return "", ErrUnsafePath
}
result := filepath.Join(s.backupsRoot, instanceID)
if !within(s.backupsRoot, result) {
return "", ErrUnsafePath
}
return result, nil
}
func (s *Service) backupPath(relative string) (string, error) {
if !fs.ValidPath(relative) {
return "", ErrUnsafePath
}
result := filepath.Join(s.backupsRoot, filepath.FromSlash(relative))
if !within(s.backupsRoot, result) {
return "", ErrUnsafePath
}
return result, nil
}
func (s *Service) serverPath(value string) (string, error) {
clean := filepath.Clean(value)
if !filepath.IsAbs(clean) || !within(s.serversRoot, clean) {
return "", ErrUnsafePath
}
return clean, nil
}
func canonicalRoot(value string) (string, error) {
if value == "" {
return "", ErrUnsafePath
}
absolute, err := filepath.Abs(value)
if err != nil {
return "", err
}
if err := os.MkdirAll(absolute, 0o750); err != nil {
return "", err
}
return filepath.EvalSymlinks(absolute)
}
func within(root, candidate string) bool {
relative, err := filepath.Rel(root, candidate)
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}
func verifyFile(name string, expectedSize int64, expectedSHA string) error {
file, err := os.Open(name)
if err != nil {
return err
}
defer file.Close()
hasher := sha256.New()
size, err := io.Copy(hasher, file)
if err != nil {
return err
}
if size != expectedSize || !strings.EqualFold(hex.EncodeToString(hasher.Sum(nil)), expectedSHA) {
return ErrIntegrity
}
return nil
}
func validOrigin(value string) bool {
switch value {
case "manual", "scheduled", "pre_update", "pre_restore", "idle_shutdown", "imported", "system":
return true
}
return false
}
func token() string {
buffer := make([]byte, 24)
if _, err := rand.Read(buffer); err != nil {
return ""
}
return base64.RawURLEncoding.EncodeToString(buffer)
}
func lifecycleFromAgent(state agentwire.InstanceState) (string, string) {
if !state.Running {
return "stopped", "stopped"
}
if state.Ready {
return "online", "ready"
}
if state.Health == "unhealthy" || state.Health == "none" {
return "degraded", "degraded"
}
return "starting", "running"
}
func cronParser() cron.Parser {
return cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
}
internal/backup/service_test.go
+205
View File
@@ -0,0 +1,205 @@
package backup_test
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
catalogdata "git.zaynet.fr/DoGaMa/DoGaMa-serv/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite"
)
type backupAgent struct{ running bool }
func (a *backupAgent) StartInstance(_ context.Context, id string) (agentwire.InstanceState, error) {
a.running = true
return agentwire.InstanceState{InstanceID: id, ContainerID: "container", Running: true, Ready: true, Health: "healthy"}, nil
}
func (a *backupAgent) StopInstance(_ context.Context, id string, _ int) (agentwire.InstanceState, error) {
a.running = false
return agentwire.InstanceState{InstanceID: id, ContainerID: "container", Health: "stopped"}, nil
}
func TestCreateRestoreExportAndRetention(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
serversRoot, backupsRoot := filepath.Join(root, "servers"), filepath.Join(root, "backups")
mount := filepath.Join(serversRoot, "instance", "saved")
if err := os.MkdirAll(mount, 0o750); err != nil {
t.Fatal(err)
}
world := filepath.Join(mount, "Level.sav")
if err := os.WriteFile(world, []byte("world-v1"), 0o640); err != nil {
t.Fatal(err)
}
db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db"))
if err != nil {
t.Fatal(err)
}
defer db.Close()
repository := sqlite.NewRepository(db)
snapshots, err := catalog.LoadFS(catalogdata.Files, ".")
if err != nil {
t.Fatal(err)
}
if err := repository.Sync(ctx, snapshots); err != nil {
t.Fatal(err)
}
authService := auth.New(db)
if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil {
t.Fatal(err)
}
adminSession, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234")
if err != nil {
t.Fatal(err)
}
admin, err := authService.Authenticate(ctx, adminSession.Token)
if err != nil {
t.Fatal(err)
}
preview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{
DisplayName: "Backup Test", Slug: "backup-test", HostPorts: map[string]int{"game": 38211},
MountPaths: map[string]string{"saved": mount}, DataOrigin: "new", BackupRetention: 2,
})
if err != nil {
t.Fatal(err)
}
const instanceID = "backup-instance"
if err := repository.CreateDraft(ctx, instance.Draft{ID: instanceID, Preview: preview}); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`UPDATE instances SET lifecycle_state='online', observed_state='ready', container_id='container', desired_running=1 WHERE id=?`, instanceID); err != nil {
t.Fatal(err)
}
agent := &backupAgent{running: true}
service, err := backup.New(repository, agent, serversRoot, backupsRoot)
if err != nil {
t.Fatal(err)
}
manual, err := service.Create(ctx, admin.ID, instanceID, "manual")
if err != nil {
t.Fatal(err)
}
if manual.Status != "available" || manual.SHA256 == "" || manual.SizeBytes == 0 || !agent.running {
t.Fatalf("backup = %#v, running=%v", manual, agent.running)
}
exported, metadata, err := service.Export(ctx, instanceID, manual.ID)
if err != nil {
t.Fatal(err)
}
if metadata.ID != manual.ID {
t.Fatalf("export metadata = %#v", metadata)
}
if err := exported.Close(); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(world, []byte("world-v2"), 0o640); err != nil {
t.Fatal(err)
}
if err := service.Restore(ctx, admin.ID, instanceID, manual.ID); err != nil {
t.Fatal(err)
}
restored, err := os.ReadFile(world)
if err != nil || string(restored) != "world-v1" {
t.Fatalf("restored world = %q, error = %v", restored, err)
}
for index := 0; index < 3; index++ {
if _, err := service.Create(ctx, admin.ID, instanceID, "scheduled"); err != nil {
t.Fatal(err)
}
}
backups, err := service.List(ctx, instanceID)
if err != nil {
t.Fatal(err)
}
var manualCount, scheduledCount, safetyCount int
for _, item := range backups {
switch item.Origin {
case "manual":
manualCount++
case "scheduled":
scheduledCount++
case "pre_restore":
safetyCount++
}
}
if manualCount != 1 || scheduledCount != 2 || safetyCount != 1 {
t.Fatalf("retained backups: manual=%d scheduled=%d safety=%d", manualCount, scheduledCount, safetyCount)
}
}
func TestBackupRejectsSymlinks(t *testing.T) {
ctx, service, actorID, instanceID, mount := backupFixture(t)
outside := filepath.Join(t.TempDir(), "outside")
if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(mount, "escape")); err != nil {
t.Fatal(err)
}
_, err := service.Create(ctx, actorID, instanceID, "manual")
if !errors.Is(err, backup.ErrUnsafePath) {
t.Fatalf("symlink backup error = %v", err)
}
}
func backupFixture(t *testing.T) (context.Context, *backup.Service, string, string, string) {
t.Helper()
ctx := context.Background()
root := t.TempDir()
serversRoot := filepath.Join(root, "servers")
backupsRoot := filepath.Join(root, "backups")
mount := filepath.Join(serversRoot, "instance", "saved")
if err := os.MkdirAll(mount, 0o750); err != nil {
t.Fatal(err)
}
db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
repository := sqlite.NewRepository(db)
snapshots, err := catalog.LoadFS(catalogdata.Files, ".")
if err != nil {
t.Fatal(err)
}
if err := repository.Sync(ctx, snapshots); err != nil {
t.Fatal(err)
}
authService := auth.New(db)
if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil {
t.Fatal(err)
}
session, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234")
if err != nil {
t.Fatal(err)
}
actor, err := authService.Authenticate(ctx, session.Token)
if err != nil {
t.Fatal(err)
}
preview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{DisplayName: "Backup Test", Slug: "backup-test", HostPorts: map[string]int{"game": 38211}, MountPaths: map[string]string{"saved": mount}, DataOrigin: "new", BackupRetention: 2})
if err != nil {
t.Fatal(err)
}
instanceID := "backup-instance"
if err := repository.CreateDraft(ctx, instance.Draft{ID: instanceID, Preview: preview}); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`UPDATE instances SET lifecycle_state='stopped', observed_state='stopped', container_id='container' WHERE id=?`, instanceID); err != nil {
t.Fatal(err)
}
service, err := backup.New(repository, &backupAgent{}, serversRoot, backupsRoot)
if err != nil {
t.Fatal(err)
}
return ctx, service, actor.ID, instanceID, mount
}
internal/catalog/template.go
+7 -2
View File
@@ -88,8 +88,13 @@ type Template struct {
IntervalSeconds int `json:"interval_seconds"`
} `json:"healthcheck"`
Imports struct {
Supported bool `json:"supported"`
DestinationMount string `json:"destination_mount"`
Supported bool `json:"supported"`
AcceptedFormats []string `json:"accepted_formats"`
MaxExtractedSizeGB int `json:"max_extracted_size_gb"`
RequiredPaths []string `json:"required_paths"`
DestinationMount string `json:"destination_mount"`
DestinationRelativePath string `json:"destination_relative_path"`
RequiresStoppedServer bool `json:"requires_stopped_server"`
} `json:"imports"`
}
internal/importexport/service.go
+521
View File
@@ -0,0 +1,521 @@
// Package importexport validates untrusted game-data archives in isolated staging.
package importexport
import (
"archive/tar"
"archive/zip"
"compress/gzip"
"context"
"crypto/rand"
"encoding/base64"
"errors"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/klauspost/compress/zstd"
)
var (
ErrInvalidInput = errors.New("invalid import input")
ErrUnsafeArchive = errors.New("unsafe import archive")
ErrLimitExceeded = errors.New("import limit exceeded")
ErrNotRecognized = errors.New("import layout not recognized")
)
const (
maxFiles = 100000
maxDepth = 20
)
type Policy struct {
TemplateID string
TemplateVersion string
AcceptedFormats []string
MaxExpandedBytes int64
RequiredPaths []string
}
type Import struct {
ID string `json:"id"`
Status string `json:"status"`
Format string `json:"format"`
DetectedType string `json:"detected_type,omitempty"`
Confidence string `json:"confidence,omitempty"`
FileCount int `json:"file_count"`
ExpandedSizeBytes int64 `json:"expanded_size_bytes"`
ExpiresAt string `json:"expires_at"`
RelativeStagePath string `json:"-"`
DataRoot string `json:"-"`
TemplateID string `json:"template_id"`
TemplateVersion string `json:"template_version"`
InstanceID string `json:"instance_id,omitempty"`
}
type Repository interface {
BeginImport(context.Context, Import, string) error
CompleteImport(context.Context, Import) error
FailImport(context.Context, string, string) error
ExpireImports(context.Context, string) ([]string, error)
GetImport(context.Context, string) (Import, error)
AttachImport(context.Context, string, string) error
}
func (s *Service) CleanupExpired(ctx context.Context) error {
paths, err := s.repository.ExpireImports(ctx, s.now().UTC().Format(time.RFC3339Nano))
if err != nil {
return err
}
for _, relative := range paths {
if !fs.ValidPath(relative) {
return ErrUnsafeArchive
}
target := filepath.Join(s.root, filepath.FromSlash(relative))
if !withinRoot(s.root, target) {
return ErrUnsafeArchive
}
if err := os.RemoveAll(target); err != nil {
return err
}
}
return nil
}
type Service struct {
repository Repository
root string
serversRoot string
now func() time.Time
}
func New(repository Repository, root, serversRoot string) (*Service, error) {
if root == "" || serversRoot == "" {
return nil, ErrInvalidInput
}
absolute, err := filepath.Abs(root)
if err != nil {
return nil, err
}
if err := os.MkdirAll(absolute, 0o750); err != nil {
return nil, err
}
canonical, err := filepath.EvalSymlinks(absolute)
if err != nil {
return nil, err
}
serverAbsolute, err := filepath.Abs(serversRoot)
if err != nil {
return nil, err
}
if err := os.MkdirAll(serverAbsolute, 0o750); err != nil {
return nil, err
}
serverCanonical, err := filepath.EvalSymlinks(serverAbsolute)
if err != nil {
return nil, err
}
return &Service{repository: repository, root: canonical, serversRoot: serverCanonical, now: time.Now}, nil
}
func (s *Service) Stage(ctx context.Context, actorID, format string, source io.Reader, policy Policy) (Import, error) {
if actorID == "" || policy.TemplateID == "" || policy.TemplateVersion == "" || !contains(policy.AcceptedFormats, format) || policy.MaxExpandedBytes < 1 {
return Import{}, ErrInvalidInput
}
id := importToken()
if id == "" {
return Import{}, errors.New("generate import identifier")
}
directory := filepath.Join(s.root, id)
if !withinRoot(s.root, directory) {
return Import{}, ErrUnsafeArchive
}
if err := os.Mkdir(directory, 0o750); err != nil {
return Import{}, err
}
value := Import{ID: id, Status: "staging", Format: format, RelativeStagePath: id, TemplateID: policy.TemplateID, TemplateVersion: policy.TemplateVersion, ExpiresAt: s.now().Add(24 * time.Hour).UTC().Format(time.RFC3339Nano)}
if err := s.repository.BeginImport(ctx, value, actorID); err != nil {
_ = os.RemoveAll(directory)
return Import{}, err
}
fail := func(code string, err error) (Import, error) {
_ = s.repository.FailImport(ctx, id, code)
_ = os.RemoveAll(directory)
return Import{}, err
}
upload := filepath.Join(directory, "upload")
file, err := os.OpenFile(upload, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return fail("import_stage_failed", err)
}
maxUpload := policy.MaxExpandedBytes
written, copyErr := io.Copy(file, io.LimitReader(source, maxUpload+1))
closeErr := file.Close()
if copyErr != nil || closeErr != nil {
if copyErr == nil {
copyErr = closeErr
}
return fail("import_upload_failed", copyErr)
}
if written > maxUpload {
return fail("import_upload_limit", ErrLimitExceeded)
}
extracted := filepath.Join(directory, "data")
if err := os.Mkdir(extracted, 0o750); err != nil {
return fail("import_stage_failed", err)
}
files, size, paths, err := extract(ctx, upload, extracted, format, policy.MaxExpandedBytes)
if err != nil {
return fail("import_validation_failed", err)
}
if !requiredPresent(paths, policy.RequiredPaths) {
return fail("import_layout_unrecognized", ErrNotRecognized)
}
dataRoot, err := detectedRoot(paths, policy.RequiredPaths)
if err != nil {
return fail("import_layout_ambiguous", err)
}
if err := os.Remove(upload); err != nil {
return fail("import_cleanup_failed", err)
}
value.Status, value.DetectedType, value.Confidence, value.FileCount, value.ExpandedSizeBytes, value.DataRoot = "validated", "game_save", "confirmed", files, size, dataRoot
if err := s.repository.CompleteImport(ctx, value); err != nil {
return fail("import_persist_failed", err)
}
return value, nil
}
func (s *Service) ValidateSelection(ctx context.Context, id, templateID, templateVersion string) error {
value, err := s.repository.GetImport(ctx, id)
if err != nil {
return err
}
if value.Status != "validated" || value.TemplateID != templateID || value.TemplateVersion != templateVersion {
return ErrInvalidInput
}
return nil
}
func (s *Service) ApplyToInstance(ctx context.Context, id, instanceID, templateID, templateVersion, mountPath, relativePath string) error {
value, err := s.repository.GetImport(ctx, id)
if err != nil {
return err
}
if value.InstanceID == instanceID && value.Status == "attached" {
return nil
}
if value.Status != "validated" || value.TemplateID != templateID || value.TemplateVersion != templateVersion {
return ErrInvalidInput
}
live := filepath.Join(mountPath, filepath.FromSlash(relativePath))
if !filepath.IsAbs(live) || !withinRoot(s.serversRoot, live) {
return ErrUnsafeArchive
}
source := filepath.Join(s.root, filepath.FromSlash(value.RelativeStagePath), "data", filepath.FromSlash(value.DataRoot))
if !withinRoot(s.root, source) {
return ErrUnsafeArchive
}
if info, err := os.Stat(source); err != nil || !info.IsDir() {
return ErrInvalidInput
}
if entries, err := os.ReadDir(live); err == nil && len(entries) != 0 {
return ErrInvalidInput
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err := os.MkdirAll(filepath.Dir(live), 0o750); err != nil {
return err
}
temporary, err := os.MkdirTemp(filepath.Dir(live), ".dogama-import-")
if err != nil {
return err
}
defer os.RemoveAll(temporary)
if err := copyValidatedTree(source, temporary); err != nil {
return err
}
if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err := os.Rename(temporary, live); err != nil {
return err
}
if err := s.repository.AttachImport(ctx, id, instanceID); err != nil {
_ = os.RemoveAll(live)
return err
}
return nil
}
func extract(ctx context.Context, archive, destination, format string, limit int64) (int, int64, []string, error) {
switch format {
case "zip":
return extractZIP(ctx, archive, destination, limit)
case "tar", "tar.gz", "tar.zst":
return extractTar(ctx, archive, destination, format, limit)
default:
return 0, 0, nil, ErrInvalidInput
}
}
func extractZIP(ctx context.Context, archive, destination string, limit int64) (int, int64, []string, error) {
reader, err := zip.OpenReader(archive)
if err != nil {
return 0, 0, nil, ErrUnsafeArchive
}
defer reader.Close()
var count int
var total int64
var paths []string
for _, entry := range reader.File {
select {
case <-ctx.Done():
return 0, 0, nil, ctx.Err()
default:
}
name, err := safeName(entry.Name)
if err != nil || entry.Mode()&os.ModeSymlink != 0 || (!entry.Mode().IsRegular() && !entry.FileInfo().IsDir()) {
return 0, 0, nil, ErrUnsafeArchive
}
if entry.FileInfo().IsDir() {
if err := os.MkdirAll(filepath.Join(destination, filepath.FromSlash(name)), 0o750); err != nil {
return 0, 0, nil, err
}
continue
}
count++
total += int64(entry.UncompressedSize64)
if count > maxFiles || total > limit {
return 0, 0, nil, ErrLimitExceeded
}
input, err := entry.Open()
if err != nil {
return 0, 0, nil, err
}
if err := writeExtracted(destination, name, input, int64(entry.UncompressedSize64)); err != nil {
input.Close()
return 0, 0, nil, err
}
if err := input.Close(); err != nil {
return 0, 0, nil, err
}
paths = append(paths, name)
}
return count, total, paths, nil
}
func extractTar(ctx context.Context, archive, destination, format string, limit int64) (int, int64, []string, error) {
file, err := os.Open(archive)
if err != nil {
return 0, 0, nil, err
}
defer file.Close()
var source io.Reader = file
var closer io.Closer
if format == "tar.gz" {
value, err := gzip.NewReader(file)
if err != nil {
return 0, 0, nil, ErrUnsafeArchive
}
source, closer = value, value
}
if format == "tar.zst" {
value, err := zstd.NewReader(file, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(2<<30))
if err != nil {
return 0, 0, nil, ErrUnsafeArchive
}
source, closer = value, value.IOReadCloser()
}
if closer != nil {
defer closer.Close()
}
reader := tar.NewReader(source)
var count int
var total int64
var paths []string
for {
select {
case <-ctx.Done():
return 0, 0, nil, ctx.Err()
default:
}
header, err := reader.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return 0, 0, nil, ErrUnsafeArchive
}
name, err := safeName(header.Name)
if err != nil || header.Linkname != "" {
return 0, 0, nil, ErrUnsafeArchive
}
if header.Typeflag == tar.TypeDir {
if err := os.MkdirAll(filepath.Join(destination, filepath.FromSlash(name)), 0o750); err != nil {
return 0, 0, nil, err
}
continue
}
if !header.FileInfo().Mode().IsRegular() {
return 0, 0, nil, ErrUnsafeArchive
}
count++
total += header.Size
if count > maxFiles || total > limit {
return 0, 0, nil, ErrLimitExceeded
}
if err := writeExtracted(destination, name, reader, header.Size); err != nil {
return 0, 0, nil, err
}
paths = append(paths, name)
}
return count, total, paths, nil
}
func writeExtracted(root, name string, source io.Reader, size int64) error {
target := filepath.Join(root, filepath.FromSlash(name))
if !withinRoot(root, target) {
return ErrUnsafeArchive
}
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
return err
}
file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
if err != nil {
return err
}
written, copyErr := io.CopyN(file, source, size)
closeErr := file.Close()
if copyErr != nil || written != size {
return ErrUnsafeArchive
}
return closeErr
}
func safeName(value string) (string, error) {
if strings.Contains(value, "\\") || strings.ContainsRune(value, 0) || strings.HasPrefix(value, "/") {
return "", ErrUnsafeArchive
}
clean := path.Clean(value)
if clean == "." || !fs.ValidPath(clean) || strings.Contains(strings.Split(clean, "/")[0], ":") || len(strings.Split(clean, "/")) > maxDepth {
return "", ErrUnsafeArchive
}
return clean, nil
}
func requiredPresent(paths, required []string) bool {
for _, expected := range required {
found := false
for _, candidate := range paths {
if candidate == expected || strings.HasSuffix(candidate, "/"+expected) || strings.HasPrefix(candidate, expected+"/") || strings.Contains(candidate, "/"+expected+"/") {
found = true
break
}
}
if !found {
return false
}
}
return true
}
func detectedRoot(paths, required []string) (string, error) {
root := ""
for _, expected := range required {
found := ""
for _, candidate := range paths {
marker := "/" + expected
if candidate == expected || strings.HasPrefix(candidate, expected+"/") {
found = "."
break
}
if index := strings.Index(candidate, marker); index >= 0 && (len(candidate) == index+len(marker) || candidate[index+len(marker)] == '/') {
found = candidate[:index]
break
}
}
if found == "" {
return "", ErrNotRecognized
}
if root == "" {
root = found
} else if root != found {
return "", ErrNotRecognized
}
}
if root == "" {
root = "."
}
return root, nil
}
func copyValidatedTree(source, destination string) error {
return filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) {
return ErrUnsafeArchive
}
relative, err := filepath.Rel(source, current)
if err != nil {
return err
}
if relative == "." {
return nil
}
target := filepath.Join(destination, relative)
if !withinRoot(destination, target) {
return ErrUnsafeArchive
}
if info.IsDir() {
return os.MkdirAll(target, 0o750)
}
input, err := os.Open(current)
if err != nil {
return err
}
output, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
if err != nil {
_ = input.Close()
return err
}
_, copyErr := io.Copy(output, input)
inputCloseErr := input.Close()
closeErr := output.Close()
if copyErr != nil {
return copyErr
}
if inputCloseErr != nil {
return inputCloseErr
}
return closeErr
})
}
func contains(values []string, expected string) bool {
for _, value := range values {
if value == expected {
return true
}
}
return false
}
func withinRoot(root, candidate string) bool {
relative, err := filepath.Rel(root, candidate)
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}
func importToken() string {
value := make([]byte, 24)
if _, err := rand.Read(value); err != nil {
return ""
}
return base64.RawURLEncoding.EncodeToString(value)
}
internal/importexport/service_test.go
+105
View File
@@ -0,0 +1,105 @@
package importexport_test
import (
"archive/zip"
"bytes"
"context"
"io"
"os"
"path/filepath"
"testing"
catalogdata "git.zaynet.fr/DoGaMa/DoGaMa-serv/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite"
)
func TestStageValidZIPAndRejectTraversal(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db"))
if err != nil {
t.Fatal(err)
}
defer db.Close()
repository := sqlite.NewRepository(db)
snapshots, err := catalog.LoadFS(catalogdata.Files, ".")
if err != nil {
t.Fatal(err)
}
if err := repository.Sync(ctx, snapshots); err != nil {
t.Fatal(err)
}
authService := auth.New(db)
if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil {
t.Fatal(err)
}
session, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234")
if err != nil {
t.Fatal(err)
}
actor, err := authService.Authenticate(ctx, session.Token)
if err != nil {
t.Fatal(err)
}
serversRoot := filepath.Join(root, "servers")
service, err := importexport.New(repository, filepath.Join(root, "imports"), serversRoot)
if err != nil {
t.Fatal(err)
}
policy := importexport.Policy{TemplateID: "palworld-official", TemplateVersion: "1.0.0", AcceptedFormats: []string{"zip"}, MaxExpandedBytes: 1 << 20, RequiredPaths: []string{"Level.sav", "Players"}}
valid := zipBytes(t, map[string]string{"Save/Level.sav": "world", "Save/Players/player.sav": "player"})
result, err := service.Stage(ctx, actor.ID, "zip", bytes.NewReader(valid), policy)
if err != nil {
t.Fatal(err)
}
if result.Status != "validated" || result.Confidence != "confirmed" || result.FileCount != 2 {
t.Fatalf("import = %#v", result)
}
destination := filepath.Join(serversRoot, "instance", "saved")
preview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{DisplayName: "Import Test", Slug: "import-test", HostPorts: map[string]int{"game": 38211}, MountPaths: map[string]string{"saved": destination}, DataOrigin: "new", BackupRetention: 2})
if err != nil {
t.Fatal(err)
}
if err := repository.CreateDraft(ctx, instance.Draft{ID: "instance-id", Preview: preview}); err != nil {
t.Fatal(err)
}
if err := service.ApplyToInstance(ctx, result.ID, "instance-id", policy.TemplateID, policy.TemplateVersion, destination, "SaveGames/0"); err != nil {
t.Fatal(err)
}
if body, err := os.ReadFile(filepath.Join(destination, "SaveGames", "0", "Level.sav")); err != nil || string(body) != "world" {
t.Fatalf("applied import=%q error=%v", body, err)
}
if err := service.ApplyToInstance(ctx, result.ID, "instance-id", policy.TemplateID, policy.TemplateVersion, destination, "SaveGames/0"); err != nil {
t.Fatalf("idempotent apply: %v", err)
}
unsafe := zipBytes(t, map[string]string{"../escape": "bad", "Level.sav": "world", "Players/player": "player"})
if _, err := service.Stage(ctx, actor.ID, "zip", bytes.NewReader(unsafe), policy); err == nil {
t.Fatal("traversal archive was accepted")
}
if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) {
t.Fatalf("escape path exists: %v", err)
}
}
func zipBytes(t *testing.T, files map[string]string) []byte {
t.Helper()
var buffer bytes.Buffer
writer := zip.NewWriter(&buffer)
for name, body := range files {
entry, err := writer.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := io.WriteString(entry, body); err != nil {
t.Fatal(err)
}
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
return buffer.Bytes()
}
internal/instance/preview.go
+15
View File
@@ -27,6 +27,7 @@ type PreviewRequest struct {
Resources catalog.Resources `json:"resources"`
DataOrigin string `json:"data_origin"`
BackupRetention int `json:"backup_retention"`
ImportID string `json:"import_id,omitempty"`
}
type Preview struct {
@@ -44,6 +45,7 @@ type Preview struct {
Settings []SettingPreview `json:"settings"`
DataOrigin string `json:"data_origin"`
Backup BackupPreview `json:"backup"`
Import ImportPreview `json:"import,omitempty"`
CanonicalJSON string `json:"canonical_json"`
PlanDigest string `json:"plan_digest"`
}
@@ -84,6 +86,12 @@ type BackupPreview struct {
RetentionCount int `json:"retention_count"`
}
type ImportPreview struct {
ID string `json:"id,omitempty"`
DestinationMount string `json:"destination_mount,omitempty"`
DestinationRelativePath string `json:"destination_relative_path,omitempty"`
}
type Draft struct {
ID string
Preview Preview
@@ -106,6 +114,12 @@ func BuildPreview(snapshot catalog.Snapshot, request PreviewRequest) (Preview, e
if request.DataOrigin == "import" && !snapshot.Template.Imports.Supported {
return Preview{}, errors.New("template does not support imports")
}
if request.DataOrigin == "import" && request.ImportID == "" {
return Preview{}, errors.New("validated import is required")
}
if request.DataOrigin == "new" && request.ImportID != "" {
return Preview{}, errors.New("import is only valid for imported data")
}
resources := request.Resources
if resources.CPUCores == 0 {
resources = snapshot.Template.Requirements.Recommended
@@ -179,6 +193,7 @@ func BuildPreview(snapshot catalog.Snapshot, request PreviewRequest) (Preview, e
Ports: ports, Mounts: mounts, Resources: resources, Settings: settings,
DataOrigin: request.DataOrigin,
Backup: BackupPreview{Strategy: snapshot.Template.Backup.Strategy, SourceMounts: append([]string(nil), snapshot.Template.Backup.SourceMounts...), RetentionCount: request.BackupRetention},
Import: ImportPreview{ID: request.ImportID, DestinationMount: snapshot.Template.Imports.DestinationMount, DestinationRelativePath: snapshot.Template.Imports.DestinationRelativePath},
}
if preview.Backup.RetentionCount < 1 || preview.Backup.RetentionCount > 1000 {
return Preview{}, errors.New("backup retention must be between 1 and 1000")
internal/persistence/sqlite/backup.go
+178
View File
@@ -0,0 +1,178 @@
package sqlite
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"time"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup"
)
func (r *Repository) BeginBackup(ctx context.Context, value backup.Backup, operationID, actorID string) error {
var operation, actor any
if operationID != "" {
operation = operationID
}
if actorID != "" {
actor = actorID
}
_, err := r.db.ExecContext(ctx, `INSERT INTO backups(id, instance_id, operation_id, origin, status, created_by, created_at)
VALUES (?, ?, ?, ?, 'creating', ?, ?)`, value.ID, value.InstanceID, operation, value.Origin, actor, value.CreatedAt)
if err != nil {
return fmt.Errorf("begin backup metadata: %w", err)
}
return nil
}
func (r *Repository) CompleteBackup(ctx context.Context, id, relativePath string, size int64, checksum string, manifest backup.Manifest) error {
encoded, err := json.Marshal(manifest)
if err != nil {
return fmt.Errorf("encode backup manifest: %w", err)
}
now := r.now().UTC().Format(time.RFC3339Nano)
result, err := r.db.ExecContext(ctx, `UPDATE backups SET status='available', relative_path=?, size_bytes=?, sha256=?, manifest_json=?, completed_at=?, error_code=NULL
WHERE id=? AND status='creating'`, relativePath, size, checksum, string(encoded), now, id)
if err != nil {
return fmt.Errorf("complete backup metadata: %w", err)
}
changed, _ := result.RowsAffected()
if changed != 1 {
return backup.ErrInvalidState
}
return nil
}
func (r *Repository) FailBackup(ctx context.Context, id, code string) error {
result, err := r.db.ExecContext(ctx, `UPDATE backups SET status='failed', error_code=?, completed_at=? WHERE id=? AND status='creating'`, code, r.now().UTC().Format(time.RFC3339Nano), id)
if err != nil {
return fmt.Errorf("fail backup metadata: %w", err)
}
changed, _ := result.RowsAffected()
if changed != 1 {
return backup.ErrInvalidState
}
return nil
}
func (r *Repository) ListBackups(ctx context.Context, instanceID string) ([]backup.Backup, error) {
rows, err := r.db.QueryContext(ctx, `SELECT id, instance_id, origin, status, COALESCE(relative_path, ''), COALESCE(size_bytes, 0), COALESCE(sha256, ''), created_at, COALESCE(completed_at, ''), COALESCE(error_code, '')
FROM backups WHERE instance_id=? AND status!='deleted' ORDER BY created_at DESC, id`, instanceID)
if err != nil {
return nil, fmt.Errorf("list backups: %w", err)
}
defer rows.Close()
var result []backup.Backup
for rows.Next() {
var value backup.Backup
if err := rows.Scan(&value.ID, &value.InstanceID, &value.Origin, &value.Status, &value.RelativePath, &value.SizeBytes, &value.SHA256, &value.CreatedAt, &value.CompletedAt, &value.ErrorCode); err != nil {
return nil, fmt.Errorf("scan backup: %w", err)
}
result = append(result, value)
}
return result, rows.Err()
}
func (r *Repository) GetBackup(ctx context.Context, instanceID, id string) (backup.Backup, backup.Manifest, error) {
var value backup.Backup
var manifestJSON string
err := r.db.QueryRowContext(ctx, `SELECT id, instance_id, origin, status, COALESCE(relative_path, ''), COALESCE(size_bytes, 0), COALESCE(sha256, ''), created_at, COALESCE(completed_at, ''), COALESCE(error_code, ''), COALESCE(manifest_json, '')
FROM backups WHERE id=? AND instance_id=? AND status!='deleted'`, id, instanceID).Scan(&value.ID, &value.InstanceID, &value.Origin, &value.Status, &value.RelativePath, &value.SizeBytes, &value.SHA256, &value.CreatedAt, &value.CompletedAt, &value.ErrorCode, &manifestJSON)
if errors.Is(err, sql.ErrNoRows) {
return backup.Backup{}, backup.Manifest{}, backup.ErrNotFound
}
if err != nil {
return backup.Backup{}, backup.Manifest{}, fmt.Errorf("get backup: %w", err)
}
var manifest backup.Manifest
if manifestJSON != "" {
if err := json.Unmarshal([]byte(manifestJSON), &manifest); err != nil {
return backup.Backup{}, backup.Manifest{}, backup.ErrIntegrity
}
}
return value, manifest, nil
}
func (r *Repository) RetentionCandidates(ctx context.Context, instanceID string, keep int) ([]backup.Backup, error) {
rows, err := r.db.QueryContext(ctx, `SELECT id, instance_id, origin, status, relative_path, size_bytes, sha256, created_at, completed_at, ''
FROM backups WHERE instance_id=? AND origin='scheduled' AND status='available'
ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?`, instanceID, keep)
if err != nil {
return nil, fmt.Errorf("list retention candidates: %w", err)
}
defer rows.Close()
var result []backup.Backup
for rows.Next() {
var value backup.Backup
if err := rows.Scan(&value.ID, &value.InstanceID, &value.Origin, &value.Status, &value.RelativePath, &value.SizeBytes, &value.SHA256, &value.CreatedAt, &value.CompletedAt, &value.ErrorCode); err != nil {
return nil, err
}
result = append(result, value)
}
return result, rows.Err()
}
func (r *Repository) MarkBackupDeleted(ctx context.Context, id string) error {
result, err := r.db.ExecContext(ctx, `UPDATE backups SET status='deleted', relative_path=NULL, deleted_at=? WHERE id=? AND status='available'`, r.now().UTC().Format(time.RFC3339Nano), id)
if err != nil {
return fmt.Errorf("mark backup deleted: %w", err)
}
changed, _ := result.RowsAffected()
if changed != 1 {
return backup.ErrInvalidState
}
return nil
}
func (r *Repository) GetBackupPolicy(ctx context.Context, instanceID string) (backup.Policy, error) {
var value backup.Policy
var enabled int
err := r.db.QueryRowContext(ctx, `SELECT instance_id, enabled, COALESCE(cron_expression, ''), timezone, retention_count, COALESCE(next_run_at, '') FROM backup_policies WHERE instance_id=?`, instanceID).Scan(&value.InstanceID, &enabled, &value.CronExpression, &value.Timezone, &value.RetentionCount, &value.NextRunAt)
if errors.Is(err, sql.ErrNoRows) {
instanceValue, instanceErr := r.GetInstance(ctx, instanceID)
if instanceErr != nil {
return backup.Policy{}, instanceErr
}
return backup.Policy{InstanceID: instanceID, Timezone: "UTC", RetentionCount: instanceValue.Preview.Backup.RetentionCount}, nil
}
if err != nil {
return backup.Policy{}, fmt.Errorf("get backup policy: %w", err)
}
value.Enabled = enabled != 0
return value, nil
}
func (r *Repository) SetBackupPolicy(ctx context.Context, value backup.Policy) error {
enabled := 0
if value.Enabled {
enabled = 1
}
now := r.now().UTC().Format(time.RFC3339Nano)
_, err := r.db.ExecContext(ctx, `INSERT INTO backup_policies(instance_id, enabled, cron_expression, timezone, retention_count, next_run_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(instance_id) DO UPDATE SET enabled=excluded.enabled, cron_expression=excluded.cron_expression, timezone=excluded.timezone, retention_count=excluded.retention_count, next_run_at=excluded.next_run_at, updated_at=excluded.updated_at`, value.InstanceID, enabled, nullable(value.CronExpression), value.Timezone, value.RetentionCount, nullable(value.NextRunAt), now, now)
if err != nil {
return fmt.Errorf("set backup policy: %w", err)
}
return nil
}
func (r *Repository) ListDueBackupPolicies(ctx context.Context, now string) ([]backup.Policy, error) {
rows, err := r.db.QueryContext(ctx, `SELECT instance_id, enabled, cron_expression, timezone, retention_count, next_run_at FROM backup_policies WHERE enabled=1 AND next_run_at<=? ORDER BY next_run_at`, now)
if err != nil {
return nil, fmt.Errorf("list due backup policies: %w", err)
}
defer rows.Close()
var result []backup.Policy
for rows.Next() {
var value backup.Policy
var enabled int
if err := rows.Scan(&value.InstanceID, &enabled, &value.CronExpression, &value.Timezone, &value.RetentionCount, &value.NextRunAt); err != nil {
return nil, err
}
value.Enabled = enabled != 0
result = append(result, value)
}
return result, rows.Err()
}
internal/persistence/sqlite/import.go
+94
View File
@@ -0,0 +1,94 @@
package sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport"
)
func (r *Repository) BeginImport(ctx context.Context, value importexport.Import, actorID string) error {
_, err := r.db.ExecContext(ctx, `INSERT INTO imports(id, requested_by, template_id, template_version, status, format, relative_stage_path, created_at, expires_at) VALUES (?, ?, ?, ?, 'staging', ?, ?, ?, ?)`, value.ID, actorID, value.TemplateID, value.TemplateVersion, value.Format, value.RelativeStagePath, r.now().UTC().Format(time.RFC3339Nano), value.ExpiresAt)
if err != nil {
return fmt.Errorf("begin import: %w", err)
}
return nil
}
func (r *Repository) CompleteImport(ctx context.Context, value importexport.Import) error {
result, err := r.db.ExecContext(ctx, `UPDATE imports SET status='validated', data_root=?, detected_type=?, confidence=?, file_count=?, expanded_size_bytes=?, completed_at=? WHERE id=? AND status='staging'`, value.DataRoot, value.DetectedType, value.Confidence, value.FileCount, value.ExpandedSizeBytes, r.now().UTC().Format(time.RFC3339Nano), value.ID)
if err != nil {
return fmt.Errorf("complete import: %w", err)
}
changed, _ := result.RowsAffected()
if changed != 1 {
return importexport.ErrInvalidInput
}
return nil
}
func (r *Repository) GetImport(ctx context.Context, id string) (importexport.Import, error) {
var value importexport.Import
err := r.db.QueryRowContext(ctx, `SELECT id, status, format, template_id, template_version, COALESCE(instance_id, ''), relative_stage_path, COALESCE(data_root, ''), COALESCE(detected_type, ''), COALESCE(confidence, ''), file_count, expanded_size_bytes, expires_at FROM imports WHERE id=?`, id).Scan(&value.ID, &value.Status, &value.Format, &value.TemplateID, &value.TemplateVersion, &value.InstanceID, &value.RelativeStagePath, &value.DataRoot, &value.DetectedType, &value.Confidence, &value.FileCount, &value.ExpandedSizeBytes, &value.ExpiresAt)
if errors.Is(err, sql.ErrNoRows) {
return importexport.Import{}, importexport.ErrInvalidInput
}
if err != nil {
return importexport.Import{}, fmt.Errorf("get import: %w", err)
}
return value, nil
}
func (r *Repository) AttachImport(ctx context.Context, id, instanceID string) error {
result, err := r.db.ExecContext(ctx, `UPDATE imports SET status='attached', instance_id=?, completed_at=? WHERE id=? AND status='validated'`, instanceID, r.now().UTC().Format(time.RFC3339Nano), id)
if err != nil {
return fmt.Errorf("attach import: %w", err)
}
changed, _ := result.RowsAffected()
if changed != 1 {
return importexport.ErrInvalidInput
}
return nil
}
func (r *Repository) FailImport(ctx context.Context, id, code string) error {
_, err := r.db.ExecContext(ctx, `UPDATE imports SET status='failed', error_code=?, completed_at=? WHERE id=? AND status='staging'`, code, r.now().UTC().Format(time.RFC3339Nano), id)
if err != nil {
return fmt.Errorf("fail import: %w", err)
}
return nil
}
func (r *Repository) ExpireImports(ctx context.Context, now string) ([]string, error) {
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return nil, fmt.Errorf("begin import expiry: %w", err)
}
defer func() { _ = tx.Rollback() }()
rows, err := tx.QueryContext(ctx, `SELECT relative_stage_path FROM imports WHERE status IN ('staging', 'validated') AND expires_at<=?`, now)
if err != nil {
return nil, fmt.Errorf("list expired imports: %w", err)
}
var paths []string
for rows.Next() {
var value string
if err := rows.Scan(&value); err != nil {
rows.Close()
return nil, err
}
paths = append(paths, value)
}
if err := rows.Close(); err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `UPDATE imports SET status='expired', completed_at=? WHERE status IN ('staging', 'validated') AND expires_at<=?`, now, now); err != nil {
return nil, fmt.Errorf("expire imports: %w", err)
}
if err := tx.Commit(); err != nil {
return nil, fmt.Errorf("commit import expiry: %w", err)
}
return paths, nil
}
internal/persistence/sqlite/store_test.go
+6 -6
View File
@@ -24,16 +24,16 @@ func TestOpenAppliesMigrationsAndConfiguration(t *testing.T) {
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&count); err != nil {
t.Fatal(err)
}
if count != 4 {
t.Fatalf("got %d migrations, want 4", count)
if count != 5 {
t.Fatalf("got %d migrations, want 5", count)
}
for _, table := range []string{"instance_memberships", "permission_overrides", "installation_requests"} {
for _, table := range []string{"instance_memberships", "permission_overrides", "installation_requests", "backup_policies", "backups", "imports"} {
var found int
if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&found); err != nil {
t.Fatal(err)
}
if found != 1 {
t.Fatalf("authorization table %q is missing", table)
t.Fatalf("required table %q is missing", table)
}
}
var foreignKeys, busyTimeout int
@@ -62,8 +62,8 @@ func TestOpenAppliesMigrationsAndConfiguration(t *testing.T) {
if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&count); err != nil {
t.Fatal(err)
}
if count != 4 {
t.Fatalf("reopened database has %d migrations, want 4", count)
if count != 5 {
t.Fatalf("reopened database has %d migrations, want 5", count)
}
}
internal/web/server.go
+210 -14
View File
@@ -18,7 +18,9 @@ import (
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/authorization"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
)
@@ -59,6 +61,8 @@ type server struct {
repository repository
lifecycle *instance.LifecycleService
permissions *authorization.Service
backups *backup.Service
imports *importexport.Service
}
type repository interface {
@@ -77,26 +81,39 @@ type pageData struct {
// NewHandler constructs the complete HTTP application.
func NewHandler(authService *auth.Service, logger *slog.Logger) (http.Handler, error) {
return newHandler(authService, nil, nil, logger)
return newHandler(authService, nil, nil, nil, logger)
}
// NewHandlerWithRepository enables the authenticated catalog and draft APIs.
func NewHandlerWithRepository(authService *auth.Service, repository repository, logger *slog.Logger) (http.Handler, error) {
return newHandler(authService, repository, nil, logger)
return newHandler(authService, repository, nil, nil, logger)
}
func NewHandlerWithRepositoryAndImports(authService *auth.Service, repository repository, importService *importexport.Service, logger *slog.Logger) (http.Handler, error) {
return newHandlerWithImports(authService, repository, nil, nil, importService, logger)
}
// NewHandlerWithLifecycle enables privileged instance lifecycle operations
// through the restricted agent boundary.
func NewHandlerWithLifecycle(authService *auth.Service, repository repository, agent instance.LifecycleAgent, logger *slog.Logger) (http.Handler, error) {
return newHandler(authService, repository, instance.NewLifecycleService(repository, agent), logger)
return newHandler(authService, repository, instance.NewLifecycleService(repository, agent), nil, logger)
}
func newHandler(authService *auth.Service, repository repository, lifecycle *instance.LifecycleService, logger *slog.Logger) (http.Handler, error) {
// NewHandlerWithLifecycleAndBackup enables lifecycle and backup operations.
func NewHandlerWithLifecycleAndBackup(authService *auth.Service, repository repository, agent instance.LifecycleAgent, backupService *backup.Service, importService *importexport.Service, logger *slog.Logger) (http.Handler, error) {
return newHandlerWithImports(authService, repository, instance.NewLifecycleService(repository, agent), backupService, importService, logger)
}
func newHandler(authService *auth.Service, repository repository, lifecycle *instance.LifecycleService, backupService *backup.Service, logger *slog.Logger) (http.Handler, error) {
return newHandlerWithImports(authService, repository, lifecycle, backupService, nil, logger)
}
func newHandlerWithImports(authService *auth.Service, repository repository, lifecycle *instance.LifecycleService, backupService *backup.Service, importService *importexport.Service, logger *slog.Logger) (http.Handler, error) {
templates, err := template.New("views").Funcs(template.FuncMap{"msg": message}).ParseFS(assets, "templates/*.html")
if err != nil {
return nil, err
}
s := &server{auth: authService, templates: templates, logger: logger, repository: repository, lifecycle: lifecycle}
s := &server{auth: authService, templates: templates, logger: logger, repository: repository, lifecycle: lifecycle, backups: backupService, imports: importService}
if repository != nil {
s.permissions = authorization.New(repository)
}
@@ -124,6 +141,17 @@ func newHandler(authService *auth.Service, repository repository, lifecycle *ins
mux.HandleFunc("POST /api/v1/instances/{id}/restart", s.instanceRestart)
mux.HandleFunc("DELETE /api/v1/instances/{id}", s.instanceDeleteContainer)
}
if backupService != nil {
mux.HandleFunc("GET /api/v1/instances/{id}/backups", s.backupList)
mux.HandleFunc("POST /api/v1/instances/{id}/backups", s.backupCreate)
mux.HandleFunc("GET /api/v1/instances/{id}/backups/{backupID}/export", s.backupExport)
mux.HandleFunc("POST /api/v1/instances/{id}/backups/{backupID}/restore", s.backupRestore)
mux.HandleFunc("GET /api/v1/instances/{id}/backup-policy", s.backupPolicyGet)
mux.HandleFunc("PUT /api/v1/instances/{id}/backup-policy", s.backupPolicySet)
}
if importService != nil {
mux.HandleFunc("POST /api/v1/imports", s.importCreate)
}
}
mux.HandleFunc("GET /static/app.v1.css", s.stylesheet)
mux.HandleFunc("GET /setup", s.setupForm)
@@ -145,6 +173,7 @@ type previewAPIRequest struct {
Resources catalog.Resources `json:"resources"`
DataOrigin string `json:"data_origin"`
BackupRetention int `json:"backup_retention"`
ImportID string `json:"import_id"`
}
func (s *server) catalogList(w http.ResponseWriter, r *http.Request) {
@@ -208,7 +237,41 @@ func (s *server) instanceStats(w http.ResponseWriter, r *http.Request) {
}
func (s *server) instanceInstall(w http.ResponseWriter, r *http.Request) {
s.lifecycleAdminAction(w, r, s.lifecycle.Install)
actor, ok := s.requireAPIUser(w, r, true)
if !ok {
return
}
if err := s.permissions.RequireRecentAdmin(actor); err != nil {
s.authorizationProblem(w, err)
return
}
current, err := s.repository.GetInstance(r.Context(), r.PathValue("id"))
if err != nil {
s.lifecycleProblem(w, err)
return
}
if current.Preview.DataOrigin == "import" {
if s.imports == nil {
s.apiProblem(w, http.StatusConflict, "import_unavailable", "The validated import is unavailable.")
return
}
mountPath := ""
for _, mount := range current.Preview.Mounts {
if mount.ID == current.Preview.Import.DestinationMount {
mountPath = mount.HostPath
break
}
}
if mountPath == "" {
s.apiProblem(w, http.StatusUnprocessableEntity, "invalid_import", "The import destination is invalid.")
return
}
if err := s.imports.ApplyToInstance(r.Context(), current.Preview.Import.ID, current.ID, current.Preview.Template.ID, current.Preview.Template.Version, mountPath, current.Preview.Import.DestinationRelativePath); err != nil {
s.apiProblem(w, http.StatusUnprocessableEntity, "invalid_import", "The validated import could not be applied.")
return
}
}
s.runLifecycleAction(w, r, s.lifecycle.Install)
}
func (s *server) instanceStart(w http.ResponseWriter, r *http.Request) {
@@ -230,13 +293,6 @@ func (s *server) lifecycleAction(w http.ResponseWriter, r *http.Request, permiss
s.runLifecycleAction(w, r, action)
}
func (s *server) lifecycleAdminAction(w http.ResponseWriter, r *http.Request, action func(context.Context, string) (instance.OperationResult, error)) {
if _, ok := s.requireAPIUser(w, r, true); !ok {
return
}
s.runLifecycleAction(w, r, action)
}
func (s *server) runLifecycleAction(w http.ResponseWriter, r *http.Request, action func(context.Context, string) (instance.OperationResult, error)) {
if r.Body != nil {
r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes)
@@ -310,10 +366,16 @@ func (s *server) buildAPIPreview(w http.ResponseWriter, r *http.Request) (previe
s.apiProblem(w, http.StatusNotFound, "template_not_found", "The template version was not found.")
return request, instance.Preview{}, false
}
if request.DataOrigin == "import" {
if s.imports == nil || s.imports.ValidateSelection(r.Context(), request.ImportID, request.TemplateID, request.TemplateVersion) != nil {
s.apiProblem(w, http.StatusUnprocessableEntity, "invalid_import", "A validated compatible import is required.")
return request, instance.Preview{}, false
}
}
preview, err := instance.BuildPreview(snapshot, instance.PreviewRequest{
DisplayName: request.DisplayName, Slug: request.Slug, HostPorts: request.HostPorts,
MountPaths: request.MountPaths, Resources: request.Resources, DataOrigin: request.DataOrigin,
BackupRetention: request.BackupRetention,
BackupRetention: request.BackupRetention, ImportID: request.ImportID,
})
if err != nil {
s.apiProblem(w, http.StatusUnprocessableEntity, "invalid_preview", "The deployment preview is invalid.")
@@ -322,6 +384,140 @@ func (s *server) buildAPIPreview(w http.ResponseWriter, r *http.Request) (previe
return request, preview, true
}
func (s *server) backupList(w http.ResponseWriter, r *http.Request) {
if _, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupList); !ok {
return
}
values, err := s.backups.List(r.Context(), r.PathValue("id"))
if err != nil {
s.backupProblem(w, err)
return
}
s.apiJSON(w, http.StatusOK, map[string]any{"backups": values})
}
func (s *server) backupCreate(w http.ResponseWriter, r *http.Request) {
actor, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupCreate)
if !ok || !s.requireEmptyBody(w, r) {
return
}
value, err := s.backups.Create(r.Context(), actor.ID, r.PathValue("id"), "manual")
if err != nil {
s.backupProblem(w, err)
return
}
s.apiJSON(w, http.StatusCreated, value)
}
func (s *server) backupExport(w http.ResponseWriter, r *http.Request) {
if _, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupExport); !ok {
return
}
file, value, err := s.backups.Export(r.Context(), r.PathValue("id"), r.PathValue("backupID"))
if err != nil {
s.backupProblem(w, err)
return
}
defer file.Close()
w.Header().Set("Content-Type", "application/zstd")
w.Header().Set("Content-Disposition", `attachment; filename="`+value.ID+`.tar.zst"`)
w.Header().Set("X-Content-SHA256", value.SHA256)
http.ServeContent(w, r, value.ID+".tar.zst", time.Time{}, file)
}
func (s *server) backupRestore(w http.ResponseWriter, r *http.Request) {
actor, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupRestore)
if !ok || !s.requireEmptyBody(w, r) {
return
}
if err := s.backups.Restore(r.Context(), actor.ID, r.PathValue("id"), r.PathValue("backupID")); err != nil {
s.backupProblem(w, err)
return
}
s.apiJSON(w, http.StatusOK, map[string]string{"state": "restored"})
}
func (s *server) backupPolicyGet(w http.ResponseWriter, r *http.Request) {
if _, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupList); !ok {
return
}
value, err := s.backups.GetPolicy(r.Context(), r.PathValue("id"))
if err != nil {
s.backupProblem(w, err)
return
}
s.apiJSON(w, http.StatusOK, value)
}
func (s *server) backupPolicySet(w http.ResponseWriter, r *http.Request) {
if _, ok := s.requireInstancePermission(w, r, authorization.PermissionInstanceConfigure); !ok {
return
}
var request struct {
Enabled bool `json:"enabled"`
CronExpression string `json:"cron_expression"`
Timezone string `json:"timezone"`
RetentionCount int `json:"retention_count"`
}
if !s.decodeAPIJSON(w, r, &request) {
return
}
value, err := s.backups.SetPolicy(r.Context(), backup.Policy{InstanceID: r.PathValue("id"), Enabled: request.Enabled, CronExpression: request.CronExpression, Timezone: request.Timezone, RetentionCount: request.RetentionCount})
if err != nil {
s.backupProblem(w, err)
return
}
s.apiJSON(w, http.StatusOK, value)
}
func (s *server) backupProblem(w http.ResponseWriter, err error) {
status, code := http.StatusInternalServerError, "backup_failed"
switch {
case errors.Is(err, backup.ErrNotFound), errors.Is(err, instance.ErrInstanceNotFound):
status, code = http.StatusNotFound, "backup_not_found"
case errors.Is(err, backup.ErrInvalidInput), errors.Is(err, backup.ErrIncompatible):
status, code = http.StatusUnprocessableEntity, "invalid_backup"
case errors.Is(err, backup.ErrInvalidState), errors.Is(err, instance.ErrOperationConflict):
status, code = http.StatusConflict, "backup_conflict"
case errors.Is(err, backup.ErrIntegrity):
status, code = http.StatusUnprocessableEntity, "backup_integrity_failed"
}
s.apiProblem(w, status, code, "The backup operation could not be completed.")
}
func (s *server) importCreate(w http.ResponseWriter, r *http.Request) {
actor, ok := s.requireAPIUser(w, r, true)
if !ok {
return
}
if err := s.permissions.RequireRecentAdmin(actor); err != nil {
s.authorizationProblem(w, err)
return
}
templateID, version, format := r.URL.Query().Get("template_id"), r.URL.Query().Get("template_version"), r.URL.Query().Get("format")
snapshot, err := s.repository.Get(r.Context(), templateID, version)
if err != nil {
s.apiProblem(w, http.StatusNotFound, "template_not_found", "The template version was not found.")
return
}
if !snapshot.Template.Imports.Supported {
s.apiProblem(w, http.StatusUnprocessableEntity, "import_unsupported", "The template does not support imports.")
return
}
maximum := int64(snapshot.Template.Imports.MaxExtractedSizeGB) << 30
r.Body = http.MaxBytesReader(w, r.Body, maximum+1)
value, err := s.imports.Stage(r.Context(), actor.ID, format, r.Body, importexport.Policy{TemplateID: templateID, TemplateVersion: version, AcceptedFormats: snapshot.Template.Imports.AcceptedFormats, MaxExpandedBytes: maximum, RequiredPaths: snapshot.Template.Imports.RequiredPaths})
if err != nil {
status, code := http.StatusUnprocessableEntity, "invalid_import"
if errors.Is(err, importexport.ErrLimitExceeded) {
status, code = http.StatusRequestEntityTooLarge, "import_limit_exceeded"
}
s.apiProblem(w, status, code, "The import could not be validated.")
return
}
s.apiJSON(w, http.StatusCreated, value)
}
func (s *server) requireInstancePermission(w http.ResponseWriter, r *http.Request, permission string) (auth.User, bool) {
user, ok := s.requireAPIUser(w, r, false)
if !ok {
internal/web/server_test.go
+147
View File
@@ -1,6 +1,7 @@
package web
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
@@ -9,6 +10,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
@@ -16,7 +18,10 @@ import (
catalogdata "git.zaynet.fr/DoGaMa/DoGaMa-serv/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance"
"git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite"
)
@@ -120,6 +125,148 @@ func TestCatalogPreviewAndDraftAPIAuthorization(t *testing.T) {
assertStatus(t, unsafeResponse, http.StatusUnprocessableEntity)
}
func TestBackupAPIEnforcesPermissionsAndRestores(t *testing.T) {
ctx := context.Background()
root := t.TempDir()
serversRoot := filepath.Join(root, "servers")
backupsRoot := filepath.Join(root, "backups")
mount := filepath.Join(serversRoot, "instance", "saved")
if err := os.MkdirAll(mount, 0o750); err != nil {
t.Fatal(err)
}
world := filepath.Join(mount, "Level.sav")
if err := os.WriteFile(world, []byte("world-v1"), 0o640); err != nil {
t.Fatal(err)
}
db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db"))
if err != nil {
t.Fatal(err)
}
defer db.Close()
repository := sqlite.NewRepository(db)
snapshots, err := catalog.LoadFS(catalogdata.Files, ".")
if err != nil {
t.Fatal(err)
}
if err := repository.Sync(ctx, snapshots); err != nil {
t.Fatal(err)
}
authService := auth.New(db)
if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil {
t.Fatal(err)
}
adminSession, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234")
if err != nil {
t.Fatal(err)
}
admin, err := authService.Authenticate(ctx, adminSession.Token)
if err != nil {
t.Fatal(err)
}
player, err := authService.CreateUser(ctx, "player", "another correct battery staple", "user")
if err != nil {
t.Fatal(err)
}
playerSession, err := authService.Login(ctx, "player", "another correct battery staple", "192.0.2.2:1234")
if err != nil {
t.Fatal(err)
}
preview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{DisplayName: "Backup API", Slug: "backup-api", HostPorts: map[string]int{"game": 38211}, MountPaths: map[string]string{"saved": mount}, DataOrigin: "new", BackupRetention: 2})
if err != nil {
t.Fatal(err)
}
const instanceID = "backup-api-instance"
if err := repository.CreateDraft(ctx, instance.Draft{ID: instanceID, Preview: preview}); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`UPDATE instances SET lifecycle_state='online', observed_state='ready', container_id='container', desired_running=1 WHERE id=?`, instanceID); err != nil {
t.Fatal(err)
}
backupService, err := backup.New(repository, webLifecycleAgent{}, serversRoot, backupsRoot)
if err != nil {
t.Fatal(err)
}
importService, err := importexport.New(repository, filepath.Join(root, "imports"), serversRoot)
if err != nil {
t.Fatal(err)
}
handler, err := NewHandlerWithLifecycleAndBackup(authService, repository, webLifecycleAgent{}, backupService, importService, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatal(err)
}
adminCookie := &http.Cookie{Name: sessionCookie, Value: adminSession.Token}
playerCookie := &http.Cookie{Name: sessionCookie, Value: playerSession.Token}
denied := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups", []*http.Cookie{playerCookie})
assertStatus(t, denied, http.StatusForbidden)
membership := jsonMethodRequest(t, handler, http.MethodPut, "/api/v1/instances/"+instanceID+"/memberships/"+player.ID, []byte(`{"role":"manager"}`), adminCookie, adminSession.CSRFToken)
assertStatus(t, membership, http.StatusNoContent)
created := jsonRequest(t, handler, "/api/v1/instances/"+instanceID+"/backups", nil, playerCookie, playerSession.CSRFToken)
assertStatus(t, created, http.StatusCreated)
var value backup.Backup
if err := json.Unmarshal(created.Body.Bytes(), &value); err != nil {
t.Fatal(err)
}
listed := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups", []*http.Cookie{playerCookie})
assertStatus(t, listed, http.StatusOK)
exportDenied := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups/"+value.ID+"/export", []*http.Cookie{playerCookie})
assertStatus(t, exportDenied, http.StatusForbidden)
exported := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups/"+value.ID+"/export", []*http.Cookie{adminCookie})
assertStatus(t, exported, http.StatusOK)
if exported.Header().Get("X-Content-SHA256") == "" {
t.Fatal("export checksum header missing")
}
if err := os.WriteFile(world, []byte("world-v2"), 0o640); err != nil {
t.Fatal(err)
}
restored := jsonRequest(t, handler, "/api/v1/instances/"+instanceID+"/backups/"+value.ID+"/restore", nil, adminCookie, adminSession.CSRFToken)
assertStatus(t, restored, http.StatusOK)
body, err := os.ReadFile(world)
if err != nil || string(body) != "world-v1" {
t.Fatalf("restored world=%q error=%v", body, err)
}
policy := jsonMethodRequest(t, handler, http.MethodPut, "/api/v1/instances/"+instanceID+"/backup-policy", []byte(`{"enabled":true,"cron_expression":"0 3 * * *","timezone":"Europe/Paris","retention_count":5}`), adminCookie, adminSession.CSRFToken)
assertStatus(t, policy, http.StatusOK)
imported, err := importService.Stage(ctx, admin.ID, "zip", bytes.NewReader(palworldImportZIP(t)), importexport.Policy{TemplateID: snapshots[0].Template.ID, TemplateVersion: snapshots[0].Template.Version, AcceptedFormats: snapshots[0].Template.Imports.AcceptedFormats, MaxExpandedBytes: int64(snapshots[0].Template.Imports.MaxExtractedSizeGB) << 30, RequiredPaths: snapshots[0].Template.Imports.RequiredPaths})
if err != nil {
t.Fatal(err)
}
importMount := filepath.Join(serversRoot, "imported", "saved")
importPreview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{DisplayName: "Imported API", Slug: "imported-api", HostPorts: map[string]int{"game": 38212}, MountPaths: map[string]string{"saved": importMount}, DataOrigin: "import", ImportID: imported.ID, BackupRetention: 2})
if err != nil {
t.Fatal(err)
}
const importedInstanceID = "imported-api-instance"
if err := repository.CreateDraft(ctx, instance.Draft{ID: importedInstanceID, Preview: importPreview}); err != nil {
t.Fatal(err)
}
installed := jsonRequest(t, handler, "/api/v1/instances/"+importedInstanceID+"/install", nil, adminCookie, adminSession.CSRFToken)
assertStatus(t, installed, http.StatusOK)
importedWorld := filepath.Join(importMount, "SaveGames", "0", "Level.sav")
if body, err := os.ReadFile(importedWorld); err != nil || string(body) != "imported-world" {
t.Fatalf("imported world=%q error=%v", body, err)
}
}
func palworldImportZIP(t *testing.T) []byte {
t.Helper()
var buffer bytes.Buffer
writer := zip.NewWriter(&buffer)
for name, body := range map[string]string{"Save/Level.sav": "imported-world", "Save/Players/player.sav": "player"} {
entry, err := writer.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := io.WriteString(entry, body); err != nil {
t.Fatal(err)
}
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
return buffer.Bytes()
}
func TestInstanceAuthorizationAndInstallationRequestWorkflow(t *testing.T) {
ctx := context.Background()
db, err := sqlite.Open(ctx, filepath.Join(t.TempDir(), "dogama.db"))
migrations/0005_backups_imports.sql
+76
View File
@@ -0,0 +1,76 @@
ALTER TABLE instance_operations RENAME TO instance_operations_v4;
CREATE TABLE instance_operations (
id TEXT PRIMARY KEY,
instance_id TEXT NOT NULL REFERENCES instances(id) ON DELETE RESTRICT,
kind TEXT NOT NULL CHECK (kind IN ('install', 'start', 'stop', 'restart', 'delete_container', 'reconcile', 'backup', 'restore', 'import')),
state TEXT NOT NULL CHECK (state IN ('running', 'succeeded', 'failed', 'intervention_required')),
phase TEXT NOT NULL,
error_code TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
completed_at TEXT
);
INSERT INTO instance_operations(id, instance_id, kind, state, phase, error_code, created_at, updated_at, completed_at)
SELECT id, instance_id, kind, state, phase, error_code, created_at, updated_at, completed_at
FROM instance_operations_v4;
DROP TABLE instance_operations_v4;
CREATE UNIQUE INDEX instance_operation_active_idx ON instance_operations(instance_id) WHERE state = 'running';
CREATE INDEX instance_operation_history_idx ON instance_operations(instance_id, created_at DESC);
CREATE TABLE backup_policies (
instance_id TEXT PRIMARY KEY REFERENCES instances(id) ON DELETE CASCADE,
enabled INTEGER NOT NULL DEFAULT 0 CHECK (enabled IN (0, 1)),
cron_expression TEXT,
timezone TEXT NOT NULL DEFAULT 'UTC',
retention_count INTEGER NOT NULL DEFAULT 7 CHECK (retention_count BETWEEN 1 AND 1000),
next_run_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE backups (
id TEXT PRIMARY KEY,
instance_id TEXT NOT NULL REFERENCES instances(id) ON DELETE RESTRICT,
operation_id TEXT REFERENCES instance_operations(id) ON DELETE SET NULL,
origin TEXT NOT NULL CHECK (origin IN ('manual', 'scheduled', 'pre_update', 'pre_restore', 'idle_shutdown', 'imported', 'system')),
status TEXT NOT NULL CHECK (status IN ('creating', 'available', 'failed', 'deleted')),
relative_path TEXT,
size_bytes INTEGER CHECK (size_bytes IS NULL OR size_bytes >= 0),
sha256 TEXT CHECK (sha256 IS NULL OR length(sha256) = 64),
manifest_json TEXT,
error_code TEXT,
created_by TEXT REFERENCES users(id) ON DELETE SET NULL,
created_at TEXT NOT NULL,
completed_at TEXT,
deleted_at TEXT
);
CREATE INDEX backups_instance_created_idx ON backups(instance_id, created_at DESC);
CREATE INDEX backups_retention_idx ON backups(instance_id, origin, status, created_at);
CREATE TABLE imports (
id TEXT PRIMARY KEY,
requested_by TEXT NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
instance_id TEXT REFERENCES instances(id) ON DELETE CASCADE,
template_id TEXT NOT NULL,
template_version TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('staging', 'validated', 'attached', 'failed', 'expired')),
format TEXT NOT NULL,
relative_stage_path TEXT NOT NULL,
data_root TEXT,
detected_type TEXT,
confidence TEXT CHECK (confidence IS NULL OR confidence IN ('confirmed', 'probable', 'recognized_unknown_version', 'unrecognized')),
file_count INTEGER NOT NULL DEFAULT 0,
expanded_size_bytes INTEGER NOT NULL DEFAULT 0,
error_code TEXT,
created_at TEXT NOT NULL,
expires_at TEXT NOT NULL,
completed_at TEXT,
FOREIGN KEY (template_id, template_version) REFERENCES template_versions(template_id, version) ON DELETE RESTRICT
);
CREATE INDEX imports_expiry_idx ON imports(status, expires_at);