diff --git a/README.md b/README.md index f2f1de5..f3e9a6c 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Only the main application's HTTP port is published. The agent and game-managemen ## Status -The first five roadmap foundations are implemented: the main application and authentication, the restricted agent boundary, the validated embedded catalog, the registered instance lifecycle and backend-enforced per-instance authorization. Administrators can manage local users, assign user or manager memberships, apply explicit permission overrides and review installation requests without approval causing deployment. Assigned users receive the documented safe baseline; managers receive the operational baseline, while restore remains an explicit grant. SQLite records identities, access policy, requests, serialized operation phases and desired/observed instance state. Backups, updates and the WebAssembly runtime remain later roadmap work. +The first six roadmap foundations are implemented: application/authentication, the restricted agent boundary, the validated catalog, registered instance lifecycle, per-instance authorization, and recoverable game-data backups. DoGaMa creates atomic `tar.zst` archives with manifests and SHA-256 metadata, selectively retains scheduled backups, supports five-field cron policies with IANA timezones, stages hostile imports under strict limits, and restores through validated staging with a default `pre_restore` safety backup. Backup, export, restore and policy APIs enforce backend permissions and recent authentication for sensitive actions. Updates and the WebAssembly runtime remain later roadmap work. ## Validate the specification diff --git a/cmd/dogama/main.go b/cmd/dogama/main.go index 4e24ee2..c42f363 100644 --- a/cmd/dogama/main.go +++ b/cmd/dogama/main.go @@ -15,7 +15,9 @@ import ( catalogdata "git.zaynet.fr/DoGaMa/DoGaMa-serv/catalog" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentclient" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/web" @@ -51,10 +53,15 @@ func run(logger *slog.Logger) error { logger.Info("local catalog synchronized", "event", "catalog.synchronized", "template_count", len(snapshots)) var handler http.Handler var lifecycle *instance.LifecycleService + var backupService *backup.Service + importService, err := importexport.New(repository, environment("DOGAMA_IMPORTS_ROOT", "/var/lib/dogama/imports/staging")) + if err != nil { + return err + } agentURL, tokenFile := os.Getenv("DOGAMA_AGENT_URL"), os.Getenv("DOGAMA_AGENT_TOKEN_FILE") if agentURL == "" && tokenFile == "" { logger.Warn("instance lifecycle disabled", "event", "lifecycle.disabled") - handler, err = web.NewHandlerWithRepository(auth.New(db), repository, logger) + handler, err = web.NewHandlerWithRepositoryAndImports(auth.New(db), repository, importService, logger) } else { if agentURL == "" || tokenFile == "" { return errors.New("DOGAMA_AGENT_URL and DOGAMA_AGENT_TOKEN_FILE must be configured together") @@ -69,6 +76,10 @@ func run(logger *slog.Logger) error { return clientErr } lifecycle = instance.NewLifecycleService(repository, agent) + backupService, err = backup.New(repository, agent, environment("DOGAMA_SERVERS_ROOT", "/srv/game-servers"), environment("DOGAMA_BACKUPS_ROOT", "/srv/game-backups")) + if err != nil { + return err + } reconcileCtx, cancel := context.WithTimeout(ctx, 15*time.Second) if recoverErr := lifecycle.RecoverInterruptedOperations(reconcileCtx); recoverErr != nil { cancel() @@ -78,7 +89,7 @@ func run(logger *slog.Logger) error { logger.Warn("instance reconciliation incomplete", "event", "lifecycle.reconcile.failed") } cancel() - handler, err = web.NewHandlerWithLifecycle(auth.New(db), repository, agent, logger) + handler, err = web.NewHandlerWithLifecycleAndBackup(auth.New(db), repository, agent, backupService, importService, logger) } if err != nil { return err @@ -86,6 +97,10 @@ func run(logger *slog.Logger) error { if lifecycle != nil { go reconcileInstances(ctx, lifecycle, logger) } + if backupService != nil { + go runBackupScheduler(ctx, backupService, logger) + } + go runImportCleanup(ctx, importService, logger) server := &http.Server{ Addr: listenAddress, Handler: handler, @@ -112,6 +127,38 @@ func run(logger *slog.Logger) error { } } +func runImportCleanup(ctx context.Context, service *importexport.Service, logger *slog.Logger) { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := service.CleanupExpired(ctx); err != nil { + logger.Warn("expired import cleanup incomplete", "event", "import.cleanup.failed") + } + } + } +} + +func runBackupScheduler(ctx context.Context, service *backup.Service, logger *slog.Logger) { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + runCtx, cancel := context.WithTimeout(ctx, 30*time.Minute) + if err := service.RunDue(runCtx); err != nil { + logger.Warn("scheduled backup run incomplete", "event", "backup.scheduler.failed") + } + cancel() + } + } +} + func reconcileInstances(ctx context.Context, lifecycle *instance.LifecycleService, logger *slog.Logger) { ticker := time.NewTicker(time.Minute) defer ticker.Stop() diff --git a/compose.yaml b/compose.yaml index 5fce9cf..4f1864c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,6 +9,9 @@ services: DOGAMA_AGENT_URL: http://agent:8081 DOGAMA_AGENT_TOKEN_FILE: /run/secrets/agent_token DOGAMA_MASTER_KEY_FILE: /run/secrets/master_key + DOGAMA_SERVERS_ROOT: /srv/game-servers + DOGAMA_BACKUPS_ROOT: /srv/game-backups + DOGAMA_IMPORTS_ROOT: /var/lib/dogama/imports/staging secrets: - agent_token - master_key diff --git a/docs/contributing/development.md b/docs/contributing/development.md index 8c52ad1..9e82647 100644 --- a/docs/contributing/development.md +++ b/docs/contributing/development.md @@ -76,6 +76,24 @@ request review additionally require an administrator session authenticated in the previous ten minutes. Approving a request only records the decision and the requested values; it never creates a draft or contacts the restricted agent. +Game-data backups are written below `DOGAMA_BACKUPS_ROOT` (default +`/srv/game-backups`) and may only read instance mounts below +`DOGAMA_SERVERS_ROOT` (default `/srv/game-servers`). Untrusted uploads are +isolated below `DOGAMA_IMPORTS_ROOT` (default +`/var/lib/dogama/imports/staging`). These are bootstrap path boundaries, not +ordinary product settings. The same canonical server and backup roots are +mounted into the main application and restricted agent by `compose.yaml`. + +The current backup engine conservatively stops a running instance before +archiving. Once the WebAssembly runtime is active, an enabled `online_save` +module can provide the documented flush-before-archive optimization without +moving traversal or archive ownership out of the main application. Archives +are finalized before SQLite marks them available; a metadata failure removes +the orphaned file. Scheduled retention considers only successful `scheduled` +backups. Restore verifies size, checksum, manifest and pinned template version, +creates a `pre_restore` backup, extracts into sibling staging and keeps the +instance stopped with `intervention_required` if readiness cannot be restored. + At main-application startup, every embedded `catalog/*/template.yaml` is validated against `specs/template.schema.json`, checked for cross-reference and asset integrity, canonicalized deterministically and synchronized into SQLite. diff --git a/docs/operations/backups-import-export.md b/docs/operations/backups-import-export.md index 9d73ffe..c8e4151 100644 --- a/docs/operations/backups-import-export.md +++ b/docs/operations/backups-import-export.md @@ -91,3 +91,22 @@ The preview reports detected game/type, file count, expanded size, world/player The reference template recognizes dedicated-server world layouts with `Level.sav` and `Players/`. A local hosted-world import may require player identity conversion. V1 must preserve the upload, warn about this possibility and never perform undocumented silent conversion. A future game-specific data converter remains separate from the WebAssembly API adapter because modules have no filesystem access. +## Implemented foundation + +The V1 backup foundation persists policies, archives and import-validation +records in SQLite. Manual backups, listing, integrity-checked export, restore +and five-field cron policy APIs are protected by the stable instance permission +identifiers. Restore and backup-policy mutation use the existing recent-session +safeguard. + +Until an integration module is activated, the generic engine uses the safe +`stop_then_archive` behavior even when a template advertises `online_save`. +The later WebAssembly milestone supplies the capability call; modules will +still never traverse or archive files. + +ZIP, tar, tar.gz and tar.zst imports are copied to a unique staging directory +before validation. Extraction uses create-new files and rejects absolute or +Windows paths, traversal, links, special files, excessive nesting, excessive +file counts and expanded-size overflow. A validated import remains staged for +the creation or existing-instance workflow; validation itself never creates a +container or writes into live player data. diff --git a/go.mod b/go.mod index 6197250..d56e847 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,11 @@ go 1.25.0 require ( github.com/dlclark/regexp2 v1.12.0 + github.com/klauspost/compress v1.18.0 + github.com/robfig/cron/v3 v3.0.1 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 golang.org/x/crypto v0.53.0 + golang.org/x/sys v0.47.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.56.0 ) @@ -16,7 +19,6 @@ require ( github.com/mattn/go-isatty v0.0.24 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.38.0 // indirect modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index a417fda..55da6a8 100644 --- a/go.sum +++ b/go.sum @@ -8,12 +8,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= diff --git a/internal/backup/service.go b/internal/backup/service.go new file mode 100644 index 0000000..4fe646c --- /dev/null +++ b/internal/backup/service.go @@ -0,0 +1,745 @@ +// Package backup owns game-data archives, retention and recoverable restores. +package backup + +import ( + "archive/tar" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance" + "github.com/klauspost/compress/zstd" + "github.com/robfig/cron/v3" + "golang.org/x/sys/unix" +) + +var ( + ErrNotFound = errors.New("backup not found") + ErrInvalidInput = errors.New("invalid backup input") + ErrInvalidState = errors.New("invalid backup state") + ErrUnsafePath = errors.New("unsafe backup path") + ErrIntegrity = errors.New("backup integrity check failed") + ErrIncompatible = errors.New("backup is incompatible") +) + +const manifestSchemaVersion = 1 + +type Backup struct { + ID string `json:"id"` + InstanceID string `json:"instance_id"` + Origin string `json:"origin"` + Status string `json:"status"` + SizeBytes int64 `json:"size_bytes,omitempty"` + SHA256 string `json:"sha256,omitempty"` + CreatedAt string `json:"created_at"` + CompletedAt string `json:"completed_at,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + RelativePath string `json:"-"` +} + +type Policy struct { + InstanceID string `json:"instance_id"` + Enabled bool `json:"enabled"` + CronExpression string `json:"cron_expression,omitempty"` + Timezone string `json:"timezone"` + RetentionCount int `json:"retention_count"` + NextRunAt string `json:"next_run_at,omitempty"` +} + +type Manifest struct { + SchemaVersion int `json:"schema_version"` + BackupID string `json:"backup_id"` + InstanceID string `json:"instance_id"` + TemplateID string `json:"template_id"` + TemplateVersion string `json:"template_version"` + Origin string `json:"origin"` + CreatedAt string `json:"created_at"` + MountIDs []string `json:"mount_ids"` +} + +type Repository interface { + instance.LifecycleRepository + BeginBackup(context.Context, Backup, string, string) error + CompleteBackup(context.Context, string, string, int64, string, Manifest) error + FailBackup(context.Context, string, string) error + ListBackups(context.Context, string) ([]Backup, error) + GetBackup(context.Context, string, string) (Backup, Manifest, error) + RetentionCandidates(context.Context, string, int) ([]Backup, error) + MarkBackupDeleted(context.Context, string) error + GetBackupPolicy(context.Context, string) (Policy, error) + SetBackupPolicy(context.Context, Policy) error + ListDueBackupPolicies(context.Context, string) ([]Policy, error) +} + +type Agent interface { + StartInstance(context.Context, string) (agentwire.InstanceState, error) + StopInstance(context.Context, string, int) (agentwire.InstanceState, error) +} + +type Service struct { + repository Repository + agent Agent + serversRoot string + backupsRoot string + now func() time.Time +} + +func New(repository Repository, agent Agent, serversRoot, backupsRoot string) (*Service, error) { + serversRoot, err := canonicalRoot(serversRoot) + if err != nil { + return nil, fmt.Errorf("server root: %w", err) + } + backupsRoot, err = canonicalRoot(backupsRoot) + if err != nil { + return nil, fmt.Errorf("backup root: %w", err) + } + return &Service{repository: repository, agent: agent, serversRoot: serversRoot, backupsRoot: backupsRoot, now: time.Now}, nil +} + +func (s *Service) Create(ctx context.Context, actorID, instanceID, origin string) (Backup, error) { + if !validOrigin(origin) || instanceID == "" { + return Backup{}, ErrInvalidInput + } + current, err := s.repository.GetInstance(ctx, instanceID) + if err != nil { + return Backup{}, err + } + if current.ContainerID == "" { + return Backup{}, ErrInvalidState + } + operationID, backupID := token(), token() + if operationID == "" || backupID == "" { + return Backup{}, errors.New("generate backup identifiers") + } + current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "backup", "backup") + if err != nil { + return Backup{}, err + } + backup := Backup{ID: backupID, InstanceID: instanceID, Origin: origin, Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)} + if err := s.repository.BeginBackup(ctx, backup, operationID, actorID); err != nil { + _ = s.repository.FailOperation(ctx, operationID, "error", "backup_metadata_failed") + return Backup{}, err + } + wasRunning := current.DesiredRunning + if wasRunning { + if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil { + return Backup{}, s.fail(ctx, operationID, backupID, "backup_stop_failed", err) + } + } + completed, manifest, err := s.writeArchive(ctx, current, backup) + if err != nil { + if wasRunning { + if _, restartErr := s.agent.StartInstance(ctx, instanceID); restartErr != nil { + _ = s.repository.FailBackup(ctx, backupID, "backup_restart_failed") + _ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed") + return Backup{}, fmt.Errorf("backup failed and restart failed: %v: %w", err, restartErr) + } + } + return Backup{}, s.fail(ctx, operationID, backupID, "backup_archive_failed", err) + } + if err := s.repository.CompleteBackup(ctx, backupID, completed.RelativePath, completed.SizeBytes, completed.SHA256, manifest); err != nil { + if full, pathErr := s.backupPath(completed.RelativePath); pathErr == nil { + _ = os.Remove(full) + } + if wasRunning { + _, _ = s.agent.StartInstance(ctx, instanceID) + } + return Backup{}, s.fail(ctx, operationID, backupID, "backup_persist_failed", err) + } + state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped" + if wasRunning { + state, err = s.agent.StartInstance(ctx, instanceID) + if err != nil { + _ = s.repository.FailOperation(ctx, operationID, "intervention_required", "backup_restart_failed") + return Backup{}, fmt.Errorf("restart after backup: %w", err) + } + lifecycle, observed = lifecycleFromAgent(state) + } + if err := s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, ""); err != nil { + return Backup{}, err + } + completed.Status, completed.CompletedAt = "available", s.now().UTC().Format(time.RFC3339Nano) + if err := s.applyRetention(ctx, instanceID, current.Preview.Backup.RetentionCount); err != nil { + return Backup{}, fmt.Errorf("apply retention: %w", err) + } + return completed, nil +} + +func (s *Service) List(ctx context.Context, instanceID string) ([]Backup, error) { + return s.repository.ListBackups(ctx, instanceID) +} + +func (s *Service) GetPolicy(ctx context.Context, instanceID string) (Policy, error) { + return s.repository.GetBackupPolicy(ctx, instanceID) +} + +func (s *Service) SetPolicy(ctx context.Context, value Policy) (Policy, error) { + if value.InstanceID == "" || value.RetentionCount < 1 || value.RetentionCount > 1000 { + return Policy{}, ErrInvalidInput + } + location, err := time.LoadLocation(value.Timezone) + if err != nil { + return Policy{}, ErrInvalidInput + } + if !value.Enabled { + value.CronExpression, value.NextRunAt = "", "" + } else { + schedule, err := cronParser().Parse(value.CronExpression) + if err != nil { + return Policy{}, ErrInvalidInput + } + value.NextRunAt = schedule.Next(s.now().In(location)).UTC().Format(time.RFC3339Nano) + } + if err := s.repository.SetBackupPolicy(ctx, value); err != nil { + return Policy{}, err + } + return value, nil +} + +func (s *Service) RunDue(ctx context.Context) error { + now := s.now().UTC() + policies, err := s.repository.ListDueBackupPolicies(ctx, now.Format(time.RFC3339Nano)) + if err != nil { + return err + } + for _, policy := range policies { + location, locationErr := time.LoadLocation(policy.Timezone) + schedule, parseErr := cronParser().Parse(policy.CronExpression) + if locationErr != nil || parseErr != nil { + continue + } + policy.NextRunAt = schedule.Next(now.In(location)).UTC().Format(time.RFC3339Nano) + if err := s.repository.SetBackupPolicy(ctx, policy); err != nil { + return err + } + if _, err := s.Create(ctx, "", policy.InstanceID, "scheduled"); err != nil && !errors.Is(err, instance.ErrOperationConflict) { + return err + } + } + return nil +} + +func (s *Service) Export(ctx context.Context, instanceID, backupID string) (*os.File, Backup, error) { + backup, _, err := s.repository.GetBackup(ctx, instanceID, backupID) + if err != nil { + return nil, Backup{}, err + } + if backup.Status != "available" { + return nil, Backup{}, ErrInvalidState + } + full, err := s.backupPath(backup.RelativePath) + if err != nil { + return nil, Backup{}, err + } + if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil { + return nil, Backup{}, err + } + file, err := os.Open(full) + return file, backup, err +} + +func (s *Service) Restore(ctx context.Context, actorID, instanceID, backupID string) error { + current, err := s.repository.GetInstance(ctx, instanceID) + if err != nil { + return err + } + backup, manifest, err := s.repository.GetBackup(ctx, instanceID, backupID) + if err != nil { + return err + } + if backup.Status != "available" { + return ErrInvalidState + } + if manifest.InstanceID != instanceID || manifest.TemplateID != current.Preview.Template.ID || manifest.TemplateVersion != current.Preview.Template.Version { + return ErrIncompatible + } + full, err := s.backupPath(backup.RelativePath) + if err != nil { + return err + } + if err := verifyFile(full, backup.SizeBytes, backup.SHA256); err != nil { + return err + } + operationID := token() + current, err = s.repository.BeginOperation(ctx, operationID, instanceID, "restore", "restore") + if err != nil { + return err + } + wasRunning := current.DesiredRunning + if wasRunning { + if _, err := s.agent.StopInstance(ctx, instanceID, current.Preview.StopTimeoutSeconds); err != nil { + return s.restoreFail(ctx, operationID, "restore_stop_failed", err) + } + } + safety := Backup{ID: token(), InstanceID: instanceID, Origin: "pre_restore", Status: "creating", CreatedAt: s.now().UTC().Format(time.RFC3339Nano)} + if err := s.repository.BeginBackup(ctx, safety, operationID, actorID); err != nil { + return s.restoreFail(ctx, operationID, "restore_safety_metadata_failed", err) + } + safety, safetyManifest, err := s.writeArchive(ctx, current, safety) + if err != nil { + return s.restoreFail(ctx, operationID, "restore_safety_backup_failed", err) + } + if err := s.repository.CompleteBackup(ctx, safety.ID, safety.RelativePath, safety.SizeBytes, safety.SHA256, safetyManifest); err != nil { + return s.restoreFail(ctx, operationID, "restore_safety_persist_failed", err) + } + if err := s.restoreArchive(full, current, manifest); err != nil { + return s.restoreFail(ctx, operationID, "restore_extract_failed", err) + } + state, lifecycle, observed := agentwire.InstanceState{InstanceID: instanceID, ContainerID: current.ContainerID, Health: "stopped"}, "stopped", "stopped" + if wasRunning { + state, err = s.agent.StartInstance(ctx, instanceID) + if err != nil { + return s.restoreFail(ctx, operationID, "restore_restart_failed", err) + } + lifecycle, observed = lifecycleFromAgent(state) + } + return s.repository.FinishOperation(ctx, operationID, lifecycle, observed, state.ContainerID, current.PlanDigest, wasRunning, "") +} + +func (s *Service) writeArchive(ctx context.Context, current instance.StoredInstance, backup Backup) (Backup, Manifest, error) { + mounts := make(map[string]string) + for _, mount := range current.Preview.Mounts { + mounts[mount.ID] = mount.HostPath + } + manifest := Manifest{SchemaVersion: manifestSchemaVersion, BackupID: backup.ID, InstanceID: backup.InstanceID, TemplateID: current.Preview.Template.ID, TemplateVersion: current.Preview.Template.Version, Origin: backup.Origin, CreatedAt: backup.CreatedAt, MountIDs: append([]string(nil), current.Preview.Backup.SourceMounts...)} + sort.Strings(manifest.MountIDs) + directory, err := s.instanceBackupDirectory(backup.InstanceID) + if err != nil { + return Backup{}, Manifest{}, err + } + if err := os.MkdirAll(directory, 0o750); err != nil { + return Backup{}, Manifest{}, err + } + var estimated int64 + for _, mountID := range manifest.MountIDs { + source := mounts[mountID] + if source == "" { + return Backup{}, Manifest{}, ErrUnsafePath + } + if _, err := s.serverPath(source); err != nil { + return Backup{}, Manifest{}, err + } + size, err := estimateTree(ctx, source) + if err != nil { + return Backup{}, Manifest{}, err + } + estimated += size + } + if err := ensureFreeSpace(directory, estimated+(64<<20)); err != nil { + return Backup{}, Manifest{}, err + } + temporary, err := os.CreateTemp(directory, ".creating-*.tar.zst") + if err != nil { + return Backup{}, Manifest{}, err + } + temporaryName := temporary.Name() + defer func() { _ = temporary.Close(); _ = os.Remove(temporaryName) }() + hasher := sha256.New() + zstdWriter, err := zstd.NewWriter(io.MultiWriter(temporary, hasher), zstd.WithEncoderConcurrency(1)) + if err != nil { + return Backup{}, Manifest{}, err + } + tarWriter := tar.NewWriter(zstdWriter) + manifestJSON, _ := json.Marshal(manifest) + err = tarWriter.WriteHeader(&tar.Header{Name: "manifest.json", Mode: 0o600, Size: int64(len(manifestJSON)), ModTime: s.now().UTC(), Typeflag: tar.TypeReg}) + if err == nil { + _, err = tarWriter.Write(manifestJSON) + } + if err == nil { + for _, mountID := range manifest.MountIDs { + source := mounts[mountID] + if source == "" { + err = ErrUnsafePath + break + } + if _, pathErr := s.serverPath(source); pathErr != nil { + err = pathErr + break + } + if walkErr := addTree(ctx, tarWriter, source, path.Join("data", mountID)); walkErr != nil { + err = walkErr + break + } + } + } + if closeErr := tarWriter.Close(); err == nil { + err = closeErr + } + if closeErr := zstdWriter.Close(); err == nil { + err = closeErr + } + if syncErr := temporary.Sync(); err == nil { + err = syncErr + } + if closeErr := temporary.Close(); err == nil { + err = closeErr + } + if err != nil { + return Backup{}, Manifest{}, err + } + info, err := os.Stat(temporaryName) + if err != nil { + return Backup{}, Manifest{}, err + } + finalName := backup.ID + ".tar.zst" + finalPath := filepath.Join(directory, finalName) + if err := os.Rename(temporaryName, finalPath); err != nil { + return Backup{}, Manifest{}, err + } + backup.RelativePath = filepath.ToSlash(filepath.Join(backup.InstanceID, finalName)) + backup.SizeBytes, backup.SHA256 = info.Size(), hex.EncodeToString(hasher.Sum(nil)) + return backup, manifest, nil +} + +func addTree(ctx context.Context, writer *tar.Writer, source, prefix string) error { + return filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + info, err := entry.Info() + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) { + return ErrUnsafePath + } + relative, err := filepath.Rel(source, current) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return ErrUnsafePath + } + name := prefix + if relative != "." { + name = path.Join(prefix, filepath.ToSlash(relative)) + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name, header.Uid, header.Gid, header.Uname, header.Gname = name, 0, 0, "", "" + if err := writer.WriteHeader(header); err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + file, err := os.Open(current) + if err != nil { + return err + } + _, copyErr := io.Copy(writer, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }) +} + +func estimateTree(ctx context.Context, source string) (int64, error) { + var total int64 + err := filepath.WalkDir(source, func(_ string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + info, err := entry.Info() + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 || (!info.Mode().IsRegular() && !info.IsDir()) { + return ErrUnsafePath + } + if info.Mode().IsRegular() { + total += info.Size() + } + return nil + }) + return total, err +} + +func ensureFreeSpace(directory string, required int64) error { + var stats unix.Statfs_t + if err := unix.Statfs(directory, &stats); err != nil { + return err + } + available := int64(stats.Bavail) * int64(stats.Bsize) + if available < required { + return ErrInvalidState + } + return nil +} + +func (s *Service) restoreArchive(archive string, current instance.StoredInstance, manifest Manifest) error { + stage, err := os.MkdirTemp(s.serversRoot, ".dogama-restore-") + if err != nil { + return err + } + defer os.RemoveAll(stage) + file, err := os.Open(archive) + if err != nil { + return err + } + defer file.Close() + decoder, err := zstd.NewReader(file, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(2<<30)) + if err != nil { + return err + } + defer decoder.Close() + reader := tar.NewReader(decoder) + var total int64 + sawManifest := false + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + if header.Name == "manifest.json" { + if sawManifest || header.Size < 1 || header.Size > 64<<10 { + return ErrIntegrity + } + var archived Manifest + if err := json.NewDecoder(io.LimitReader(reader, header.Size)).Decode(&archived); err != nil { + return ErrIntegrity + } + if archived.SchemaVersion != manifestSchemaVersion || archived.BackupID != manifest.BackupID || archived.InstanceID != manifest.InstanceID || archived.TemplateID != manifest.TemplateID || archived.TemplateVersion != manifest.TemplateVersion || strings.Join(archived.MountIDs, "\x00") != strings.Join(manifest.MountIDs, "\x00") { + return ErrIntegrity + } + sawManifest = true + continue + } + if !fs.ValidPath(header.Name) || !strings.HasPrefix(header.Name, "data/") || header.Linkname != "" { + return ErrUnsafePath + } + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeDir { + return ErrUnsafePath + } + total += header.Size + if total > 1<<40 { + return ErrInvalidInput + } + target := filepath.Join(stage, filepath.FromSlash(header.Name)) + if !within(stage, target) { + return ErrUnsafePath + } + if header.Typeflag == tar.TypeDir { + if err := os.MkdirAll(target, 0o750); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640) + if err != nil { + return err + } + _, copyErr := io.CopyN(out, reader, header.Size) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } + if !sawManifest { + return ErrIntegrity + } + mountPaths := make(map[string]string) + for _, mount := range current.Preview.Mounts { + mountPaths[mount.ID] = mount.HostPath + } + var swapped []struct{ live, previous string } + rollback := func() { + for index := len(swapped) - 1; index >= 0; index-- { + _ = os.RemoveAll(swapped[index].live) + _ = os.Rename(swapped[index].previous, swapped[index].live) + } + } + for _, mountID := range manifest.MountIDs { + live := mountPaths[mountID] + if _, err := s.serverPath(live); err != nil { + rollback() + return err + } + staged := filepath.Join(stage, "data", mountID) + if _, err := os.Stat(staged); err != nil { + rollback() + return ErrIntegrity + } + previous := live + ".dogama-previous-" + manifest.BackupID + if err := os.Rename(live, previous); err != nil { + rollback() + return err + } + if err := os.Rename(staged, live); err != nil { + _ = os.Rename(previous, live) + rollback() + return err + } + swapped = append(swapped, struct{ live, previous string }{live, previous}) + } + for _, item := range swapped { + if err := os.RemoveAll(item.previous); err != nil { + return err + } + } + return nil +} + +func (s *Service) applyRetention(ctx context.Context, instanceID string, count int) error { + candidates, err := s.repository.RetentionCandidates(ctx, instanceID, count) + if err != nil { + return err + } + for _, candidate := range candidates { + full, err := s.backupPath(candidate.RelativePath) + if err != nil { + return err + } + if err := os.Remove(full); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := s.repository.MarkBackupDeleted(ctx, candidate.ID); err != nil { + return err + } + } + return nil +} + +func (s *Service) fail(ctx context.Context, operationID, backupID, code string, cause error) error { + _ = s.repository.FailBackup(ctx, backupID, code) + _ = s.repository.FailOperation(ctx, operationID, "error", code) + return fmt.Errorf("%s: %w", code, cause) +} + +func (s *Service) restoreFail(ctx context.Context, operationID, code string, cause error) error { + _ = s.repository.FailOperation(ctx, operationID, "intervention_required", code) + return fmt.Errorf("%s: %w", code, cause) +} + +func (s *Service) instanceBackupDirectory(instanceID string) (string, error) { + if instanceID == "" || strings.ContainsAny(instanceID, `/\\`) { + return "", ErrUnsafePath + } + result := filepath.Join(s.backupsRoot, instanceID) + if !within(s.backupsRoot, result) { + return "", ErrUnsafePath + } + return result, nil +} + +func (s *Service) backupPath(relative string) (string, error) { + if !fs.ValidPath(relative) { + return "", ErrUnsafePath + } + result := filepath.Join(s.backupsRoot, filepath.FromSlash(relative)) + if !within(s.backupsRoot, result) { + return "", ErrUnsafePath + } + return result, nil +} + +func (s *Service) serverPath(value string) (string, error) { + clean := filepath.Clean(value) + if !filepath.IsAbs(clean) || !within(s.serversRoot, clean) { + return "", ErrUnsafePath + } + return clean, nil +} + +func canonicalRoot(value string) (string, error) { + if value == "" { + return "", ErrUnsafePath + } + absolute, err := filepath.Abs(value) + if err != nil { + return "", err + } + if err := os.MkdirAll(absolute, 0o750); err != nil { + return "", err + } + return filepath.EvalSymlinks(absolute) +} + +func within(root, candidate string) bool { + relative, err := filepath.Rel(root, candidate) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func verifyFile(name string, expectedSize int64, expectedSHA string) error { + file, err := os.Open(name) + if err != nil { + return err + } + defer file.Close() + hasher := sha256.New() + size, err := io.Copy(hasher, file) + if err != nil { + return err + } + if size != expectedSize || !strings.EqualFold(hex.EncodeToString(hasher.Sum(nil)), expectedSHA) { + return ErrIntegrity + } + return nil +} + +func validOrigin(value string) bool { + switch value { + case "manual", "scheduled", "pre_update", "pre_restore", "idle_shutdown", "imported", "system": + return true + } + return false +} + +func token() string { + buffer := make([]byte, 24) + if _, err := rand.Read(buffer); err != nil { + return "" + } + return base64.RawURLEncoding.EncodeToString(buffer) +} + +func lifecycleFromAgent(state agentwire.InstanceState) (string, string) { + if !state.Running { + return "stopped", "stopped" + } + if state.Ready { + return "online", "ready" + } + if state.Health == "unhealthy" || state.Health == "none" { + return "degraded", "degraded" + } + return "starting", "running" +} + +func cronParser() cron.Parser { + return cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow) +} diff --git a/internal/backup/service_test.go b/internal/backup/service_test.go new file mode 100644 index 0000000..03de0b0 --- /dev/null +++ b/internal/backup/service_test.go @@ -0,0 +1,205 @@ +package backup_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + catalogdata "git.zaynet.fr/DoGaMa/DoGaMa-serv/catalog" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite" +) + +type backupAgent struct{ running bool } + +func (a *backupAgent) StartInstance(_ context.Context, id string) (agentwire.InstanceState, error) { + a.running = true + return agentwire.InstanceState{InstanceID: id, ContainerID: "container", Running: true, Ready: true, Health: "healthy"}, nil +} + +func (a *backupAgent) StopInstance(_ context.Context, id string, _ int) (agentwire.InstanceState, error) { + a.running = false + return agentwire.InstanceState{InstanceID: id, ContainerID: "container", Health: "stopped"}, nil +} + +func TestCreateRestoreExportAndRetention(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + serversRoot, backupsRoot := filepath.Join(root, "servers"), filepath.Join(root, "backups") + mount := filepath.Join(serversRoot, "instance", "saved") + if err := os.MkdirAll(mount, 0o750); err != nil { + t.Fatal(err) + } + world := filepath.Join(mount, "Level.sav") + if err := os.WriteFile(world, []byte("world-v1"), 0o640); err != nil { + t.Fatal(err) + } + db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + repository := sqlite.NewRepository(db) + snapshots, err := catalog.LoadFS(catalogdata.Files, ".") + if err != nil { + t.Fatal(err) + } + if err := repository.Sync(ctx, snapshots); err != nil { + t.Fatal(err) + } + authService := auth.New(db) + if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil { + t.Fatal(err) + } + adminSession, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234") + if err != nil { + t.Fatal(err) + } + admin, err := authService.Authenticate(ctx, adminSession.Token) + if err != nil { + t.Fatal(err) + } + preview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{ + DisplayName: "Backup Test", Slug: "backup-test", HostPorts: map[string]int{"game": 38211}, + MountPaths: map[string]string{"saved": mount}, DataOrigin: "new", BackupRetention: 2, + }) + if err != nil { + t.Fatal(err) + } + const instanceID = "backup-instance" + if err := repository.CreateDraft(ctx, instance.Draft{ID: instanceID, Preview: preview}); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE instances SET lifecycle_state='online', observed_state='ready', container_id='container', desired_running=1 WHERE id=?`, instanceID); err != nil { + t.Fatal(err) + } + agent := &backupAgent{running: true} + service, err := backup.New(repository, agent, serversRoot, backupsRoot) + if err != nil { + t.Fatal(err) + } + manual, err := service.Create(ctx, admin.ID, instanceID, "manual") + if err != nil { + t.Fatal(err) + } + if manual.Status != "available" || manual.SHA256 == "" || manual.SizeBytes == 0 || !agent.running { + t.Fatalf("backup = %#v, running=%v", manual, agent.running) + } + exported, metadata, err := service.Export(ctx, instanceID, manual.ID) + if err != nil { + t.Fatal(err) + } + if metadata.ID != manual.ID { + t.Fatalf("export metadata = %#v", metadata) + } + if err := exported.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(world, []byte("world-v2"), 0o640); err != nil { + t.Fatal(err) + } + if err := service.Restore(ctx, admin.ID, instanceID, manual.ID); err != nil { + t.Fatal(err) + } + restored, err := os.ReadFile(world) + if err != nil || string(restored) != "world-v1" { + t.Fatalf("restored world = %q, error = %v", restored, err) + } + for index := 0; index < 3; index++ { + if _, err := service.Create(ctx, admin.ID, instanceID, "scheduled"); err != nil { + t.Fatal(err) + } + } + backups, err := service.List(ctx, instanceID) + if err != nil { + t.Fatal(err) + } + var manualCount, scheduledCount, safetyCount int + for _, item := range backups { + switch item.Origin { + case "manual": + manualCount++ + case "scheduled": + scheduledCount++ + case "pre_restore": + safetyCount++ + } + } + if manualCount != 1 || scheduledCount != 2 || safetyCount != 1 { + t.Fatalf("retained backups: manual=%d scheduled=%d safety=%d", manualCount, scheduledCount, safetyCount) + } +} + +func TestBackupRejectsSymlinks(t *testing.T) { + ctx, service, actorID, instanceID, mount := backupFixture(t) + outside := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(mount, "escape")); err != nil { + t.Fatal(err) + } + _, err := service.Create(ctx, actorID, instanceID, "manual") + if !errors.Is(err, backup.ErrUnsafePath) { + t.Fatalf("symlink backup error = %v", err) + } +} + +func backupFixture(t *testing.T) (context.Context, *backup.Service, string, string, string) { + t.Helper() + ctx := context.Background() + root := t.TempDir() + serversRoot := filepath.Join(root, "servers") + backupsRoot := filepath.Join(root, "backups") + mount := filepath.Join(serversRoot, "instance", "saved") + if err := os.MkdirAll(mount, 0o750); err != nil { + t.Fatal(err) + } + db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + repository := sqlite.NewRepository(db) + snapshots, err := catalog.LoadFS(catalogdata.Files, ".") + if err != nil { + t.Fatal(err) + } + if err := repository.Sync(ctx, snapshots); err != nil { + t.Fatal(err) + } + authService := auth.New(db) + if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil { + t.Fatal(err) + } + session, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234") + if err != nil { + t.Fatal(err) + } + actor, err := authService.Authenticate(ctx, session.Token) + if err != nil { + t.Fatal(err) + } + preview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{DisplayName: "Backup Test", Slug: "backup-test", HostPorts: map[string]int{"game": 38211}, MountPaths: map[string]string{"saved": mount}, DataOrigin: "new", BackupRetention: 2}) + if err != nil { + t.Fatal(err) + } + instanceID := "backup-instance" + if err := repository.CreateDraft(ctx, instance.Draft{ID: instanceID, Preview: preview}); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE instances SET lifecycle_state='stopped', observed_state='stopped', container_id='container' WHERE id=?`, instanceID); err != nil { + t.Fatal(err) + } + service, err := backup.New(repository, &backupAgent{}, serversRoot, backupsRoot) + if err != nil { + t.Fatal(err) + } + return ctx, service, actor.ID, instanceID, mount +} diff --git a/internal/catalog/template.go b/internal/catalog/template.go index c045d39..ec51a6f 100644 --- a/internal/catalog/template.go +++ b/internal/catalog/template.go @@ -88,8 +88,13 @@ type Template struct { IntervalSeconds int `json:"interval_seconds"` } `json:"healthcheck"` Imports struct { - Supported bool `json:"supported"` - DestinationMount string `json:"destination_mount"` + Supported bool `json:"supported"` + AcceptedFormats []string `json:"accepted_formats"` + MaxExtractedSizeGB int `json:"max_extracted_size_gb"` + RequiredPaths []string `json:"required_paths"` + DestinationMount string `json:"destination_mount"` + DestinationRelativePath string `json:"destination_relative_path"` + RequiresStoppedServer bool `json:"requires_stopped_server"` } `json:"imports"` } diff --git a/internal/importexport/service.go b/internal/importexport/service.go new file mode 100644 index 0000000..d6bcc2a --- /dev/null +++ b/internal/importexport/service.go @@ -0,0 +1,357 @@ +// Package importexport validates untrusted game-data archives in isolated staging. +package importexport + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "crypto/rand" + "encoding/base64" + "errors" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/klauspost/compress/zstd" +) + +var ( + ErrInvalidInput = errors.New("invalid import input") + ErrUnsafeArchive = errors.New("unsafe import archive") + ErrLimitExceeded = errors.New("import limit exceeded") + ErrNotRecognized = errors.New("import layout not recognized") +) + +const ( + maxFiles = 100000 + maxDepth = 20 +) + +type Policy struct { + AcceptedFormats []string + MaxExpandedBytes int64 + RequiredPaths []string +} + +type Import struct { + ID string `json:"id"` + Status string `json:"status"` + Format string `json:"format"` + DetectedType string `json:"detected_type,omitempty"` + Confidence string `json:"confidence,omitempty"` + FileCount int `json:"file_count"` + ExpandedSizeBytes int64 `json:"expanded_size_bytes"` + ExpiresAt string `json:"expires_at"` + RelativeStagePath string `json:"-"` +} + +type Repository interface { + BeginImport(context.Context, Import, string) error + CompleteImport(context.Context, Import) error + FailImport(context.Context, string, string) error + ExpireImports(context.Context, string) ([]string, error) +} + +func (s *Service) CleanupExpired(ctx context.Context) error { + paths, err := s.repository.ExpireImports(ctx, s.now().UTC().Format(time.RFC3339Nano)) + if err != nil { + return err + } + for _, relative := range paths { + if !fs.ValidPath(relative) { + return ErrUnsafeArchive + } + target := filepath.Join(s.root, filepath.FromSlash(relative)) + if !withinRoot(s.root, target) { + return ErrUnsafeArchive + } + if err := os.RemoveAll(target); err != nil { + return err + } + } + return nil +} + +type Service struct { + repository Repository + root string + now func() time.Time +} + +func New(repository Repository, root string) (*Service, error) { + if root == "" { + return nil, ErrInvalidInput + } + absolute, err := filepath.Abs(root) + if err != nil { + return nil, err + } + if err := os.MkdirAll(absolute, 0o750); err != nil { + return nil, err + } + canonical, err := filepath.EvalSymlinks(absolute) + if err != nil { + return nil, err + } + return &Service{repository: repository, root: canonical, now: time.Now}, nil +} + +func (s *Service) Stage(ctx context.Context, actorID, format string, source io.Reader, policy Policy) (Import, error) { + if actorID == "" || !contains(policy.AcceptedFormats, format) || policy.MaxExpandedBytes < 1 { + return Import{}, ErrInvalidInput + } + id := importToken() + if id == "" { + return Import{}, errors.New("generate import identifier") + } + directory := filepath.Join(s.root, id) + if !withinRoot(s.root, directory) { + return Import{}, ErrUnsafeArchive + } + if err := os.Mkdir(directory, 0o750); err != nil { + return Import{}, err + } + value := Import{ID: id, Status: "staging", Format: format, RelativeStagePath: id, ExpiresAt: s.now().Add(24 * time.Hour).UTC().Format(time.RFC3339Nano)} + if err := s.repository.BeginImport(ctx, value, actorID); err != nil { + _ = os.RemoveAll(directory) + return Import{}, err + } + fail := func(code string, err error) (Import, error) { + _ = s.repository.FailImport(ctx, id, code) + _ = os.RemoveAll(directory) + return Import{}, err + } + upload := filepath.Join(directory, "upload") + file, err := os.OpenFile(upload, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fail("import_stage_failed", err) + } + maxUpload := policy.MaxExpandedBytes + written, copyErr := io.Copy(file, io.LimitReader(source, maxUpload+1)) + closeErr := file.Close() + if copyErr != nil || closeErr != nil { + if copyErr == nil { + copyErr = closeErr + } + return fail("import_upload_failed", copyErr) + } + if written > maxUpload { + return fail("import_upload_limit", ErrLimitExceeded) + } + extracted := filepath.Join(directory, "data") + if err := os.Mkdir(extracted, 0o750); err != nil { + return fail("import_stage_failed", err) + } + files, size, paths, err := extract(ctx, upload, extracted, format, policy.MaxExpandedBytes) + if err != nil { + return fail("import_validation_failed", err) + } + if !requiredPresent(paths, policy.RequiredPaths) { + return fail("import_layout_unrecognized", ErrNotRecognized) + } + if err := os.Remove(upload); err != nil { + return fail("import_cleanup_failed", err) + } + value.Status, value.DetectedType, value.Confidence, value.FileCount, value.ExpandedSizeBytes = "validated", "game_save", "confirmed", files, size + if err := s.repository.CompleteImport(ctx, value); err != nil { + return fail("import_persist_failed", err) + } + return value, nil +} + +func extract(ctx context.Context, archive, destination, format string, limit int64) (int, int64, []string, error) { + switch format { + case "zip": + return extractZIP(ctx, archive, destination, limit) + case "tar", "tar.gz", "tar.zst": + return extractTar(ctx, archive, destination, format, limit) + default: + return 0, 0, nil, ErrInvalidInput + } +} + +func extractZIP(ctx context.Context, archive, destination string, limit int64) (int, int64, []string, error) { + reader, err := zip.OpenReader(archive) + if err != nil { + return 0, 0, nil, ErrUnsafeArchive + } + defer reader.Close() + var count int + var total int64 + var paths []string + for _, entry := range reader.File { + select { + case <-ctx.Done(): + return 0, 0, nil, ctx.Err() + default: + } + name, err := safeName(entry.Name) + if err != nil || entry.Mode()&os.ModeSymlink != 0 || (!entry.Mode().IsRegular() && !entry.FileInfo().IsDir()) { + return 0, 0, nil, ErrUnsafeArchive + } + if entry.FileInfo().IsDir() { + if err := os.MkdirAll(filepath.Join(destination, filepath.FromSlash(name)), 0o750); err != nil { + return 0, 0, nil, err + } + continue + } + count++ + total += int64(entry.UncompressedSize64) + if count > maxFiles || total > limit { + return 0, 0, nil, ErrLimitExceeded + } + input, err := entry.Open() + if err != nil { + return 0, 0, nil, err + } + if err := writeExtracted(destination, name, input, int64(entry.UncompressedSize64)); err != nil { + input.Close() + return 0, 0, nil, err + } + if err := input.Close(); err != nil { + return 0, 0, nil, err + } + paths = append(paths, name) + } + return count, total, paths, nil +} + +func extractTar(ctx context.Context, archive, destination, format string, limit int64) (int, int64, []string, error) { + file, err := os.Open(archive) + if err != nil { + return 0, 0, nil, err + } + defer file.Close() + var source io.Reader = file + var closer io.Closer + if format == "tar.gz" { + value, err := gzip.NewReader(file) + if err != nil { + return 0, 0, nil, ErrUnsafeArchive + } + source, closer = value, value + } + if format == "tar.zst" { + value, err := zstd.NewReader(file, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(2<<30)) + if err != nil { + return 0, 0, nil, ErrUnsafeArchive + } + source, closer = value, value.IOReadCloser() + } + if closer != nil { + defer closer.Close() + } + reader := tar.NewReader(source) + var count int + var total int64 + var paths []string + for { + select { + case <-ctx.Done(): + return 0, 0, nil, ctx.Err() + default: + } + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return 0, 0, nil, ErrUnsafeArchive + } + name, err := safeName(header.Name) + if err != nil || header.Linkname != "" { + return 0, 0, nil, ErrUnsafeArchive + } + if header.Typeflag == tar.TypeDir { + if err := os.MkdirAll(filepath.Join(destination, filepath.FromSlash(name)), 0o750); err != nil { + return 0, 0, nil, err + } + continue + } + if !header.FileInfo().Mode().IsRegular() { + return 0, 0, nil, ErrUnsafeArchive + } + count++ + total += header.Size + if count > maxFiles || total > limit { + return 0, 0, nil, ErrLimitExceeded + } + if err := writeExtracted(destination, name, reader, header.Size); err != nil { + return 0, 0, nil, err + } + paths = append(paths, name) + } + return count, total, paths, nil +} + +func writeExtracted(root, name string, source io.Reader, size int64) error { + target := filepath.Join(root, filepath.FromSlash(name)) + if !withinRoot(root, target) { + return ErrUnsafeArchive + } + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640) + if err != nil { + return err + } + written, copyErr := io.CopyN(file, source, size) + closeErr := file.Close() + if copyErr != nil || written != size { + return ErrUnsafeArchive + } + return closeErr +} + +func safeName(value string) (string, error) { + if strings.Contains(value, "\\") || strings.ContainsRune(value, 0) || strings.HasPrefix(value, "/") { + return "", ErrUnsafeArchive + } + clean := path.Clean(value) + if clean == "." || !fs.ValidPath(clean) || strings.Contains(strings.Split(clean, "/")[0], ":") || len(strings.Split(clean, "/")) > maxDepth { + return "", ErrUnsafeArchive + } + return clean, nil +} + +func requiredPresent(paths, required []string) bool { + for _, expected := range required { + found := false + for _, candidate := range paths { + if candidate == expected || strings.HasSuffix(candidate, "/"+expected) || strings.HasPrefix(candidate, expected+"/") || strings.Contains(candidate, "/"+expected+"/") { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func contains(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} +func withinRoot(root, candidate string) bool { + relative, err := filepath.Rel(root, candidate) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} +func importToken() string { + value := make([]byte, 24) + if _, err := rand.Read(value); err != nil { + return "" + } + return base64.RawURLEncoding.EncodeToString(value) +} diff --git a/internal/importexport/service_test.go b/internal/importexport/service_test.go new file mode 100644 index 0000000..52cde68 --- /dev/null +++ b/internal/importexport/service_test.go @@ -0,0 +1,76 @@ +package importexport_test + +import ( + "archive/zip" + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite" +) + +func TestStageValidZIPAndRejectTraversal(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + authService := auth.New(db) + if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil { + t.Fatal(err) + } + session, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234") + if err != nil { + t.Fatal(err) + } + actor, err := authService.Authenticate(ctx, session.Token) + if err != nil { + t.Fatal(err) + } + service, err := importexport.New(sqlite.NewRepository(db), filepath.Join(root, "imports")) + if err != nil { + t.Fatal(err) + } + policy := importexport.Policy{AcceptedFormats: []string{"zip"}, MaxExpandedBytes: 1 << 20, RequiredPaths: []string{"Level.sav", "Players"}} + valid := zipBytes(t, map[string]string{"Save/Level.sav": "world", "Save/Players/player.sav": "player"}) + result, err := service.Stage(ctx, actor.ID, "zip", bytes.NewReader(valid), policy) + if err != nil { + t.Fatal(err) + } + if result.Status != "validated" || result.Confidence != "confirmed" || result.FileCount != 2 { + t.Fatalf("import = %#v", result) + } + unsafe := zipBytes(t, map[string]string{"../escape": "bad", "Level.sav": "world", "Players/player": "player"}) + if _, err := service.Stage(ctx, actor.ID, "zip", bytes.NewReader(unsafe), policy); err == nil { + t.Fatal("traversal archive was accepted") + } + if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) { + t.Fatalf("escape path exists: %v", err) + } +} + +func zipBytes(t *testing.T, files map[string]string) []byte { + t.Helper() + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + for name, body := range files { + entry, err := writer.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(entry, body); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} diff --git a/internal/persistence/sqlite/backup.go b/internal/persistence/sqlite/backup.go new file mode 100644 index 0000000..f7dbea2 --- /dev/null +++ b/internal/persistence/sqlite/backup.go @@ -0,0 +1,178 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup" +) + +func (r *Repository) BeginBackup(ctx context.Context, value backup.Backup, operationID, actorID string) error { + var operation, actor any + if operationID != "" { + operation = operationID + } + if actorID != "" { + actor = actorID + } + _, err := r.db.ExecContext(ctx, `INSERT INTO backups(id, instance_id, operation_id, origin, status, created_by, created_at) + VALUES (?, ?, ?, ?, 'creating', ?, ?)`, value.ID, value.InstanceID, operation, value.Origin, actor, value.CreatedAt) + if err != nil { + return fmt.Errorf("begin backup metadata: %w", err) + } + return nil +} + +func (r *Repository) CompleteBackup(ctx context.Context, id, relativePath string, size int64, checksum string, manifest backup.Manifest) error { + encoded, err := json.Marshal(manifest) + if err != nil { + return fmt.Errorf("encode backup manifest: %w", err) + } + now := r.now().UTC().Format(time.RFC3339Nano) + result, err := r.db.ExecContext(ctx, `UPDATE backups SET status='available', relative_path=?, size_bytes=?, sha256=?, manifest_json=?, completed_at=?, error_code=NULL + WHERE id=? AND status='creating'`, relativePath, size, checksum, string(encoded), now, id) + if err != nil { + return fmt.Errorf("complete backup metadata: %w", err) + } + changed, _ := result.RowsAffected() + if changed != 1 { + return backup.ErrInvalidState + } + return nil +} + +func (r *Repository) FailBackup(ctx context.Context, id, code string) error { + result, err := r.db.ExecContext(ctx, `UPDATE backups SET status='failed', error_code=?, completed_at=? WHERE id=? AND status='creating'`, code, r.now().UTC().Format(time.RFC3339Nano), id) + if err != nil { + return fmt.Errorf("fail backup metadata: %w", err) + } + changed, _ := result.RowsAffected() + if changed != 1 { + return backup.ErrInvalidState + } + return nil +} + +func (r *Repository) ListBackups(ctx context.Context, instanceID string) ([]backup.Backup, error) { + rows, err := r.db.QueryContext(ctx, `SELECT id, instance_id, origin, status, COALESCE(relative_path, ''), COALESCE(size_bytes, 0), COALESCE(sha256, ''), created_at, COALESCE(completed_at, ''), COALESCE(error_code, '') + FROM backups WHERE instance_id=? AND status!='deleted' ORDER BY created_at DESC, id`, instanceID) + if err != nil { + return nil, fmt.Errorf("list backups: %w", err) + } + defer rows.Close() + var result []backup.Backup + for rows.Next() { + var value backup.Backup + if err := rows.Scan(&value.ID, &value.InstanceID, &value.Origin, &value.Status, &value.RelativePath, &value.SizeBytes, &value.SHA256, &value.CreatedAt, &value.CompletedAt, &value.ErrorCode); err != nil { + return nil, fmt.Errorf("scan backup: %w", err) + } + result = append(result, value) + } + return result, rows.Err() +} + +func (r *Repository) GetBackup(ctx context.Context, instanceID, id string) (backup.Backup, backup.Manifest, error) { + var value backup.Backup + var manifestJSON string + err := r.db.QueryRowContext(ctx, `SELECT id, instance_id, origin, status, COALESCE(relative_path, ''), COALESCE(size_bytes, 0), COALESCE(sha256, ''), created_at, COALESCE(completed_at, ''), COALESCE(error_code, ''), COALESCE(manifest_json, '') + FROM backups WHERE id=? AND instance_id=? AND status!='deleted'`, id, instanceID).Scan(&value.ID, &value.InstanceID, &value.Origin, &value.Status, &value.RelativePath, &value.SizeBytes, &value.SHA256, &value.CreatedAt, &value.CompletedAt, &value.ErrorCode, &manifestJSON) + if errors.Is(err, sql.ErrNoRows) { + return backup.Backup{}, backup.Manifest{}, backup.ErrNotFound + } + if err != nil { + return backup.Backup{}, backup.Manifest{}, fmt.Errorf("get backup: %w", err) + } + var manifest backup.Manifest + if manifestJSON != "" { + if err := json.Unmarshal([]byte(manifestJSON), &manifest); err != nil { + return backup.Backup{}, backup.Manifest{}, backup.ErrIntegrity + } + } + return value, manifest, nil +} + +func (r *Repository) RetentionCandidates(ctx context.Context, instanceID string, keep int) ([]backup.Backup, error) { + rows, err := r.db.QueryContext(ctx, `SELECT id, instance_id, origin, status, relative_path, size_bytes, sha256, created_at, completed_at, '' + FROM backups WHERE instance_id=? AND origin='scheduled' AND status='available' + ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?`, instanceID, keep) + if err != nil { + return nil, fmt.Errorf("list retention candidates: %w", err) + } + defer rows.Close() + var result []backup.Backup + for rows.Next() { + var value backup.Backup + if err := rows.Scan(&value.ID, &value.InstanceID, &value.Origin, &value.Status, &value.RelativePath, &value.SizeBytes, &value.SHA256, &value.CreatedAt, &value.CompletedAt, &value.ErrorCode); err != nil { + return nil, err + } + result = append(result, value) + } + return result, rows.Err() +} + +func (r *Repository) MarkBackupDeleted(ctx context.Context, id string) error { + result, err := r.db.ExecContext(ctx, `UPDATE backups SET status='deleted', relative_path=NULL, deleted_at=? WHERE id=? AND status='available'`, r.now().UTC().Format(time.RFC3339Nano), id) + if err != nil { + return fmt.Errorf("mark backup deleted: %w", err) + } + changed, _ := result.RowsAffected() + if changed != 1 { + return backup.ErrInvalidState + } + return nil +} + +func (r *Repository) GetBackupPolicy(ctx context.Context, instanceID string) (backup.Policy, error) { + var value backup.Policy + var enabled int + err := r.db.QueryRowContext(ctx, `SELECT instance_id, enabled, COALESCE(cron_expression, ''), timezone, retention_count, COALESCE(next_run_at, '') FROM backup_policies WHERE instance_id=?`, instanceID).Scan(&value.InstanceID, &enabled, &value.CronExpression, &value.Timezone, &value.RetentionCount, &value.NextRunAt) + if errors.Is(err, sql.ErrNoRows) { + instanceValue, instanceErr := r.GetInstance(ctx, instanceID) + if instanceErr != nil { + return backup.Policy{}, instanceErr + } + return backup.Policy{InstanceID: instanceID, Timezone: "UTC", RetentionCount: instanceValue.Preview.Backup.RetentionCount}, nil + } + if err != nil { + return backup.Policy{}, fmt.Errorf("get backup policy: %w", err) + } + value.Enabled = enabled != 0 + return value, nil +} + +func (r *Repository) SetBackupPolicy(ctx context.Context, value backup.Policy) error { + enabled := 0 + if value.Enabled { + enabled = 1 + } + now := r.now().UTC().Format(time.RFC3339Nano) + _, err := r.db.ExecContext(ctx, `INSERT INTO backup_policies(instance_id, enabled, cron_expression, timezone, retention_count, next_run_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(instance_id) DO UPDATE SET enabled=excluded.enabled, cron_expression=excluded.cron_expression, timezone=excluded.timezone, retention_count=excluded.retention_count, next_run_at=excluded.next_run_at, updated_at=excluded.updated_at`, value.InstanceID, enabled, nullable(value.CronExpression), value.Timezone, value.RetentionCount, nullable(value.NextRunAt), now, now) + if err != nil { + return fmt.Errorf("set backup policy: %w", err) + } + return nil +} + +func (r *Repository) ListDueBackupPolicies(ctx context.Context, now string) ([]backup.Policy, error) { + rows, err := r.db.QueryContext(ctx, `SELECT instance_id, enabled, cron_expression, timezone, retention_count, next_run_at FROM backup_policies WHERE enabled=1 AND next_run_at<=? ORDER BY next_run_at`, now) + if err != nil { + return nil, fmt.Errorf("list due backup policies: %w", err) + } + defer rows.Close() + var result []backup.Policy + for rows.Next() { + var value backup.Policy + var enabled int + if err := rows.Scan(&value.InstanceID, &enabled, &value.CronExpression, &value.Timezone, &value.RetentionCount, &value.NextRunAt); err != nil { + return nil, err + } + value.Enabled = enabled != 0 + result = append(result, value) + } + return result, rows.Err() +} diff --git a/internal/persistence/sqlite/import.go b/internal/persistence/sqlite/import.go new file mode 100644 index 0000000..11fa99d --- /dev/null +++ b/internal/persistence/sqlite/import.go @@ -0,0 +1,68 @@ +package sqlite + +import ( + "context" + "fmt" + "time" + + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport" +) + +func (r *Repository) BeginImport(ctx context.Context, value importexport.Import, actorID string) error { + _, err := r.db.ExecContext(ctx, `INSERT INTO imports(id, requested_by, status, format, relative_stage_path, created_at, expires_at) VALUES (?, ?, 'staging', ?, ?, ?, ?)`, value.ID, actorID, value.Format, value.RelativeStagePath, r.now().UTC().Format(time.RFC3339Nano), value.ExpiresAt) + if err != nil { + return fmt.Errorf("begin import: %w", err) + } + return nil +} + +func (r *Repository) CompleteImport(ctx context.Context, value importexport.Import) error { + result, err := r.db.ExecContext(ctx, `UPDATE imports SET status='validated', detected_type=?, confidence=?, file_count=?, expanded_size_bytes=?, completed_at=? WHERE id=? AND status='staging'`, value.DetectedType, value.Confidence, value.FileCount, value.ExpandedSizeBytes, r.now().UTC().Format(time.RFC3339Nano), value.ID) + if err != nil { + return fmt.Errorf("complete import: %w", err) + } + changed, _ := result.RowsAffected() + if changed != 1 { + return importexport.ErrInvalidInput + } + return nil +} + +func (r *Repository) FailImport(ctx context.Context, id, code string) error { + _, err := r.db.ExecContext(ctx, `UPDATE imports SET status='failed', error_code=?, completed_at=? WHERE id=? AND status='staging'`, code, r.now().UTC().Format(time.RFC3339Nano), id) + if err != nil { + return fmt.Errorf("fail import: %w", err) + } + return nil +} + +func (r *Repository) ExpireImports(ctx context.Context, now string) ([]string, error) { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin import expiry: %w", err) + } + defer func() { _ = tx.Rollback() }() + rows, err := tx.QueryContext(ctx, `SELECT relative_stage_path FROM imports WHERE status IN ('staging', 'validated') AND expires_at<=?`, now) + if err != nil { + return nil, fmt.Errorf("list expired imports: %w", err) + } + var paths []string + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + rows.Close() + return nil, err + } + paths = append(paths, value) + } + if err := rows.Close(); err != nil { + return nil, err + } + if _, err := tx.ExecContext(ctx, `UPDATE imports SET status='expired', completed_at=? WHERE status IN ('staging', 'validated') AND expires_at<=?`, now, now); err != nil { + return nil, fmt.Errorf("expire imports: %w", err) + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("commit import expiry: %w", err) + } + return paths, nil +} diff --git a/internal/persistence/sqlite/store_test.go b/internal/persistence/sqlite/store_test.go index 6d97145..afe840e 100644 --- a/internal/persistence/sqlite/store_test.go +++ b/internal/persistence/sqlite/store_test.go @@ -24,16 +24,16 @@ func TestOpenAppliesMigrationsAndConfiguration(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&count); err != nil { t.Fatal(err) } - if count != 4 { - t.Fatalf("got %d migrations, want 4", count) + if count != 5 { + t.Fatalf("got %d migrations, want 5", count) } - for _, table := range []string{"instance_memberships", "permission_overrides", "installation_requests"} { + for _, table := range []string{"instance_memberships", "permission_overrides", "installation_requests", "backup_policies", "backups", "imports"} { var found int if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&found); err != nil { t.Fatal(err) } if found != 1 { - t.Fatalf("authorization table %q is missing", table) + t.Fatalf("required table %q is missing", table) } } var foreignKeys, busyTimeout int @@ -62,8 +62,8 @@ func TestOpenAppliesMigrationsAndConfiguration(t *testing.T) { if err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&count); err != nil { t.Fatal(err) } - if count != 4 { - t.Fatalf("reopened database has %d migrations, want 4", count) + if count != 5 { + t.Fatalf("reopened database has %d migrations, want 5", count) } } diff --git a/internal/web/server.go b/internal/web/server.go index 3243c8d..ec7d735 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -18,7 +18,9 @@ import ( "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/authorization" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/importexport" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance" ) @@ -59,6 +61,8 @@ type server struct { repository repository lifecycle *instance.LifecycleService permissions *authorization.Service + backups *backup.Service + imports *importexport.Service } type repository interface { @@ -77,26 +81,39 @@ type pageData struct { // NewHandler constructs the complete HTTP application. func NewHandler(authService *auth.Service, logger *slog.Logger) (http.Handler, error) { - return newHandler(authService, nil, nil, logger) + return newHandler(authService, nil, nil, nil, logger) } // NewHandlerWithRepository enables the authenticated catalog and draft APIs. func NewHandlerWithRepository(authService *auth.Service, repository repository, logger *slog.Logger) (http.Handler, error) { - return newHandler(authService, repository, nil, logger) + return newHandler(authService, repository, nil, nil, logger) +} + +func NewHandlerWithRepositoryAndImports(authService *auth.Service, repository repository, importService *importexport.Service, logger *slog.Logger) (http.Handler, error) { + return newHandlerWithImports(authService, repository, nil, nil, importService, logger) } // NewHandlerWithLifecycle enables privileged instance lifecycle operations // through the restricted agent boundary. func NewHandlerWithLifecycle(authService *auth.Service, repository repository, agent instance.LifecycleAgent, logger *slog.Logger) (http.Handler, error) { - return newHandler(authService, repository, instance.NewLifecycleService(repository, agent), logger) + return newHandler(authService, repository, instance.NewLifecycleService(repository, agent), nil, logger) } -func newHandler(authService *auth.Service, repository repository, lifecycle *instance.LifecycleService, logger *slog.Logger) (http.Handler, error) { +// NewHandlerWithLifecycleAndBackup enables lifecycle and backup operations. +func NewHandlerWithLifecycleAndBackup(authService *auth.Service, repository repository, agent instance.LifecycleAgent, backupService *backup.Service, importService *importexport.Service, logger *slog.Logger) (http.Handler, error) { + return newHandlerWithImports(authService, repository, instance.NewLifecycleService(repository, agent), backupService, importService, logger) +} + +func newHandler(authService *auth.Service, repository repository, lifecycle *instance.LifecycleService, backupService *backup.Service, logger *slog.Logger) (http.Handler, error) { + return newHandlerWithImports(authService, repository, lifecycle, backupService, nil, logger) +} + +func newHandlerWithImports(authService *auth.Service, repository repository, lifecycle *instance.LifecycleService, backupService *backup.Service, importService *importexport.Service, logger *slog.Logger) (http.Handler, error) { templates, err := template.New("views").Funcs(template.FuncMap{"msg": message}).ParseFS(assets, "templates/*.html") if err != nil { return nil, err } - s := &server{auth: authService, templates: templates, logger: logger, repository: repository, lifecycle: lifecycle} + s := &server{auth: authService, templates: templates, logger: logger, repository: repository, lifecycle: lifecycle, backups: backupService, imports: importService} if repository != nil { s.permissions = authorization.New(repository) } @@ -124,6 +141,17 @@ func newHandler(authService *auth.Service, repository repository, lifecycle *ins mux.HandleFunc("POST /api/v1/instances/{id}/restart", s.instanceRestart) mux.HandleFunc("DELETE /api/v1/instances/{id}", s.instanceDeleteContainer) } + if backupService != nil { + mux.HandleFunc("GET /api/v1/instances/{id}/backups", s.backupList) + mux.HandleFunc("POST /api/v1/instances/{id}/backups", s.backupCreate) + mux.HandleFunc("GET /api/v1/instances/{id}/backups/{backupID}/export", s.backupExport) + mux.HandleFunc("POST /api/v1/instances/{id}/backups/{backupID}/restore", s.backupRestore) + mux.HandleFunc("GET /api/v1/instances/{id}/backup-policy", s.backupPolicyGet) + mux.HandleFunc("PUT /api/v1/instances/{id}/backup-policy", s.backupPolicySet) + } + if importService != nil { + mux.HandleFunc("POST /api/v1/imports", s.importCreate) + } } mux.HandleFunc("GET /static/app.v1.css", s.stylesheet) mux.HandleFunc("GET /setup", s.setupForm) @@ -322,6 +350,140 @@ func (s *server) buildAPIPreview(w http.ResponseWriter, r *http.Request) (previe return request, preview, true } +func (s *server) backupList(w http.ResponseWriter, r *http.Request) { + if _, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupList); !ok { + return + } + values, err := s.backups.List(r.Context(), r.PathValue("id")) + if err != nil { + s.backupProblem(w, err) + return + } + s.apiJSON(w, http.StatusOK, map[string]any{"backups": values}) +} + +func (s *server) backupCreate(w http.ResponseWriter, r *http.Request) { + actor, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupCreate) + if !ok || !s.requireEmptyBody(w, r) { + return + } + value, err := s.backups.Create(r.Context(), actor.ID, r.PathValue("id"), "manual") + if err != nil { + s.backupProblem(w, err) + return + } + s.apiJSON(w, http.StatusCreated, value) +} + +func (s *server) backupExport(w http.ResponseWriter, r *http.Request) { + if _, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupExport); !ok { + return + } + file, value, err := s.backups.Export(r.Context(), r.PathValue("id"), r.PathValue("backupID")) + if err != nil { + s.backupProblem(w, err) + return + } + defer file.Close() + w.Header().Set("Content-Type", "application/zstd") + w.Header().Set("Content-Disposition", `attachment; filename="`+value.ID+`.tar.zst"`) + w.Header().Set("X-Content-SHA256", value.SHA256) + http.ServeContent(w, r, value.ID+".tar.zst", time.Time{}, file) +} + +func (s *server) backupRestore(w http.ResponseWriter, r *http.Request) { + actor, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupRestore) + if !ok || !s.requireEmptyBody(w, r) { + return + } + if err := s.backups.Restore(r.Context(), actor.ID, r.PathValue("id"), r.PathValue("backupID")); err != nil { + s.backupProblem(w, err) + return + } + s.apiJSON(w, http.StatusOK, map[string]string{"state": "restored"}) +} + +func (s *server) backupPolicyGet(w http.ResponseWriter, r *http.Request) { + if _, ok := s.requireInstancePermission(w, r, authorization.PermissionBackupList); !ok { + return + } + value, err := s.backups.GetPolicy(r.Context(), r.PathValue("id")) + if err != nil { + s.backupProblem(w, err) + return + } + s.apiJSON(w, http.StatusOK, value) +} + +func (s *server) backupPolicySet(w http.ResponseWriter, r *http.Request) { + if _, ok := s.requireInstancePermission(w, r, authorization.PermissionInstanceConfigure); !ok { + return + } + var request struct { + Enabled bool `json:"enabled"` + CronExpression string `json:"cron_expression"` + Timezone string `json:"timezone"` + RetentionCount int `json:"retention_count"` + } + if !s.decodeAPIJSON(w, r, &request) { + return + } + value, err := s.backups.SetPolicy(r.Context(), backup.Policy{InstanceID: r.PathValue("id"), Enabled: request.Enabled, CronExpression: request.CronExpression, Timezone: request.Timezone, RetentionCount: request.RetentionCount}) + if err != nil { + s.backupProblem(w, err) + return + } + s.apiJSON(w, http.StatusOK, value) +} + +func (s *server) backupProblem(w http.ResponseWriter, err error) { + status, code := http.StatusInternalServerError, "backup_failed" + switch { + case errors.Is(err, backup.ErrNotFound), errors.Is(err, instance.ErrInstanceNotFound): + status, code = http.StatusNotFound, "backup_not_found" + case errors.Is(err, backup.ErrInvalidInput), errors.Is(err, backup.ErrIncompatible): + status, code = http.StatusUnprocessableEntity, "invalid_backup" + case errors.Is(err, backup.ErrInvalidState), errors.Is(err, instance.ErrOperationConflict): + status, code = http.StatusConflict, "backup_conflict" + case errors.Is(err, backup.ErrIntegrity): + status, code = http.StatusUnprocessableEntity, "backup_integrity_failed" + } + s.apiProblem(w, status, code, "The backup operation could not be completed.") +} + +func (s *server) importCreate(w http.ResponseWriter, r *http.Request) { + actor, ok := s.requireAPIUser(w, r, true) + if !ok { + return + } + if err := s.permissions.RequireRecentAdmin(actor); err != nil { + s.authorizationProblem(w, err) + return + } + templateID, version, format := r.URL.Query().Get("template_id"), r.URL.Query().Get("template_version"), r.URL.Query().Get("format") + snapshot, err := s.repository.Get(r.Context(), templateID, version) + if err != nil { + s.apiProblem(w, http.StatusNotFound, "template_not_found", "The template version was not found.") + return + } + if !snapshot.Template.Imports.Supported { + s.apiProblem(w, http.StatusUnprocessableEntity, "import_unsupported", "The template does not support imports.") + return + } + maximum := int64(snapshot.Template.Imports.MaxExtractedSizeGB) << 30 + r.Body = http.MaxBytesReader(w, r.Body, maximum+1) + value, err := s.imports.Stage(r.Context(), actor.ID, format, r.Body, importexport.Policy{AcceptedFormats: snapshot.Template.Imports.AcceptedFormats, MaxExpandedBytes: maximum, RequiredPaths: snapshot.Template.Imports.RequiredPaths}) + if err != nil { + status, code := http.StatusUnprocessableEntity, "invalid_import" + if errors.Is(err, importexport.ErrLimitExceeded) { + status, code = http.StatusRequestEntityTooLarge, "import_limit_exceeded" + } + s.apiProblem(w, status, code, "The import could not be validated.") + return + } + s.apiJSON(w, http.StatusCreated, value) +} + func (s *server) requireInstancePermission(w http.ResponseWriter, r *http.Request, permission string) (auth.User, bool) { user, ok := s.requireAPIUser(w, r, false) if !ok { diff --git a/internal/web/server_test.go b/internal/web/server_test.go index cb01ebd..0efc0d4 100644 --- a/internal/web/server_test.go +++ b/internal/web/server_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "path/filepath" "strings" "testing" @@ -16,7 +17,9 @@ import ( catalogdata "git.zaynet.fr/DoGaMa/DoGaMa-serv/catalog" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/agentwire" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/auth" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/backup" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/catalog" + "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/instance" "git.zaynet.fr/DoGaMa/DoGaMa-serv/internal/persistence/sqlite" ) @@ -120,6 +123,101 @@ func TestCatalogPreviewAndDraftAPIAuthorization(t *testing.T) { assertStatus(t, unsafeResponse, http.StatusUnprocessableEntity) } +func TestBackupAPIEnforcesPermissionsAndRestores(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + serversRoot := filepath.Join(root, "servers") + backupsRoot := filepath.Join(root, "backups") + mount := filepath.Join(serversRoot, "instance", "saved") + if err := os.MkdirAll(mount, 0o750); err != nil { + t.Fatal(err) + } + world := filepath.Join(mount, "Level.sav") + if err := os.WriteFile(world, []byte("world-v1"), 0o640); err != nil { + t.Fatal(err) + } + db, err := sqlite.Open(ctx, filepath.Join(root, "dogama.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + repository := sqlite.NewRepository(db) + snapshots, err := catalog.LoadFS(catalogdata.Files, ".") + if err != nil { + t.Fatal(err) + } + if err := repository.Sync(ctx, snapshots); err != nil { + t.Fatal(err) + } + authService := auth.New(db) + if err := authService.BootstrapAdmin(ctx, "admin", "correct horse battery staple"); err != nil { + t.Fatal(err) + } + adminSession, err := authService.Login(ctx, "admin", "correct horse battery staple", "192.0.2.1:1234") + if err != nil { + t.Fatal(err) + } + player, err := authService.CreateUser(ctx, "player", "another correct battery staple", "user") + if err != nil { + t.Fatal(err) + } + playerSession, err := authService.Login(ctx, "player", "another correct battery staple", "192.0.2.2:1234") + if err != nil { + t.Fatal(err) + } + preview, err := instance.BuildPreview(snapshots[0], instance.PreviewRequest{DisplayName: "Backup API", Slug: "backup-api", HostPorts: map[string]int{"game": 38211}, MountPaths: map[string]string{"saved": mount}, DataOrigin: "new", BackupRetention: 2}) + if err != nil { + t.Fatal(err) + } + const instanceID = "backup-api-instance" + if err := repository.CreateDraft(ctx, instance.Draft{ID: instanceID, Preview: preview}); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE instances SET lifecycle_state='online', observed_state='ready', container_id='container', desired_running=1 WHERE id=?`, instanceID); err != nil { + t.Fatal(err) + } + backupService, err := backup.New(repository, webLifecycleAgent{}, serversRoot, backupsRoot) + if err != nil { + t.Fatal(err) + } + handler, err := NewHandlerWithLifecycleAndBackup(authService, repository, webLifecycleAgent{}, backupService, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + adminCookie := &http.Cookie{Name: sessionCookie, Value: adminSession.Token} + playerCookie := &http.Cookie{Name: sessionCookie, Value: playerSession.Token} + denied := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups", []*http.Cookie{playerCookie}) + assertStatus(t, denied, http.StatusForbidden) + membership := jsonMethodRequest(t, handler, http.MethodPut, "/api/v1/instances/"+instanceID+"/memberships/"+player.ID, []byte(`{"role":"manager"}`), adminCookie, adminSession.CSRFToken) + assertStatus(t, membership, http.StatusNoContent) + created := jsonRequest(t, handler, "/api/v1/instances/"+instanceID+"/backups", nil, playerCookie, playerSession.CSRFToken) + assertStatus(t, created, http.StatusCreated) + var value backup.Backup + if err := json.Unmarshal(created.Body.Bytes(), &value); err != nil { + t.Fatal(err) + } + listed := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups", []*http.Cookie{playerCookie}) + assertStatus(t, listed, http.StatusOK) + exportDenied := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups/"+value.ID+"/export", []*http.Cookie{playerCookie}) + assertStatus(t, exportDenied, http.StatusForbidden) + exported := request(t, handler, http.MethodGet, "/api/v1/instances/"+instanceID+"/backups/"+value.ID+"/export", []*http.Cookie{adminCookie}) + assertStatus(t, exported, http.StatusOK) + if exported.Header().Get("X-Content-SHA256") == "" { + t.Fatal("export checksum header missing") + } + if err := os.WriteFile(world, []byte("world-v2"), 0o640); err != nil { + t.Fatal(err) + } + restored := jsonRequest(t, handler, "/api/v1/instances/"+instanceID+"/backups/"+value.ID+"/restore", nil, adminCookie, adminSession.CSRFToken) + assertStatus(t, restored, http.StatusOK) + body, err := os.ReadFile(world) + if err != nil || string(body) != "world-v1" { + t.Fatalf("restored world=%q error=%v", body, err) + } + policy := jsonMethodRequest(t, handler, http.MethodPut, "/api/v1/instances/"+instanceID+"/backup-policy", []byte(`{"enabled":true,"cron_expression":"0 3 * * *","timezone":"Europe/Paris","retention_count":5}`), adminCookie, adminSession.CSRFToken) + assertStatus(t, policy, http.StatusOK) +} + func TestInstanceAuthorizationAndInstallationRequestWorkflow(t *testing.T) { ctx := context.Background() db, err := sqlite.Open(ctx, filepath.Join(t.TempDir(), "dogama.db")) diff --git a/migrations/0005_backups_imports.sql b/migrations/0005_backups_imports.sql new file mode 100644 index 0000000..ef98eb9 --- /dev/null +++ b/migrations/0005_backups_imports.sql @@ -0,0 +1,72 @@ +ALTER TABLE instance_operations RENAME TO instance_operations_v4; + +CREATE TABLE instance_operations ( + id TEXT PRIMARY KEY, + instance_id TEXT NOT NULL REFERENCES instances(id) ON DELETE RESTRICT, + kind TEXT NOT NULL CHECK (kind IN ('install', 'start', 'stop', 'restart', 'delete_container', 'reconcile', 'backup', 'restore', 'import')), + state TEXT NOT NULL CHECK (state IN ('running', 'succeeded', 'failed', 'intervention_required')), + phase TEXT NOT NULL, + error_code TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + completed_at TEXT +); + +INSERT INTO instance_operations(id, instance_id, kind, state, phase, error_code, created_at, updated_at, completed_at) +SELECT id, instance_id, kind, state, phase, error_code, created_at, updated_at, completed_at +FROM instance_operations_v4; + +DROP TABLE instance_operations_v4; + +CREATE UNIQUE INDEX instance_operation_active_idx ON instance_operations(instance_id) WHERE state = 'running'; +CREATE INDEX instance_operation_history_idx ON instance_operations(instance_id, created_at DESC); + +CREATE TABLE backup_policies ( + instance_id TEXT PRIMARY KEY REFERENCES instances(id) ON DELETE CASCADE, + enabled INTEGER NOT NULL DEFAULT 0 CHECK (enabled IN (0, 1)), + cron_expression TEXT, + timezone TEXT NOT NULL DEFAULT 'UTC', + retention_count INTEGER NOT NULL DEFAULT 7 CHECK (retention_count BETWEEN 1 AND 1000), + next_run_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE backups ( + id TEXT PRIMARY KEY, + instance_id TEXT NOT NULL REFERENCES instances(id) ON DELETE RESTRICT, + operation_id TEXT REFERENCES instance_operations(id) ON DELETE SET NULL, + origin TEXT NOT NULL CHECK (origin IN ('manual', 'scheduled', 'pre_update', 'pre_restore', 'idle_shutdown', 'imported', 'system')), + status TEXT NOT NULL CHECK (status IN ('creating', 'available', 'failed', 'deleted')), + relative_path TEXT, + size_bytes INTEGER CHECK (size_bytes IS NULL OR size_bytes >= 0), + sha256 TEXT CHECK (sha256 IS NULL OR length(sha256) = 64), + manifest_json TEXT, + error_code TEXT, + created_by TEXT REFERENCES users(id) ON DELETE SET NULL, + created_at TEXT NOT NULL, + completed_at TEXT, + deleted_at TEXT +); + +CREATE INDEX backups_instance_created_idx ON backups(instance_id, created_at DESC); +CREATE INDEX backups_retention_idx ON backups(instance_id, origin, status, created_at); + +CREATE TABLE imports ( + id TEXT PRIMARY KEY, + requested_by TEXT NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + instance_id TEXT REFERENCES instances(id) ON DELETE CASCADE, + status TEXT NOT NULL CHECK (status IN ('staging', 'validated', 'attached', 'failed', 'expired')), + format TEXT NOT NULL, + relative_stage_path TEXT NOT NULL, + detected_type TEXT, + confidence TEXT CHECK (confidence IS NULL OR confidence IN ('confirmed', 'probable', 'recognized_unknown_version', 'unrecognized')), + file_count INTEGER NOT NULL DEFAULT 0, + expanded_size_bytes INTEGER NOT NULL DEFAULT 0, + error_code TEXT, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + completed_at TEXT +); + +CREATE INDEX imports_expiry_idx ON imports(status, expires_at);