358 lines
9.7 KiB
Go
358 lines
9.7 KiB
Go
// Package importexport validates untrusted game-data archives in isolated staging.
|
|
package importexport
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"errors"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
var (
|
|
ErrInvalidInput = errors.New("invalid import input")
|
|
ErrUnsafeArchive = errors.New("unsafe import archive")
|
|
ErrLimitExceeded = errors.New("import limit exceeded")
|
|
ErrNotRecognized = errors.New("import layout not recognized")
|
|
)
|
|
|
|
const (
|
|
maxFiles = 100000
|
|
maxDepth = 20
|
|
)
|
|
|
|
type Policy struct {
|
|
AcceptedFormats []string
|
|
MaxExpandedBytes int64
|
|
RequiredPaths []string
|
|
}
|
|
|
|
type Import struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
Format string `json:"format"`
|
|
DetectedType string `json:"detected_type,omitempty"`
|
|
Confidence string `json:"confidence,omitempty"`
|
|
FileCount int `json:"file_count"`
|
|
ExpandedSizeBytes int64 `json:"expanded_size_bytes"`
|
|
ExpiresAt string `json:"expires_at"`
|
|
RelativeStagePath string `json:"-"`
|
|
}
|
|
|
|
type Repository interface {
|
|
BeginImport(context.Context, Import, string) error
|
|
CompleteImport(context.Context, Import) error
|
|
FailImport(context.Context, string, string) error
|
|
ExpireImports(context.Context, string) ([]string, error)
|
|
}
|
|
|
|
func (s *Service) CleanupExpired(ctx context.Context) error {
|
|
paths, err := s.repository.ExpireImports(ctx, s.now().UTC().Format(time.RFC3339Nano))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, relative := range paths {
|
|
if !fs.ValidPath(relative) {
|
|
return ErrUnsafeArchive
|
|
}
|
|
target := filepath.Join(s.root, filepath.FromSlash(relative))
|
|
if !withinRoot(s.root, target) {
|
|
return ErrUnsafeArchive
|
|
}
|
|
if err := os.RemoveAll(target); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type Service struct {
|
|
repository Repository
|
|
root string
|
|
now func() time.Time
|
|
}
|
|
|
|
func New(repository Repository, root string) (*Service, error) {
|
|
if root == "" {
|
|
return nil, ErrInvalidInput
|
|
}
|
|
absolute, err := filepath.Abs(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.MkdirAll(absolute, 0o750); err != nil {
|
|
return nil, err
|
|
}
|
|
canonical, err := filepath.EvalSymlinks(absolute)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Service{repository: repository, root: canonical, now: time.Now}, nil
|
|
}
|
|
|
|
func (s *Service) Stage(ctx context.Context, actorID, format string, source io.Reader, policy Policy) (Import, error) {
|
|
if actorID == "" || !contains(policy.AcceptedFormats, format) || policy.MaxExpandedBytes < 1 {
|
|
return Import{}, ErrInvalidInput
|
|
}
|
|
id := importToken()
|
|
if id == "" {
|
|
return Import{}, errors.New("generate import identifier")
|
|
}
|
|
directory := filepath.Join(s.root, id)
|
|
if !withinRoot(s.root, directory) {
|
|
return Import{}, ErrUnsafeArchive
|
|
}
|
|
if err := os.Mkdir(directory, 0o750); err != nil {
|
|
return Import{}, err
|
|
}
|
|
value := Import{ID: id, Status: "staging", Format: format, RelativeStagePath: id, ExpiresAt: s.now().Add(24 * time.Hour).UTC().Format(time.RFC3339Nano)}
|
|
if err := s.repository.BeginImport(ctx, value, actorID); err != nil {
|
|
_ = os.RemoveAll(directory)
|
|
return Import{}, err
|
|
}
|
|
fail := func(code string, err error) (Import, error) {
|
|
_ = s.repository.FailImport(ctx, id, code)
|
|
_ = os.RemoveAll(directory)
|
|
return Import{}, err
|
|
}
|
|
upload := filepath.Join(directory, "upload")
|
|
file, err := os.OpenFile(upload, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
|
if err != nil {
|
|
return fail("import_stage_failed", err)
|
|
}
|
|
maxUpload := policy.MaxExpandedBytes
|
|
written, copyErr := io.Copy(file, io.LimitReader(source, maxUpload+1))
|
|
closeErr := file.Close()
|
|
if copyErr != nil || closeErr != nil {
|
|
if copyErr == nil {
|
|
copyErr = closeErr
|
|
}
|
|
return fail("import_upload_failed", copyErr)
|
|
}
|
|
if written > maxUpload {
|
|
return fail("import_upload_limit", ErrLimitExceeded)
|
|
}
|
|
extracted := filepath.Join(directory, "data")
|
|
if err := os.Mkdir(extracted, 0o750); err != nil {
|
|
return fail("import_stage_failed", err)
|
|
}
|
|
files, size, paths, err := extract(ctx, upload, extracted, format, policy.MaxExpandedBytes)
|
|
if err != nil {
|
|
return fail("import_validation_failed", err)
|
|
}
|
|
if !requiredPresent(paths, policy.RequiredPaths) {
|
|
return fail("import_layout_unrecognized", ErrNotRecognized)
|
|
}
|
|
if err := os.Remove(upload); err != nil {
|
|
return fail("import_cleanup_failed", err)
|
|
}
|
|
value.Status, value.DetectedType, value.Confidence, value.FileCount, value.ExpandedSizeBytes = "validated", "game_save", "confirmed", files, size
|
|
if err := s.repository.CompleteImport(ctx, value); err != nil {
|
|
return fail("import_persist_failed", err)
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func extract(ctx context.Context, archive, destination, format string, limit int64) (int, int64, []string, error) {
|
|
switch format {
|
|
case "zip":
|
|
return extractZIP(ctx, archive, destination, limit)
|
|
case "tar", "tar.gz", "tar.zst":
|
|
return extractTar(ctx, archive, destination, format, limit)
|
|
default:
|
|
return 0, 0, nil, ErrInvalidInput
|
|
}
|
|
}
|
|
|
|
func extractZIP(ctx context.Context, archive, destination string, limit int64) (int, int64, []string, error) {
|
|
reader, err := zip.OpenReader(archive)
|
|
if err != nil {
|
|
return 0, 0, nil, ErrUnsafeArchive
|
|
}
|
|
defer reader.Close()
|
|
var count int
|
|
var total int64
|
|
var paths []string
|
|
for _, entry := range reader.File {
|
|
select {
|
|
case <-ctx.Done():
|
|
return 0, 0, nil, ctx.Err()
|
|
default:
|
|
}
|
|
name, err := safeName(entry.Name)
|
|
if err != nil || entry.Mode()&os.ModeSymlink != 0 || (!entry.Mode().IsRegular() && !entry.FileInfo().IsDir()) {
|
|
return 0, 0, nil, ErrUnsafeArchive
|
|
}
|
|
if entry.FileInfo().IsDir() {
|
|
if err := os.MkdirAll(filepath.Join(destination, filepath.FromSlash(name)), 0o750); err != nil {
|
|
return 0, 0, nil, err
|
|
}
|
|
continue
|
|
}
|
|
count++
|
|
total += int64(entry.UncompressedSize64)
|
|
if count > maxFiles || total > limit {
|
|
return 0, 0, nil, ErrLimitExceeded
|
|
}
|
|
input, err := entry.Open()
|
|
if err != nil {
|
|
return 0, 0, nil, err
|
|
}
|
|
if err := writeExtracted(destination, name, input, int64(entry.UncompressedSize64)); err != nil {
|
|
input.Close()
|
|
return 0, 0, nil, err
|
|
}
|
|
if err := input.Close(); err != nil {
|
|
return 0, 0, nil, err
|
|
}
|
|
paths = append(paths, name)
|
|
}
|
|
return count, total, paths, nil
|
|
}
|
|
|
|
func extractTar(ctx context.Context, archive, destination, format string, limit int64) (int, int64, []string, error) {
|
|
file, err := os.Open(archive)
|
|
if err != nil {
|
|
return 0, 0, nil, err
|
|
}
|
|
defer file.Close()
|
|
var source io.Reader = file
|
|
var closer io.Closer
|
|
if format == "tar.gz" {
|
|
value, err := gzip.NewReader(file)
|
|
if err != nil {
|
|
return 0, 0, nil, ErrUnsafeArchive
|
|
}
|
|
source, closer = value, value
|
|
}
|
|
if format == "tar.zst" {
|
|
value, err := zstd.NewReader(file, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(2<<30))
|
|
if err != nil {
|
|
return 0, 0, nil, ErrUnsafeArchive
|
|
}
|
|
source, closer = value, value.IOReadCloser()
|
|
}
|
|
if closer != nil {
|
|
defer closer.Close()
|
|
}
|
|
reader := tar.NewReader(source)
|
|
var count int
|
|
var total int64
|
|
var paths []string
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return 0, 0, nil, ctx.Err()
|
|
default:
|
|
}
|
|
header, err := reader.Next()
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return 0, 0, nil, ErrUnsafeArchive
|
|
}
|
|
name, err := safeName(header.Name)
|
|
if err != nil || header.Linkname != "" {
|
|
return 0, 0, nil, ErrUnsafeArchive
|
|
}
|
|
if header.Typeflag == tar.TypeDir {
|
|
if err := os.MkdirAll(filepath.Join(destination, filepath.FromSlash(name)), 0o750); err != nil {
|
|
return 0, 0, nil, err
|
|
}
|
|
continue
|
|
}
|
|
if !header.FileInfo().Mode().IsRegular() {
|
|
return 0, 0, nil, ErrUnsafeArchive
|
|
}
|
|
count++
|
|
total += header.Size
|
|
if count > maxFiles || total > limit {
|
|
return 0, 0, nil, ErrLimitExceeded
|
|
}
|
|
if err := writeExtracted(destination, name, reader, header.Size); err != nil {
|
|
return 0, 0, nil, err
|
|
}
|
|
paths = append(paths, name)
|
|
}
|
|
return count, total, paths, nil
|
|
}
|
|
|
|
func writeExtracted(root, name string, source io.Reader, size int64) error {
|
|
target := filepath.Join(root, filepath.FromSlash(name))
|
|
if !withinRoot(root, target) {
|
|
return ErrUnsafeArchive
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
|
|
return err
|
|
}
|
|
file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
written, copyErr := io.CopyN(file, source, size)
|
|
closeErr := file.Close()
|
|
if copyErr != nil || written != size {
|
|
return ErrUnsafeArchive
|
|
}
|
|
return closeErr
|
|
}
|
|
|
|
func safeName(value string) (string, error) {
|
|
if strings.Contains(value, "\\") || strings.ContainsRune(value, 0) || strings.HasPrefix(value, "/") {
|
|
return "", ErrUnsafeArchive
|
|
}
|
|
clean := path.Clean(value)
|
|
if clean == "." || !fs.ValidPath(clean) || strings.Contains(strings.Split(clean, "/")[0], ":") || len(strings.Split(clean, "/")) > maxDepth {
|
|
return "", ErrUnsafeArchive
|
|
}
|
|
return clean, nil
|
|
}
|
|
|
|
func requiredPresent(paths, required []string) bool {
|
|
for _, expected := range required {
|
|
found := false
|
|
for _, candidate := range paths {
|
|
if candidate == expected || strings.HasSuffix(candidate, "/"+expected) || strings.HasPrefix(candidate, expected+"/") || strings.Contains(candidate, "/"+expected+"/") {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func contains(values []string, expected string) bool {
|
|
for _, value := range values {
|
|
if value == expected {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func withinRoot(root, candidate string) bool {
|
|
relative, err := filepath.Rel(root, candidate)
|
|
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
|
}
|
|
func importToken() string {
|
|
value := make([]byte, 24)
|
|
if _, err := rand.Read(value); err != nil {
|
|
return ""
|
|
}
|
|
return base64.RawURLEncoding.EncodeToString(value)
|
|
}
|