refactor(modules): load wasm modules from templates #43

Merged
tony merged 1 commits from codex/block-16-template-modules into main 2026-08-25 17:56:07 +02:00
30 changed files with 409 additions and 67 deletions
-1
View File
@@ -17,7 +17,6 @@ RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache
FROM gcr.io/distroless/static-debian12:latest@sha256:a9fcaedd4c9b59e12dd65d954f0b5044f19b0647a8a3712e77205df9e7b102cd AS dogama
WORKDIR /var/lib/dogama
COPY --from=build /out/dogama /usr/local/bin/dogama
COPY --from=build /src/modules /usr/share/dogama/modules
EXPOSE 8080
ENTRYPOINT ["/usr/local/bin/dogama"]
+1 -1
View File
@@ -128,7 +128,7 @@ Report reproducible problems in the [Gitea issue tracker](https://git.zaynet.fr/
## License
No repository-wide license file is currently present, so no general redistribution license is asserted here. The Palworld reference module has its own [license](modules/palworld-rest/LICENSE). A project-wide license must be added by the owner before public distribution.
No repository-wide license file is currently present, so no general redistribution license is asserted here. The Palworld reference module has its own [license](catalog/palworld/module/LICENSE). A project-wide license must be added by the owner before public distribution.
### Web access and languages
+1 -1
View File
@@ -5,5 +5,5 @@ import "embed"
// Files contains built-in templates and their packaged assets.
//
//go:embed */template.yaml */assets/*
//go:embed */template.yaml */assets/* */module/* */module/*/*
var Files embed.FS
+4 -1
View File
@@ -17,4 +17,7 @@ The upstream API may change. A template snapshot version remains independent fro
- The template ships separate local logo and horizontal artwork assets. The source URL and attribution remain in the template so the cached raster can be audited without loading third-party content in the UI.
- The schema's storage sizes are conservative product defaults because upstream specifies SSD performance but not a fixed disk-size requirement.
- Local hosted-world migration may require player identity conversion. The generic V1 importer detects structure and warns; it does not silently convert identities.
- The checked-in module manifest is a source example. It becomes installable only after `module.wasm` is built and its real SHA-256 replaces the all-zero placeholder.
- `module/` contains the complete Palworld REST adapter: its manifest, source,
fixture, license and compiled `module.wasm`. It is declared by
`template.yaml` and is loaded from the template snapshot, not a global module
registry. Rebuild it with the command in `module/README.md`.
@@ -31,8 +31,11 @@ From the repository root, using Go 1.25 or newer:
```sh
CGO_ENABLED=0 GOOS=wasip1 GOARCH=wasm go build \
-trimpath -buildmode=c-shared \
-o modules/palworld-rest/module.wasm ./modules/palworld-rest/src
sha256sum modules/palworld-rest/module.wasm
-o catalog/palworld/module/module.wasm ./catalog/palworld/module/src
sha256sum catalog/palworld/module/module.wasm
```
The checksum must exactly match `manifest.yaml`. Repository tests instantiate the real artifact under wazero and exercise it against a bounded fake Palworld transport.
The checksum must exactly match `manifest.yaml`. The Palworld template declares
`module/manifest.yaml`; DoGaMa discovers the artifact from that template-local
bundle without a game-specific registry. Repository tests instantiate the real
artifact under wazero and exercise it against a bounded fake Palworld transport.
+4 -1
View File
@@ -1,6 +1,6 @@
schema_version: 1
id: palworld-official
version: 1.1.2
version: 1.1.3
source:
type: official
@@ -186,6 +186,9 @@ integration:
port_id: rest_api
required: false
module:
path: module/manifest.yaml
backup:
strategy: online_save
source_mounts:
+1 -2
View File
@@ -46,7 +46,6 @@ func run(logger *slog.Logger) error {
databasePath := environment("DOGAMA_DATABASE_PATH", "dogama.db")
serversRoot := environment("DOGAMA_SERVERS_ROOT", "/srv/game-servers")
templatesRoot := environment("DOGAMA_TEMPLATES_ROOT", "/var/lib/dogama/templates")
modulesRoot := environment("DOGAMA_MODULES_ROOT", "/usr/share/dogama/modules")
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
@@ -119,7 +118,7 @@ func run(logger *slog.Logger) error {
}
handler, err = web.NewHandlerCompleteWithCatalogDeploymentAndRuntime(auth.New(db), repository, lifecycle, backupService, importService, auditService, notificationService, func(ctx context.Context) (catalog.ScanResult, error) {
return synchronizeCatalog(ctx, repository, templatesRoot)
}, serversRoot, instance.NewModuleService(repository, repository, modulesRoot), logger)
}, serversRoot, instance.NewModuleService(repository, repository), logger)
if err != nil {
return err
}
+1 -1
View File
@@ -58,7 +58,7 @@ Read this compact operational baseline before starting a milestone. Open detaile
- Label values support only the explicit allowlist in `internal/instance/container_config.go`; unknown variables are errors, not arbitrary templates.
- `{{game.icon_url}}` is the public icon for the game. `{{instance.slug}}` remains supported.
- Template game artwork contains separate required local `logo` and horizontal `image` assets; template validation rejects missing files. Deployment previews expose distinct logo and artwork URLs while retaining `icon_url` as a compatible logo alias. Palworld template `1.1.0` is the first snapshot with this contract.
- Embedded catalog validation is collection-based: every discovered template is schema- and cross-field-validated, including referenced assets, declared integration modules and ports/configuration. Template asset contents are not SHA-256-pinned, allowing administrator-maintained local assets while preserving required-path validation. A template `version` identifies a DoGaMa snapshot, whereas `container.tag` selects the game-server image; the official Palworld template follows Pocketpair's `latest` tag for new deployments and ordinary pulls.
- Embedded catalog validation is collection-based: every discovered template is schema- and cross-field-validated, including referenced assets, declared template-local integration modules and ports/configuration. A module bundle lives at `<template>/module/`, is declared by `module.path`, is path-confined and is retained with the selected snapshot; no game-specific internal registry exists. Template asset contents are not SHA-256-pinned, allowing administrator-maintained local assets while preserving required-path validation. A template `version` identifies a DoGaMa snapshot, whereas `container.tag` selects the game-server image; the official Palworld template follows Pocketpair's `latest` tag for new deployments and ordinary pulls.
- Instance slugs are derived from the display name, not canonical IDs. Accents are normalized to ASCII; whitespace, `/`, punctuation and special characters become safe hyphen separators; repeated and edge hyphens are removed.
- Docker user mode is fixed at creation to DoGaMa UID/GID, custom numeric UID/GID, or image-defined user. Never perform automatic recursive ownership changes.
- A pinned image tag is an explicit mutable tag, not an immutable digest. Tracked mode follows the template's declared default tag.
+1 -2
View File
@@ -57,7 +57,7 @@ Suggested container paths:
/var/lib/dogama/
dogama.db
catalog/
modules/
catalog/<game>/module/
imports/staging/
cache/
/srv/game-servers/<instance-slug>/
@@ -84,4 +84,3 @@ Paths stored in SQLite use stable instance and mount identifiers. User-supplied
Long-running operations are durable jobs in SQLite. A per-instance lock serializes mutually exclusive actions. Jobs use explicit phases and checkpoints so a restart can resume, retry safely or mark manual intervention required. UI requests enqueue work and return an operation identifier rather than keeping a long HTTP request open.
The scheduler is internal for V1 and handles cron backups, retention, audit purge, optional start/stop schedules, module/catalog update checks and queued notifications. Only one scheduler leader exists because V1 runs one main-application replica.
+21 -16
View File
@@ -10,17 +10,26 @@ Game-specific API <-> WebAssembly adapter <-> normalized DoGaMa API
It may query status, list players, request an in-game save, announce, shut down gracefully, kick, ban or unban when the game supports those operations. It does not own container lifecycle, files, users, backups, scheduling or UI.
## Package
## Template-local package
```text
palworld-rest-1.0.0.dogama-module/
module.wasm
manifest.yaml
README.md
LICENSE
catalog/palworld/
template.yaml
assets/
module/
manifest.yaml
module.wasm
src/
README.md
LICENSE
```
The installed artifact is content-addressed. Manifest, binary checksum, source/trust status and installation time are recorded. A package cannot contain executable helpers or dynamic libraries.
`template.yaml` declares the optional manifest as `module.path`. The path must
remain under the template root's `module/` directory; absolute paths, traversal
and local symbolic links are rejected. The validated module bundle contributes
to the template snapshot digest and is retained with that snapshot, so a later
local-template update cannot change a pinned instance's adapter. A package
cannot contain executable helpers or dynamic libraries.
## Runtime contract
@@ -62,16 +71,12 @@ Per call, enforce a deadline, instruction/fuel budget, memory ceiling, maximum h
## Independent versioning
Templates and modules have independent semantic versions.
- A template pins an acceptable module ID and version range.
- A manifest states supported manager module-API versions and game IDs.
- Installation of a new module does not activate it automatically for existing instances.
- Activation runs schema, checksum, ABI, capability and connection tests.
- The previous version stays available for rollback until the new version is healthy.
- A module update never silently changes instance settings or template snapshots.
The manifest retains its module ID and API compatibility contract, but discovery
is template-driven: there is no game-to-module registry in the application.
An administrator can copy/import `my-game/template.yaml`, `assets/` and
`module/` together. A module update is accepted on the next local scan and
creates a distinct digest; selected snapshots retain their own bundle.
## Prohibited behavior
Modules cannot create/delete containers, read SQLite, access host/game files, create backups, execute commands, manage users, expose routes or UI, contact other instances, or make unrestricted network calls. If a proposed integration needs those powers, the generic DoGaMa contract must be extended safely instead of bypassed.
+9 -5
View File
@@ -20,14 +20,19 @@ Create:
```text
catalog/<game>/template.yaml
modules/<module-id>/manifest.yaml
modules/<module-id>/README.md
modules/<module-id>/src/... implementation phase
modules/<module-id>/tests/...
catalog/<game>/module/manifest.yaml
catalog/<game>/module/README.md
catalog/<game>/module/src/... implementation phase
catalog/<game>/module/tests/...
```
Choose only required capabilities. Translate game errors into normalized errors. Use the logical `instance_api` host functions; never accept arbitrary destination URLs. Keep game API credentials as declared secret configuration.
Declare the optional bundle with `module.path: module/manifest.yaml` in the
same template. The path is confined to the template's root `module/`
directory. Local administrator templates use precisely this layout; no
application recompilation or internal module registration is involved.
## Required verification for a contribution
- Template validates against `specs/template.schema.json`.
@@ -53,4 +58,3 @@ Then implement the smallest valid surface:
## Contract-edit rule
If a new game cannot fit the current schema, first determine whether it exposes a genuinely generic need. Extend the schema narrowly with documentation, migration/compatibility analysis, validation, negative tests and updated examples. Never add an escape hatch such as arbitrary commands, raw Compose fragments, host paths or unrestricted network permissions.
+22 -1
View File
@@ -21,13 +21,34 @@ web/ embedded UI source
internal/persistence/sqlite/schema.sql embedded current SQLite schema
specs/ schemas and normalized contracts
catalog/ reference templates
modules/ reference modules and fixtures
catalog/<game>/module/ optional template-local WASM adapters and sources
docs/
tests/integration/
```
## Initial application development
## Template-local integration modules
An administrator may create or import a local template as a self-contained
directory:
```text
my-game/
template.yaml
assets/
module/ # optional
manifest.yaml
custom.wasm
```
When present, declare the manifest in `template.yaml` with
`module.path: module/manifest.yaml`. The module directory is confined to the
template root: absolute paths, traversal and symbolic links are rejected. The
manifest and WASM artifact remain subject to the generic manifest schema and
the capability-limited WASM sandbox. No rebuild of DoGaMa or internal game
registry entry is needed for a local module.
The main application requires Go 1.25. SQLite uses the pure-Go `modernc.org/sqlite` driver, so neither cgo nor a system SQLite development library is required. Standard Compose supplies all internal bootstrap contracts. The agent generates its shared token and the application generates its master key independently. Direct developer execution may override `DOGAMA_LISTEN_ADDRESS`, `DOGAMA_DATABASE_PATH`, `DOGAMA_AGENT_URL`, `DOGAMA_AGENT_TOKEN_FILE` and `DOGAMA_MASTER_KEY_FILE`; lifecycle routes remain disabled when both agent overrides are absent. These are development controls, not public deployment settings. Run it with:
```sh
+94
View File
@@ -100,3 +100,97 @@ func TestScanDirAcceptsLocallyModifiedReferencedAsset(t *testing.T) {
}
t.Skip("embedded catalog has no container assets")
}
func TestScanDirLoadsTemplateLocalModuleAndAcceptsChanges(t *testing.T) {
root := t.TempDir()
if err := catalog.InitializeOfficial(root, catalogdata.Files); err != nil {
t.Fatal(err)
}
first, err := catalog.ScanDir(root)
if err != nil || len(first.Valid) == 0 {
t.Fatalf("initial scan = %#v, %v", first, err)
}
var snapshot catalog.Snapshot
for _, candidate := range first.Valid {
if candidate.Template.Module != nil {
snapshot = candidate
break
}
}
if snapshot.Template.Module == nil || len(snapshot.ModuleFiles) == 0 {
t.Fatal("template-local module was not discovered")
}
wasmPath := filepath.Join(root, snapshot.AssetRoot, "module", "module.wasm")
body, err := os.ReadFile(wasmPath)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(wasmPath, append(body, 0), 0o644); err != nil {
t.Fatal(err)
}
second, err := catalog.ScanDir(root)
if err != nil || len(second.Errors) != 0 {
t.Fatalf("modified local module scan = %#v, %v", second, err)
}
var updated catalog.Snapshot
for _, candidate := range second.Valid {
if candidate.Template.ID == snapshot.Template.ID {
updated = candidate
break
}
}
if updated.Digest == snapshot.Digest {
t.Fatal("module change did not affect template snapshot digest")
}
}
func TestScanDirAcceptsLocalTemplateWithoutModule(t *testing.T) {
root := t.TempDir()
if err := catalog.InitializeOfficial(root, catalogdata.Files); err != nil {
t.Fatal(err)
}
result, err := catalog.ScanDir(root)
if err != nil {
t.Fatal(err)
}
for _, snapshot := range result.Valid {
if snapshot.Template.Module == nil {
return
}
}
t.Fatal("a template without an optional module was not accepted")
}
func TestScanDirRejectsMissingAndEscapingLocalModule(t *testing.T) {
for _, modulePath := range []string{"module/missing.yaml", "../outside.wasm", "/outside.wasm"} {
t.Run(modulePath, func(t *testing.T) {
root := t.TempDir()
if err := catalog.InitializeOfficial(root, catalogdata.Files); err != nil {
t.Fatal(err)
}
initial, err := catalog.ScanDir(root)
if err != nil {
t.Fatal(err)
}
var snapshot catalog.Snapshot
for _, candidate := range initial.Valid {
if candidate.Template.Module != nil {
snapshot = candidate
break
}
}
path := filepath.Join(root, snapshot.AssetRoot, "template.yaml")
body, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(strings.Replace(string(body), snapshot.Template.Module.Path, modulePath, 1)), 0o644); err != nil {
t.Fatal(err)
}
result, err := catalog.ScanDir(root)
if err != nil || len(result.Errors) != 1 || result.Errors[0].Template != snapshot.AssetRoot {
t.Fatalf("unsafe local module scan = %#v, %v", result, err)
}
})
}
}
+125 -2
View File
@@ -22,6 +22,8 @@ import (
)
const templateSchemaURL = "https://dogama.dev/schemas/template-v1.json"
const moduleManifestSchemaURL = "https://dogama.dev/schemas/module-manifest-v1.json"
const maxModuleBundleBytes = 16 << 20
// ValidationIssue points to one invalid field without exposing input secrets.
type ValidationIssue struct {
@@ -86,6 +88,9 @@ type Template struct {
ModuleID string `json:"module_id"`
PortID string `json:"port_id"`
} `json:"integration,omitempty"`
Module *struct {
Path string `json:"path"`
} `json:"module,omitempty"`
Backup struct {
Strategy string `json:"strategy"`
SourceMounts []string `json:"source_mounts"`
@@ -177,6 +182,7 @@ type Snapshot struct {
Digest string
Origin string
AssetRoot string
ModuleFiles map[string][]byte `json:"-"`
}
// LoadFS validates every template.yaml below root and returns stable snapshots.
@@ -320,14 +326,27 @@ func Validate(body []byte, assetRoot string, source fs.FS) (Snapshot, error) {
if len(issues) != 0 {
return Snapshot{}, &ValidationErrors{Issues: issues}
}
moduleFiles, moduleIssues := collectModuleFiles(template, assetRoot, source)
if len(moduleIssues) != 0 {
return Snapshot{}, &ValidationErrors{Issues: moduleIssues}
}
pretty, _ := json.MarshalIndent(raw, "", " ")
digest := sha256.Sum256(canonical)
digester := sha256.New()
_, _ = digester.Write(canonical)
for _, name := range sortedModuleFiles(moduleFiles) {
_, _ = digester.Write([]byte{0})
_, _ = digester.Write([]byte(name))
_, _ = digester.Write([]byte{0})
_, _ = digester.Write(moduleFiles[name])
}
digest := digester.Sum(nil)
return Snapshot{
Template: template,
CanonicalYAML: string(pretty) + "\n",
Digest: hex.EncodeToString(digest[:]),
Digest: hex.EncodeToString(digest),
Origin: template.Source.Type,
AssetRoot: assetRoot,
ModuleFiles: moduleFiles,
}, nil
}
@@ -353,6 +372,24 @@ func compileSchema() (*jsonschema.Schema, error) {
return schema, nil
}
func compileModuleManifestSchema() (*jsonschema.Schema, error) {
body, err := specs.Files.ReadFile("module-manifest.schema.json")
if err != nil {
return nil, err
}
compiler := jsonschema.NewCompiler()
compiler.AssertFormat()
compiler.UseRegexpEngine(compileECMAScript)
var document any
if err := json.Unmarshal(body, &document); err != nil {
return nil, err
}
if err := compiler.AddResource(moduleManifestSchemaURL, document); err != nil {
return nil, err
}
return compiler.Compile(moduleManifestSchemaURL)
}
type ecmaRegexp regexp2.Regexp
func (expression *ecmaRegexp) MatchString(value string) bool {
@@ -457,6 +494,9 @@ func crossValidate(template Template, assetRoot string, source fs.FS) []Validati
issues = append(issues, ValidationIssue{Path: "/integration/port_id", Message: "integration must reference an integration port"})
}
}
if template.Integration != nil && template.Module == nil {
issues = append(issues, ValidationIssue{Path: "/module", Message: "integration requires a template-local module"})
}
if template.Healthcheck.PortID != "" {
if _, exists := ports[template.Healthcheck.PortID]; !exists {
issues = append(issues, ValidationIssue{Path: "/healthcheck/port_id", Message: "healthcheck port does not exist"})
@@ -478,6 +518,89 @@ func crossValidate(template Template, assetRoot string, source fs.FS) []Validati
return issues
}
func collectModuleFiles(template Template, assetRoot string, source fs.FS) (map[string][]byte, []ValidationIssue) {
if template.Module == nil {
return nil, nil
}
if !validModulePath(template.Module.Path) {
return nil, []ValidationIssue{{Path: "/module/path", Message: "module path must remain inside the template module directory"}}
}
root := path.Join(assetRoot, "module")
files := map[string][]byte{}
var totalBytes int
err := fs.WalkDir(source, root, func(name string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
if entry.Type()&fs.ModeSymlink != 0 {
return errors.New("symbolic link")
}
body, err := fs.ReadFile(source, name)
if err != nil {
return err
}
totalBytes += len(body)
if totalBytes > maxModuleBundleBytes {
return errors.New("module bundle exceeds size limit")
}
rel := strings.TrimPrefix(name, strings.TrimSuffix(assetRoot, "/")+"/")
if !strings.HasPrefix(rel, "module/") {
return errors.New("invalid module path")
}
files[rel] = body
return nil
})
if err != nil {
return nil, []ValidationIssue{{Path: "/module", Message: "module directory cannot be read"}}
}
if _, ok := files[template.Module.Path]; !ok {
return nil, []ValidationIssue{{Path: "/module/path", Message: "declared module manifest is missing"}}
}
if err := validateModuleManifest(files[template.Module.Path], files); err != nil {
return nil, []ValidationIssue{{Path: "/module", Message: "declared module bundle is invalid"}}
}
return files, nil
}
func validateModuleManifest(body []byte, files map[string][]byte) error {
var raw any
if err := yaml.Unmarshal(body, &raw); err != nil {
return err
}
schema, err := compileModuleManifestSchema()
if err != nil || schema.Validate(raw) != nil {
return errors.New("invalid manifest")
}
var manifest struct {
Artifacts struct {
WASM string `yaml:"wasm"`
} `yaml:"artifacts"`
}
if err := yaml.Unmarshal(body, &manifest); err != nil || manifest.Artifacts.WASM == "" {
return errors.New("invalid artifact")
}
if _, ok := files["module/"+manifest.Artifacts.WASM]; !ok {
return errors.New("missing wasm artifact")
}
return nil
}
func validModulePath(value string) bool {
return value != "" && !path.IsAbs(value) && path.Clean(value) == value && strings.HasPrefix(value, "module/") && !strings.Contains(value, "..")
}
func sortedModuleFiles(files map[string][]byte) []string {
names := make([]string, 0, len(files))
for name := range files {
names = append(names, name)
}
sort.Strings(names)
return names
}
func hasAlias(node *yaml.Node) bool {
if node.Kind == yaml.AliasNode {
return true
+32 -6
View File
@@ -3,9 +3,7 @@ package catalog_test
import (
"errors"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"testing"
@@ -59,15 +57,43 @@ func TestBuiltInCatalogValidatesEveryDiscoveredTemplate(t *testing.T) {
t.Fatalf("template path=%q id=%q asset=%q is missing: %v", name, snapshot.Template.ID, assetPath, statErr)
}
}
if snapshot.Template.Integration != nil {
manifest := filepath.Join("..", "..", "modules", snapshot.Template.Integration.ModuleID, "manifest.yaml")
if _, statErr := os.Stat(manifest); statErr != nil {
t.Fatalf("template path=%q id=%q declared module=%q is missing at %q: %v", name, snapshot.Template.ID, snapshot.Template.Integration.ModuleID, manifest, statErr)
if snapshot.Template.Module != nil {
if _, ok := snapshot.ModuleFiles[snapshot.Template.Module.Path]; !ok {
t.Fatalf("template path=%q id=%q declared module=%q is missing", name, snapshot.Template.ID, snapshot.Template.Module.Path)
}
}
}
}
func TestCrossValidationRejectsMissingDeclaredModule(t *testing.T) {
name, body, snapshot := embeddedTemplate(t)
if snapshot.Template.Module == nil {
t.Skip("embedded fixture has no module")
}
body = []byte(strings.Replace(string(body), snapshot.Template.Module.Path, "module/missing.yaml", 1))
_, err := catalog.Validate(body, path.Dir(name), catalogdata.Files)
var validation *catalog.ValidationErrors
if !errors.As(err, &validation) || len(validation.Issues) == 0 || validation.Issues[0].Path != "/module/path" {
t.Fatalf("validation error = %#v", err)
}
}
func TestCrossValidationRejectsModuleTraversalAndAbsolutePath(t *testing.T) {
name, body, snapshot := embeddedTemplate(t)
if snapshot.Template.Module == nil {
t.Skip("embedded fixture has no module")
}
for _, modulePath := range []string{"../outside.wasm", "/outside.wasm"} {
t.Run(modulePath, func(t *testing.T) {
invalid := []byte(strings.Replace(string(body), snapshot.Template.Module.Path, modulePath, 1))
_, err := catalog.Validate(invalid, path.Dir(name), catalogdata.Files)
if err == nil {
t.Fatalf("unsafe module path accepted: %q", modulePath)
}
})
}
}
func TestBuiltInCatalogIsDeterministic(t *testing.T) {
first, err := catalog.LoadFS(catalogdata.Files, ".")
if err != nil {
+13 -11
View File
@@ -8,8 +8,6 @@ import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
@@ -23,11 +21,10 @@ var ErrModuleUnavailable = errors.New("module unavailable")
type ModuleService struct {
secrets SecretRepository
catalog catalog.Repository
root string
}
func NewModuleService(secrets SecretRepository, repository catalog.Repository, root string) *ModuleService {
return &ModuleService{secrets: secrets, catalog: repository, root: filepath.Clean(root)}
func NewModuleService(secrets SecretRepository, repository catalog.Repository) *ModuleService {
return &ModuleService{secrets: secrets, catalog: repository}
}
type ServerInfo struct {
@@ -105,11 +102,11 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
return nil, moduleManifest{}, ErrModuleUnavailable
}
integration := snapshot.Template.Integration
if integration.ModuleID == "" || strings.Contains(integration.ModuleID, "/") || strings.Contains(integration.ModuleID, "..") {
if integration.ModuleID == "" || snapshot.Template.Module == nil || !validTemplateModulePath(snapshot.Template.Module.Path) {
return nil, moduleManifest{}, ErrModuleUnavailable
}
body, err := os.ReadFile(filepath.Join(s.root, integration.ModuleID, "manifest.yaml"))
if err != nil {
body, ok := snapshot.ModuleFiles[snapshot.Template.Module.Path]
if !ok {
return nil, moduleManifest{}, ErrModuleUnavailable
}
var manifest moduleManifest
@@ -134,7 +131,7 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
return nil, moduleManifest{}, ErrModuleUnavailable
}
}
if len(methods) == 0 || manifest.Artifacts.WASM == "" || filepath.Base(manifest.Artifacts.WASM) != manifest.Artifacts.WASM {
if len(methods) == 0 || manifest.Artifacts.WASM == "" || strings.Contains(manifest.Artifacts.WASM, "/") || strings.Contains(manifest.Artifacts.WASM, "..") {
return nil, moduleManifest{}, ErrModuleUnavailable
}
config, secrets := map[string]string{}, map[string]string{}
@@ -154,8 +151,8 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
config[field.ID] = v
}
}
wasm, err := os.ReadFile(filepath.Join(s.root, integration.ModuleID, manifest.Artifacts.WASM))
if err != nil {
wasm, ok := snapshot.ModuleFiles["module/"+manifest.Artifacts.WASM]
if !ok {
return nil, moduleManifest{}, ErrModuleUnavailable
}
r, err := module.New(ctx, wasm, manifest.Artifacts.SHA256, manifest.Capabilities, module.Limits{MemoryMB: manifest.Limits.MemoryMB, Timeout: durationMS(manifest.Limits.TimeoutMS), MaxResponseBytes: manifest.Limits.MaxResponseBytes, MaxConcurrentCall: manifest.Limits.MaxConcurrentCalls}, module.Binding{InstanceID: value.ID, ContainerPort: port, AllowedMethods: methods, Configuration: config, Secrets: secrets})
@@ -166,6 +163,11 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
}
func durationMS(v int) time.Duration { return time.Duration(v) * time.Millisecond }
func validTemplateModulePath(value string) bool {
return strings.HasPrefix(value, "module/") && !strings.Contains(value, "..") && !strings.HasPrefix(value, "/")
}
func contains(values []string, needle string) bool {
for _, v := range values {
if v == needle {
+2 -2
View File
@@ -29,7 +29,7 @@ func TestValidRelativeAPIPath(t *testing.T) {
}
func TestPalworldAdapterReportsBoundedFailures(t *testing.T) {
wasm, err := os.ReadFile("../../modules/palworld-rest/module.wasm")
wasm, err := os.ReadFile("../../catalog/palworld/module/module.wasm")
if err != nil {
t.Fatal(err)
}
@@ -75,7 +75,7 @@ func TestPalworldAdapterReportsBoundedFailures(t *testing.T) {
}
func TestPalworldAdapterExecutesInSandbox(t *testing.T) {
wasm, err := os.ReadFile("../../modules/palworld-rest/module.wasm")
wasm, err := os.ReadFile("../../catalog/palworld/module/module.wasm")
if err != nil {
t.Fatal(err)
}
+15 -6
View File
@@ -122,6 +122,10 @@ func (r *Repository) Replace(ctx context.Context, snapshots []catalog.Snapshot)
now := r.now().UTC().Format(time.RFC3339Nano)
ids := make([]string, 0, len(snapshots))
for _, snapshot := range snapshots {
moduleBundle, marshalErr := json.Marshal(snapshot.ModuleFiles)
if marshalErr != nil {
return fmt.Errorf("encode template module bundle: %w", marshalErr)
}
ids = append(ids, snapshot.Template.ID)
_, err = tx.ExecContext(ctx, `INSERT INTO templates(id, origin, trust_status, active_version, available, created_at, updated_at)
VALUES (?, ?, ?, ?, 1, ?, ?)
@@ -130,9 +134,9 @@ func (r *Repository) Replace(ctx context.Context, snapshots []catalog.Snapshot)
if err != nil {
return fmt.Errorf("upsert catalog template: %w", err)
}
_, err = tx.ExecContext(ctx, `INSERT INTO template_versions(template_id, version, schema_version, canonical_yaml, digest, game_id, game_name, description, image, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(template_id, version) DO UPDATE SET schema_version=excluded.schema_version, canonical_yaml=excluded.canonical_yaml, digest=excluded.digest, game_id=excluded.game_id, game_name=excluded.game_name, description=excluded.description, image=excluded.image`, snapshot.Template.ID, snapshot.Template.Version, snapshot.Template.SchemaVersion, snapshot.CanonicalYAML, snapshot.Digest, snapshot.Template.Game.ID, snapshot.Template.Game.Name, snapshot.Template.Game.Description, snapshot.Template.Game.Artwork.Image, now)
_, err = tx.ExecContext(ctx, `INSERT INTO template_versions(template_id, version, schema_version, canonical_yaml, digest, game_id, game_name, description, image, module_bundle, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(template_id, version) DO UPDATE SET schema_version=excluded.schema_version, canonical_yaml=excluded.canonical_yaml, digest=excluded.digest, game_id=excluded.game_id, game_name=excluded.game_name, description=excluded.description, image=excluded.image, module_bundle=excluded.module_bundle`, snapshot.Template.ID, snapshot.Template.Version, snapshot.Template.SchemaVersion, snapshot.CanonicalYAML, snapshot.Digest, snapshot.Template.Game.ID, snapshot.Template.Game.Name, snapshot.Template.Game.Description, snapshot.Template.Game.Artwork.Image, moduleBundle, now)
if err != nil {
return fmt.Errorf("upsert template snapshot: %w", err)
}
@@ -177,8 +181,9 @@ func (r *Repository) List(ctx context.Context) ([]catalog.Summary, error) {
func (r *Repository) Get(ctx context.Context, id, version string) (catalog.Snapshot, error) {
var canonical, digest, origin string
err := r.db.QueryRowContext(ctx, `SELECT v.canonical_yaml, v.digest, t.origin FROM template_versions v JOIN templates t ON t.id=v.template_id
WHERE v.template_id=? AND v.version=?`, id, version).Scan(&canonical, &digest, &origin)
var moduleBundle []byte
err := r.db.QueryRowContext(ctx, `SELECT v.canonical_yaml, v.digest, t.origin, v.module_bundle FROM template_versions v JOIN templates t ON t.id=v.template_id
WHERE v.template_id=? AND v.version=?`, id, version).Scan(&canonical, &digest, &origin, &moduleBundle)
if errors.Is(err, sql.ErrNoRows) {
return catalog.Snapshot{}, catalog.ErrTemplateNotFound
}
@@ -189,7 +194,11 @@ func (r *Repository) Get(ctx context.Context, id, version string) (catalog.Snaps
if err := json.Unmarshal([]byte(canonical), &template); err != nil {
return catalog.Snapshot{}, fmt.Errorf("decode stored template snapshot: %w", err)
}
return catalog.Snapshot{Template: template, CanonicalYAML: canonical, Digest: digest, Origin: origin}, nil
var moduleFiles map[string][]byte
if len(moduleBundle) != 0 && json.Unmarshal(moduleBundle, &moduleFiles) != nil {
return catalog.Snapshot{}, errors.New("decode stored template module bundle")
}
return catalog.Snapshot{Template: template, CanonicalYAML: canonical, Digest: digest, Origin: origin, ModuleFiles: moduleFiles}, nil
}
func (r *Repository) CreateDraft(ctx context.Context, draft instance.Draft) error {
@@ -80,6 +80,9 @@ func TestCatalogSyncIsImmutableAndDraftPinsSnapshot(t *testing.T) {
if err != nil || loaded.Digest != palworld.Digest {
t.Fatalf("loaded snapshot = %#v, error = %v", loaded, err)
}
if len(loaded.ModuleFiles) == 0 || loaded.Template.Module == nil || loaded.ModuleFiles[loaded.Template.Module.Path] == nil {
t.Fatalf("template-local module bundle was not retained: %#v", loaded.Template.Module)
}
tampered := palworld
tampered.Digest = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+1
View File
@@ -53,6 +53,7 @@ CREATE TABLE template_versions (
game_name TEXT NOT NULL,
description TEXT NOT NULL,
image TEXT NOT NULL,
module_bundle BLOB,
created_at TEXT NOT NULL,
PRIMARY KEY (template_id, version),
UNIQUE (digest)
+36 -2
View File
@@ -57,12 +57,46 @@ func initialize(ctx context.Context, db *sql.DB) error {
return fmt.Errorf("inspect sqlite schema: %w", err)
}
if existing == 1 {
return ensureDiagnosticSchema(ctx, db)
if err := ensureDiagnosticSchema(ctx, db); err != nil {
return err
}
return ensureTemplateModuleSchema(ctx, db)
}
if _, err := db.ExecContext(ctx, schema); err != nil {
return fmt.Errorf("initialize sqlite schema: %w", err)
}
return ensureDiagnosticSchema(ctx, db)
if err := ensureDiagnosticSchema(ctx, db); err != nil {
return err
}
return ensureTemplateModuleSchema(ctx, db)
}
// ensureTemplateModuleSchema preserves the complete module bundle with an
// immutable template snapshot for stores created before template-local modules.
func ensureTemplateModuleSchema(ctx context.Context, db *sql.DB) error {
rows, err := db.QueryContext(ctx, "PRAGMA table_info(template_versions)")
if err != nil {
return fmt.Errorf("inspect template module schema: %w", err)
}
defer rows.Close()
for rows.Next() {
var cid, notNull, primaryKey int
var name, typ string
var defaultValue any
if err := rows.Scan(&cid, &name, &typ, &notNull, &defaultValue, &primaryKey); err != nil {
return err
}
if name == "module_bundle" {
return nil
}
}
if err := rows.Err(); err != nil {
return err
}
if _, err := db.ExecContext(ctx, "ALTER TABLE template_versions ADD COLUMN module_bundle BLOB"); err != nil {
return fmt.Errorf("migrate template module schema: %w", err)
}
return nil
}
// This additive migration is safe for existing V1 databases and keeps the
+1 -1
View File
@@ -206,7 +206,7 @@ func TestInstanceUnbanCapabilityFallback(t *testing.T) {
})
t.Run("palworld manual fallback validates and audits", func(t *testing.T) {
manifest, err := os.ReadFile("../../modules/palworld-rest/manifest.yaml")
manifest, err := os.ReadFile("../../catalog/palworld/module/manifest.yaml")
if err != nil {
t.Fatal(err)
}
+8
View File
@@ -149,6 +149,14 @@
"required": { "type": "boolean", "default": false }
}
},
"module": {
"type": "object",
"additionalProperties": false,
"required": ["path"],
"properties": {
"path": { "type": "string", "pattern": "^module/(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+\\.yaml$", "maxLength": 240 }
}
},
"backup": {
"type": "object",
"additionalProperties": false,
+8 -2
View File
@@ -85,7 +85,7 @@ def validate_cross_references(template: dict, manifest: dict | None, template_pa
return warnings
wasm_digest = manifest["artifacts"]["sha256"]
wasm_path = ROOT / "modules" / manifest["id"] / manifest["artifacts"]["wasm"]
wasm_path = template_path.parent / "module" / manifest["artifacts"]["wasm"]
if wasm_digest == "0" * 64 and not wasm_path.exists():
warnings.append("Palworld module is a source specification: module.wasm and its final checksum are intentionally pending.")
elif wasm_path.is_file():
@@ -110,8 +110,14 @@ def validate_catalog() -> list[str]:
identities.add(identity)
integration = template.get("integration")
manifest = None
module = template.get("module")
if integration:
manifest_path = ROOT / "modules" / integration["module_id"] / "manifest.yaml"
assert module is not None, "Integration requires a template-local module"
if module:
module_path = Path(module["path"])
assert not module_path.is_absolute() and ".." not in module_path.parts, "Module path escapes template"
manifest_path = (template_path.parent / module_path).resolve()
assert manifest_path.parent.is_relative_to((template_path.parent / "module").resolve()), "Module path is outside template module directory"
assert manifest_path.is_file(), f"Declared module manifest is missing: {manifest_path}"
manifest = validate(manifest_schema, manifest_path)
warnings.extend(validate_cross_references(template, manifest, template_path))