refactor(modules): load wasm modules from templates
CI / validate (pull_request) Canceled after 3m32s
CI / validate (pull_request) Canceled after 3m32s
This commit is contained in:
@@ -17,7 +17,6 @@ RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache
|
||||
FROM gcr.io/distroless/static-debian12:latest@sha256:a9fcaedd4c9b59e12dd65d954f0b5044f19b0647a8a3712e77205df9e7b102cd AS dogama
|
||||
WORKDIR /var/lib/dogama
|
||||
COPY --from=build /out/dogama /usr/local/bin/dogama
|
||||
COPY --from=build /src/modules /usr/share/dogama/modules
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/usr/local/bin/dogama"]
|
||||
|
||||
|
||||
@@ -128,7 +128,7 @@ Report reproducible problems in the [Gitea issue tracker](https://git.zaynet.fr/
|
||||
|
||||
## License
|
||||
|
||||
No repository-wide license file is currently present, so no general redistribution license is asserted here. The Palworld reference module has its own [license](modules/palworld-rest/LICENSE). A project-wide license must be added by the owner before public distribution.
|
||||
No repository-wide license file is currently present, so no general redistribution license is asserted here. The Palworld reference module has its own [license](catalog/palworld/module/LICENSE). A project-wide license must be added by the owner before public distribution.
|
||||
|
||||
### Web access and languages
|
||||
|
||||
|
||||
+1
-1
@@ -5,5 +5,5 @@ import "embed"
|
||||
|
||||
// Files contains built-in templates and their packaged assets.
|
||||
//
|
||||
//go:embed */template.yaml */assets/*
|
||||
//go:embed */template.yaml */assets/* */module/* */module/*/*
|
||||
var Files embed.FS
|
||||
|
||||
@@ -17,4 +17,7 @@ The upstream API may change. A template snapshot version remains independent fro
|
||||
- The template ships separate local logo and horizontal artwork assets. The source URL and attribution remain in the template so the cached raster can be audited without loading third-party content in the UI.
|
||||
- The schema's storage sizes are conservative product defaults because upstream specifies SSD performance but not a fixed disk-size requirement.
|
||||
- Local hosted-world migration may require player identity conversion. The generic V1 importer detects structure and warns; it does not silently convert identities.
|
||||
- The checked-in module manifest is a source example. It becomes installable only after `module.wasm` is built and its real SHA-256 replaces the all-zero placeholder.
|
||||
- `module/` contains the complete Palworld REST adapter: its manifest, source,
|
||||
fixture, license and compiled `module.wasm`. It is declared by
|
||||
`template.yaml` and is loaded from the template snapshot, not a global module
|
||||
registry. Rebuild it with the command in `module/README.md`.
|
||||
|
||||
@@ -31,8 +31,11 @@ From the repository root, using Go 1.25 or newer:
|
||||
```sh
|
||||
CGO_ENABLED=0 GOOS=wasip1 GOARCH=wasm go build \
|
||||
-trimpath -buildmode=c-shared \
|
||||
-o modules/palworld-rest/module.wasm ./modules/palworld-rest/src
|
||||
sha256sum modules/palworld-rest/module.wasm
|
||||
-o catalog/palworld/module/module.wasm ./catalog/palworld/module/src
|
||||
sha256sum catalog/palworld/module/module.wasm
|
||||
```
|
||||
|
||||
The checksum must exactly match `manifest.yaml`. Repository tests instantiate the real artifact under wazero and exercise it against a bounded fake Palworld transport.
|
||||
The checksum must exactly match `manifest.yaml`. The Palworld template declares
|
||||
`module/manifest.yaml`; DoGaMa discovers the artifact from that template-local
|
||||
bundle without a game-specific registry. Repository tests instantiate the real
|
||||
artifact under wazero and exercise it against a bounded fake Palworld transport.
|
||||
@@ -1,6 +1,6 @@
|
||||
schema_version: 1
|
||||
id: palworld-official
|
||||
version: 1.1.2
|
||||
version: 1.1.3
|
||||
|
||||
source:
|
||||
type: official
|
||||
@@ -186,6 +186,9 @@ integration:
|
||||
port_id: rest_api
|
||||
required: false
|
||||
|
||||
module:
|
||||
path: module/manifest.yaml
|
||||
|
||||
backup:
|
||||
strategy: online_save
|
||||
source_mounts:
|
||||
|
||||
+1
-2
@@ -46,7 +46,6 @@ func run(logger *slog.Logger) error {
|
||||
databasePath := environment("DOGAMA_DATABASE_PATH", "dogama.db")
|
||||
serversRoot := environment("DOGAMA_SERVERS_ROOT", "/srv/game-servers")
|
||||
templatesRoot := environment("DOGAMA_TEMPLATES_ROOT", "/var/lib/dogama/templates")
|
||||
modulesRoot := environment("DOGAMA_MODULES_ROOT", "/usr/share/dogama/modules")
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
@@ -119,7 +118,7 @@ func run(logger *slog.Logger) error {
|
||||
}
|
||||
handler, err = web.NewHandlerCompleteWithCatalogDeploymentAndRuntime(auth.New(db), repository, lifecycle, backupService, importService, auditService, notificationService, func(ctx context.Context) (catalog.ScanResult, error) {
|
||||
return synchronizeCatalog(ctx, repository, templatesRoot)
|
||||
}, serversRoot, instance.NewModuleService(repository, repository, modulesRoot), logger)
|
||||
}, serversRoot, instance.NewModuleService(repository, repository), logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ Read this compact operational baseline before starting a milestone. Open detaile
|
||||
- Label values support only the explicit allowlist in `internal/instance/container_config.go`; unknown variables are errors, not arbitrary templates.
|
||||
- `{{game.icon_url}}` is the public icon for the game. `{{instance.slug}}` remains supported.
|
||||
- Template game artwork contains separate required local `logo` and horizontal `image` assets; template validation rejects missing files. Deployment previews expose distinct logo and artwork URLs while retaining `icon_url` as a compatible logo alias. Palworld template `1.1.0` is the first snapshot with this contract.
|
||||
- Embedded catalog validation is collection-based: every discovered template is schema- and cross-field-validated, including referenced assets, declared integration modules and ports/configuration. Template asset contents are not SHA-256-pinned, allowing administrator-maintained local assets while preserving required-path validation. A template `version` identifies a DoGaMa snapshot, whereas `container.tag` selects the game-server image; the official Palworld template follows Pocketpair's `latest` tag for new deployments and ordinary pulls.
|
||||
- Embedded catalog validation is collection-based: every discovered template is schema- and cross-field-validated, including referenced assets, declared template-local integration modules and ports/configuration. A module bundle lives at `<template>/module/`, is declared by `module.path`, is path-confined and is retained with the selected snapshot; no game-specific internal registry exists. Template asset contents are not SHA-256-pinned, allowing administrator-maintained local assets while preserving required-path validation. A template `version` identifies a DoGaMa snapshot, whereas `container.tag` selects the game-server image; the official Palworld template follows Pocketpair's `latest` tag for new deployments and ordinary pulls.
|
||||
- Instance slugs are derived from the display name, not canonical IDs. Accents are normalized to ASCII; whitespace, `/`, punctuation and special characters become safe hyphen separators; repeated and edge hyphens are removed.
|
||||
- Docker user mode is fixed at creation to DoGaMa UID/GID, custom numeric UID/GID, or image-defined user. Never perform automatic recursive ownership changes.
|
||||
- A pinned image tag is an explicit mutable tag, not an immutable digest. Tracked mode follows the template's declared default tag.
|
||||
|
||||
@@ -57,7 +57,7 @@ Suggested container paths:
|
||||
/var/lib/dogama/
|
||||
dogama.db
|
||||
catalog/
|
||||
modules/
|
||||
catalog/<game>/module/
|
||||
imports/staging/
|
||||
cache/
|
||||
/srv/game-servers/<instance-slug>/
|
||||
@@ -84,4 +84,3 @@ Paths stored in SQLite use stable instance and mount identifiers. User-supplied
|
||||
Long-running operations are durable jobs in SQLite. A per-instance lock serializes mutually exclusive actions. Jobs use explicit phases and checkpoints so a restart can resume, retry safely or mark manual intervention required. UI requests enqueue work and return an operation identifier rather than keeping a long HTTP request open.
|
||||
|
||||
The scheduler is internal for V1 and handles cron backups, retention, audit purge, optional start/stop schedules, module/catalog update checks and queued notifications. Only one scheduler leader exists because V1 runs one main-application replica.
|
||||
|
||||
|
||||
@@ -10,17 +10,26 @@ Game-specific API <-> WebAssembly adapter <-> normalized DoGaMa API
|
||||
|
||||
It may query status, list players, request an in-game save, announce, shut down gracefully, kick, ban or unban when the game supports those operations. It does not own container lifecycle, files, users, backups, scheduling or UI.
|
||||
|
||||
## Package
|
||||
## Template-local package
|
||||
|
||||
```text
|
||||
palworld-rest-1.0.0.dogama-module/
|
||||
module.wasm
|
||||
manifest.yaml
|
||||
README.md
|
||||
LICENSE
|
||||
catalog/palworld/
|
||||
template.yaml
|
||||
assets/
|
||||
module/
|
||||
manifest.yaml
|
||||
module.wasm
|
||||
src/
|
||||
README.md
|
||||
LICENSE
|
||||
```
|
||||
|
||||
The installed artifact is content-addressed. Manifest, binary checksum, source/trust status and installation time are recorded. A package cannot contain executable helpers or dynamic libraries.
|
||||
`template.yaml` declares the optional manifest as `module.path`. The path must
|
||||
remain under the template root's `module/` directory; absolute paths, traversal
|
||||
and local symbolic links are rejected. The validated module bundle contributes
|
||||
to the template snapshot digest and is retained with that snapshot, so a later
|
||||
local-template update cannot change a pinned instance's adapter. A package
|
||||
cannot contain executable helpers or dynamic libraries.
|
||||
|
||||
## Runtime contract
|
||||
|
||||
@@ -62,16 +71,12 @@ Per call, enforce a deadline, instruction/fuel budget, memory ceiling, maximum h
|
||||
|
||||
## Independent versioning
|
||||
|
||||
Templates and modules have independent semantic versions.
|
||||
|
||||
- A template pins an acceptable module ID and version range.
|
||||
- A manifest states supported manager module-API versions and game IDs.
|
||||
- Installation of a new module does not activate it automatically for existing instances.
|
||||
- Activation runs schema, checksum, ABI, capability and connection tests.
|
||||
- The previous version stays available for rollback until the new version is healthy.
|
||||
- A module update never silently changes instance settings or template snapshots.
|
||||
The manifest retains its module ID and API compatibility contract, but discovery
|
||||
is template-driven: there is no game-to-module registry in the application.
|
||||
An administrator can copy/import `my-game/template.yaml`, `assets/` and
|
||||
`module/` together. A module update is accepted on the next local scan and
|
||||
creates a distinct digest; selected snapshots retain their own bundle.
|
||||
|
||||
## Prohibited behavior
|
||||
|
||||
Modules cannot create/delete containers, read SQLite, access host/game files, create backups, execute commands, manage users, expose routes or UI, contact other instances, or make unrestricted network calls. If a proposed integration needs those powers, the generic DoGaMa contract must be extended safely instead of bypassed.
|
||||
|
||||
|
||||
@@ -20,14 +20,19 @@ Create:
|
||||
|
||||
```text
|
||||
catalog/<game>/template.yaml
|
||||
modules/<module-id>/manifest.yaml
|
||||
modules/<module-id>/README.md
|
||||
modules/<module-id>/src/... implementation phase
|
||||
modules/<module-id>/tests/...
|
||||
catalog/<game>/module/manifest.yaml
|
||||
catalog/<game>/module/README.md
|
||||
catalog/<game>/module/src/... implementation phase
|
||||
catalog/<game>/module/tests/...
|
||||
```
|
||||
|
||||
Choose only required capabilities. Translate game errors into normalized errors. Use the logical `instance_api` host functions; never accept arbitrary destination URLs. Keep game API credentials as declared secret configuration.
|
||||
|
||||
Declare the optional bundle with `module.path: module/manifest.yaml` in the
|
||||
same template. The path is confined to the template's root `module/`
|
||||
directory. Local administrator templates use precisely this layout; no
|
||||
application recompilation or internal module registration is involved.
|
||||
|
||||
## Required verification for a contribution
|
||||
|
||||
- Template validates against `specs/template.schema.json`.
|
||||
@@ -53,4 +58,3 @@ Then implement the smallest valid surface:
|
||||
## Contract-edit rule
|
||||
|
||||
If a new game cannot fit the current schema, first determine whether it exposes a genuinely generic need. Extend the schema narrowly with documentation, migration/compatibility analysis, validation, negative tests and updated examples. Never add an escape hatch such as arbitrary commands, raw Compose fragments, host paths or unrestricted network permissions.
|
||||
|
||||
|
||||
@@ -21,13 +21,34 @@ web/ embedded UI source
|
||||
internal/persistence/sqlite/schema.sql embedded current SQLite schema
|
||||
specs/ schemas and normalized contracts
|
||||
catalog/ reference templates
|
||||
modules/ reference modules and fixtures
|
||||
catalog/<game>/module/ optional template-local WASM adapters and sources
|
||||
docs/
|
||||
tests/integration/
|
||||
```
|
||||
|
||||
## Initial application development
|
||||
|
||||
## Template-local integration modules
|
||||
|
||||
An administrator may create or import a local template as a self-contained
|
||||
directory:
|
||||
|
||||
```text
|
||||
my-game/
|
||||
template.yaml
|
||||
assets/
|
||||
module/ # optional
|
||||
manifest.yaml
|
||||
custom.wasm
|
||||
```
|
||||
|
||||
When present, declare the manifest in `template.yaml` with
|
||||
`module.path: module/manifest.yaml`. The module directory is confined to the
|
||||
template root: absolute paths, traversal and symbolic links are rejected. The
|
||||
manifest and WASM artifact remain subject to the generic manifest schema and
|
||||
the capability-limited WASM sandbox. No rebuild of DoGaMa or internal game
|
||||
registry entry is needed for a local module.
|
||||
|
||||
The main application requires Go 1.25. SQLite uses the pure-Go `modernc.org/sqlite` driver, so neither cgo nor a system SQLite development library is required. Standard Compose supplies all internal bootstrap contracts. The agent generates its shared token and the application generates its master key independently. Direct developer execution may override `DOGAMA_LISTEN_ADDRESS`, `DOGAMA_DATABASE_PATH`, `DOGAMA_AGENT_URL`, `DOGAMA_AGENT_TOKEN_FILE` and `DOGAMA_MASTER_KEY_FILE`; lifecycle routes remain disabled when both agent overrides are absent. These are development controls, not public deployment settings. Run it with:
|
||||
|
||||
```sh
|
||||
|
||||
@@ -100,3 +100,97 @@ func TestScanDirAcceptsLocallyModifiedReferencedAsset(t *testing.T) {
|
||||
}
|
||||
t.Skip("embedded catalog has no container assets")
|
||||
}
|
||||
|
||||
func TestScanDirLoadsTemplateLocalModuleAndAcceptsChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := catalog.InitializeOfficial(root, catalogdata.Files); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := catalog.ScanDir(root)
|
||||
if err != nil || len(first.Valid) == 0 {
|
||||
t.Fatalf("initial scan = %#v, %v", first, err)
|
||||
}
|
||||
var snapshot catalog.Snapshot
|
||||
for _, candidate := range first.Valid {
|
||||
if candidate.Template.Module != nil {
|
||||
snapshot = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if snapshot.Template.Module == nil || len(snapshot.ModuleFiles) == 0 {
|
||||
t.Fatal("template-local module was not discovered")
|
||||
}
|
||||
wasmPath := filepath.Join(root, snapshot.AssetRoot, "module", "module.wasm")
|
||||
body, err := os.ReadFile(wasmPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(wasmPath, append(body, 0), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := catalog.ScanDir(root)
|
||||
if err != nil || len(second.Errors) != 0 {
|
||||
t.Fatalf("modified local module scan = %#v, %v", second, err)
|
||||
}
|
||||
var updated catalog.Snapshot
|
||||
for _, candidate := range second.Valid {
|
||||
if candidate.Template.ID == snapshot.Template.ID {
|
||||
updated = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if updated.Digest == snapshot.Digest {
|
||||
t.Fatal("module change did not affect template snapshot digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanDirAcceptsLocalTemplateWithoutModule(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := catalog.InitializeOfficial(root, catalogdata.Files); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := catalog.ScanDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, snapshot := range result.Valid {
|
||||
if snapshot.Template.Module == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("a template without an optional module was not accepted")
|
||||
}
|
||||
|
||||
func TestScanDirRejectsMissingAndEscapingLocalModule(t *testing.T) {
|
||||
for _, modulePath := range []string{"module/missing.yaml", "../outside.wasm", "/outside.wasm"} {
|
||||
t.Run(modulePath, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := catalog.InitializeOfficial(root, catalogdata.Files); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initial, err := catalog.ScanDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var snapshot catalog.Snapshot
|
||||
for _, candidate := range initial.Valid {
|
||||
if candidate.Template.Module != nil {
|
||||
snapshot = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
path := filepath.Join(root, snapshot.AssetRoot, "template.yaml")
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(strings.Replace(string(body), snapshot.Template.Module.Path, modulePath, 1)), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := catalog.ScanDir(root)
|
||||
if err != nil || len(result.Errors) != 1 || result.Errors[0].Template != snapshot.AssetRoot {
|
||||
t.Fatalf("unsafe local module scan = %#v, %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ import (
|
||||
)
|
||||
|
||||
const templateSchemaURL = "https://dogama.dev/schemas/template-v1.json"
|
||||
const moduleManifestSchemaURL = "https://dogama.dev/schemas/module-manifest-v1.json"
|
||||
const maxModuleBundleBytes = 16 << 20
|
||||
|
||||
// ValidationIssue points to one invalid field without exposing input secrets.
|
||||
type ValidationIssue struct {
|
||||
@@ -86,6 +88,9 @@ type Template struct {
|
||||
ModuleID string `json:"module_id"`
|
||||
PortID string `json:"port_id"`
|
||||
} `json:"integration,omitempty"`
|
||||
Module *struct {
|
||||
Path string `json:"path"`
|
||||
} `json:"module,omitempty"`
|
||||
Backup struct {
|
||||
Strategy string `json:"strategy"`
|
||||
SourceMounts []string `json:"source_mounts"`
|
||||
@@ -177,6 +182,7 @@ type Snapshot struct {
|
||||
Digest string
|
||||
Origin string
|
||||
AssetRoot string
|
||||
ModuleFiles map[string][]byte `json:"-"`
|
||||
}
|
||||
|
||||
// LoadFS validates every template.yaml below root and returns stable snapshots.
|
||||
@@ -320,14 +326,27 @@ func Validate(body []byte, assetRoot string, source fs.FS) (Snapshot, error) {
|
||||
if len(issues) != 0 {
|
||||
return Snapshot{}, &ValidationErrors{Issues: issues}
|
||||
}
|
||||
moduleFiles, moduleIssues := collectModuleFiles(template, assetRoot, source)
|
||||
if len(moduleIssues) != 0 {
|
||||
return Snapshot{}, &ValidationErrors{Issues: moduleIssues}
|
||||
}
|
||||
pretty, _ := json.MarshalIndent(raw, "", " ")
|
||||
digest := sha256.Sum256(canonical)
|
||||
digester := sha256.New()
|
||||
_, _ = digester.Write(canonical)
|
||||
for _, name := range sortedModuleFiles(moduleFiles) {
|
||||
_, _ = digester.Write([]byte{0})
|
||||
_, _ = digester.Write([]byte(name))
|
||||
_, _ = digester.Write([]byte{0})
|
||||
_, _ = digester.Write(moduleFiles[name])
|
||||
}
|
||||
digest := digester.Sum(nil)
|
||||
return Snapshot{
|
||||
Template: template,
|
||||
CanonicalYAML: string(pretty) + "\n",
|
||||
Digest: hex.EncodeToString(digest[:]),
|
||||
Digest: hex.EncodeToString(digest),
|
||||
Origin: template.Source.Type,
|
||||
AssetRoot: assetRoot,
|
||||
ModuleFiles: moduleFiles,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -353,6 +372,24 @@ func compileSchema() (*jsonschema.Schema, error) {
|
||||
return schema, nil
|
||||
}
|
||||
|
||||
func compileModuleManifestSchema() (*jsonschema.Schema, error) {
|
||||
body, err := specs.Files.ReadFile("module-manifest.schema.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
compiler.AssertFormat()
|
||||
compiler.UseRegexpEngine(compileECMAScript)
|
||||
var document any
|
||||
if err := json.Unmarshal(body, &document); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := compiler.AddResource(moduleManifestSchemaURL, document); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return compiler.Compile(moduleManifestSchemaURL)
|
||||
}
|
||||
|
||||
type ecmaRegexp regexp2.Regexp
|
||||
|
||||
func (expression *ecmaRegexp) MatchString(value string) bool {
|
||||
@@ -457,6 +494,9 @@ func crossValidate(template Template, assetRoot string, source fs.FS) []Validati
|
||||
issues = append(issues, ValidationIssue{Path: "/integration/port_id", Message: "integration must reference an integration port"})
|
||||
}
|
||||
}
|
||||
if template.Integration != nil && template.Module == nil {
|
||||
issues = append(issues, ValidationIssue{Path: "/module", Message: "integration requires a template-local module"})
|
||||
}
|
||||
if template.Healthcheck.PortID != "" {
|
||||
if _, exists := ports[template.Healthcheck.PortID]; !exists {
|
||||
issues = append(issues, ValidationIssue{Path: "/healthcheck/port_id", Message: "healthcheck port does not exist"})
|
||||
@@ -478,6 +518,89 @@ func crossValidate(template Template, assetRoot string, source fs.FS) []Validati
|
||||
return issues
|
||||
}
|
||||
|
||||
func collectModuleFiles(template Template, assetRoot string, source fs.FS) (map[string][]byte, []ValidationIssue) {
|
||||
if template.Module == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if !validModulePath(template.Module.Path) {
|
||||
return nil, []ValidationIssue{{Path: "/module/path", Message: "module path must remain inside the template module directory"}}
|
||||
}
|
||||
root := path.Join(assetRoot, "module")
|
||||
files := map[string][]byte{}
|
||||
var totalBytes int
|
||||
err := fs.WalkDir(source, root, func(name string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if entry.Type()&fs.ModeSymlink != 0 {
|
||||
return errors.New("symbolic link")
|
||||
}
|
||||
body, err := fs.ReadFile(source, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
totalBytes += len(body)
|
||||
if totalBytes > maxModuleBundleBytes {
|
||||
return errors.New("module bundle exceeds size limit")
|
||||
}
|
||||
rel := strings.TrimPrefix(name, strings.TrimSuffix(assetRoot, "/")+"/")
|
||||
if !strings.HasPrefix(rel, "module/") {
|
||||
return errors.New("invalid module path")
|
||||
}
|
||||
files[rel] = body
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, []ValidationIssue{{Path: "/module", Message: "module directory cannot be read"}}
|
||||
}
|
||||
if _, ok := files[template.Module.Path]; !ok {
|
||||
return nil, []ValidationIssue{{Path: "/module/path", Message: "declared module manifest is missing"}}
|
||||
}
|
||||
if err := validateModuleManifest(files[template.Module.Path], files); err != nil {
|
||||
return nil, []ValidationIssue{{Path: "/module", Message: "declared module bundle is invalid"}}
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func validateModuleManifest(body []byte, files map[string][]byte) error {
|
||||
var raw any
|
||||
if err := yaml.Unmarshal(body, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
schema, err := compileModuleManifestSchema()
|
||||
if err != nil || schema.Validate(raw) != nil {
|
||||
return errors.New("invalid manifest")
|
||||
}
|
||||
var manifest struct {
|
||||
Artifacts struct {
|
||||
WASM string `yaml:"wasm"`
|
||||
} `yaml:"artifacts"`
|
||||
}
|
||||
if err := yaml.Unmarshal(body, &manifest); err != nil || manifest.Artifacts.WASM == "" {
|
||||
return errors.New("invalid artifact")
|
||||
}
|
||||
if _, ok := files["module/"+manifest.Artifacts.WASM]; !ok {
|
||||
return errors.New("missing wasm artifact")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validModulePath(value string) bool {
|
||||
return value != "" && !path.IsAbs(value) && path.Clean(value) == value && strings.HasPrefix(value, "module/") && !strings.Contains(value, "..")
|
||||
}
|
||||
|
||||
func sortedModuleFiles(files map[string][]byte) []string {
|
||||
names := make([]string, 0, len(files))
|
||||
for name := range files {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func hasAlias(node *yaml.Node) bool {
|
||||
if node.Kind == yaml.AliasNode {
|
||||
return true
|
||||
|
||||
@@ -3,9 +3,7 @@ package catalog_test
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -59,15 +57,43 @@ func TestBuiltInCatalogValidatesEveryDiscoveredTemplate(t *testing.T) {
|
||||
t.Fatalf("template path=%q id=%q asset=%q is missing: %v", name, snapshot.Template.ID, assetPath, statErr)
|
||||
}
|
||||
}
|
||||
if snapshot.Template.Integration != nil {
|
||||
manifest := filepath.Join("..", "..", "modules", snapshot.Template.Integration.ModuleID, "manifest.yaml")
|
||||
if _, statErr := os.Stat(manifest); statErr != nil {
|
||||
t.Fatalf("template path=%q id=%q declared module=%q is missing at %q: %v", name, snapshot.Template.ID, snapshot.Template.Integration.ModuleID, manifest, statErr)
|
||||
if snapshot.Template.Module != nil {
|
||||
if _, ok := snapshot.ModuleFiles[snapshot.Template.Module.Path]; !ok {
|
||||
t.Fatalf("template path=%q id=%q declared module=%q is missing", name, snapshot.Template.ID, snapshot.Template.Module.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrossValidationRejectsMissingDeclaredModule(t *testing.T) {
|
||||
name, body, snapshot := embeddedTemplate(t)
|
||||
if snapshot.Template.Module == nil {
|
||||
t.Skip("embedded fixture has no module")
|
||||
}
|
||||
body = []byte(strings.Replace(string(body), snapshot.Template.Module.Path, "module/missing.yaml", 1))
|
||||
_, err := catalog.Validate(body, path.Dir(name), catalogdata.Files)
|
||||
var validation *catalog.ValidationErrors
|
||||
if !errors.As(err, &validation) || len(validation.Issues) == 0 || validation.Issues[0].Path != "/module/path" {
|
||||
t.Fatalf("validation error = %#v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrossValidationRejectsModuleTraversalAndAbsolutePath(t *testing.T) {
|
||||
name, body, snapshot := embeddedTemplate(t)
|
||||
if snapshot.Template.Module == nil {
|
||||
t.Skip("embedded fixture has no module")
|
||||
}
|
||||
for _, modulePath := range []string{"../outside.wasm", "/outside.wasm"} {
|
||||
t.Run(modulePath, func(t *testing.T) {
|
||||
invalid := []byte(strings.Replace(string(body), snapshot.Template.Module.Path, modulePath, 1))
|
||||
_, err := catalog.Validate(invalid, path.Dir(name), catalogdata.Files)
|
||||
if err == nil {
|
||||
t.Fatalf("unsafe module path accepted: %q", modulePath)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInCatalogIsDeterministic(t *testing.T) {
|
||||
first, err := catalog.LoadFS(catalogdata.Files, ".")
|
||||
if err != nil {
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -23,11 +21,10 @@ var ErrModuleUnavailable = errors.New("module unavailable")
|
||||
type ModuleService struct {
|
||||
secrets SecretRepository
|
||||
catalog catalog.Repository
|
||||
root string
|
||||
}
|
||||
|
||||
func NewModuleService(secrets SecretRepository, repository catalog.Repository, root string) *ModuleService {
|
||||
return &ModuleService{secrets: secrets, catalog: repository, root: filepath.Clean(root)}
|
||||
func NewModuleService(secrets SecretRepository, repository catalog.Repository) *ModuleService {
|
||||
return &ModuleService{secrets: secrets, catalog: repository}
|
||||
}
|
||||
|
||||
type ServerInfo struct {
|
||||
@@ -105,11 +102,11 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
|
||||
return nil, moduleManifest{}, ErrModuleUnavailable
|
||||
}
|
||||
integration := snapshot.Template.Integration
|
||||
if integration.ModuleID == "" || strings.Contains(integration.ModuleID, "/") || strings.Contains(integration.ModuleID, "..") {
|
||||
if integration.ModuleID == "" || snapshot.Template.Module == nil || !validTemplateModulePath(snapshot.Template.Module.Path) {
|
||||
return nil, moduleManifest{}, ErrModuleUnavailable
|
||||
}
|
||||
body, err := os.ReadFile(filepath.Join(s.root, integration.ModuleID, "manifest.yaml"))
|
||||
if err != nil {
|
||||
body, ok := snapshot.ModuleFiles[snapshot.Template.Module.Path]
|
||||
if !ok {
|
||||
return nil, moduleManifest{}, ErrModuleUnavailable
|
||||
}
|
||||
var manifest moduleManifest
|
||||
@@ -134,7 +131,7 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
|
||||
return nil, moduleManifest{}, ErrModuleUnavailable
|
||||
}
|
||||
}
|
||||
if len(methods) == 0 || manifest.Artifacts.WASM == "" || filepath.Base(manifest.Artifacts.WASM) != manifest.Artifacts.WASM {
|
||||
if len(methods) == 0 || manifest.Artifacts.WASM == "" || strings.Contains(manifest.Artifacts.WASM, "/") || strings.Contains(manifest.Artifacts.WASM, "..") {
|
||||
return nil, moduleManifest{}, ErrModuleUnavailable
|
||||
}
|
||||
config, secrets := map[string]string{}, map[string]string{}
|
||||
@@ -154,8 +151,8 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
|
||||
config[field.ID] = v
|
||||
}
|
||||
}
|
||||
wasm, err := os.ReadFile(filepath.Join(s.root, integration.ModuleID, manifest.Artifacts.WASM))
|
||||
if err != nil {
|
||||
wasm, ok := snapshot.ModuleFiles["module/"+manifest.Artifacts.WASM]
|
||||
if !ok {
|
||||
return nil, moduleManifest{}, ErrModuleUnavailable
|
||||
}
|
||||
r, err := module.New(ctx, wasm, manifest.Artifacts.SHA256, manifest.Capabilities, module.Limits{MemoryMB: manifest.Limits.MemoryMB, Timeout: durationMS(manifest.Limits.TimeoutMS), MaxResponseBytes: manifest.Limits.MaxResponseBytes, MaxConcurrentCall: manifest.Limits.MaxConcurrentCalls}, module.Binding{InstanceID: value.ID, ContainerPort: port, AllowedMethods: methods, Configuration: config, Secrets: secrets})
|
||||
@@ -166,6 +163,11 @@ func (s *ModuleService) runtime(ctx context.Context, value StoredInstance) (*mod
|
||||
}
|
||||
|
||||
func durationMS(v int) time.Duration { return time.Duration(v) * time.Millisecond }
|
||||
|
||||
func validTemplateModulePath(value string) bool {
|
||||
return strings.HasPrefix(value, "module/") && !strings.Contains(value, "..") && !strings.HasPrefix(value, "/")
|
||||
}
|
||||
|
||||
func contains(values []string, needle string) bool {
|
||||
for _, v := range values {
|
||||
if v == needle {
|
||||
|
||||
@@ -29,7 +29,7 @@ func TestValidRelativeAPIPath(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPalworldAdapterReportsBoundedFailures(t *testing.T) {
|
||||
wasm, err := os.ReadFile("../../modules/palworld-rest/module.wasm")
|
||||
wasm, err := os.ReadFile("../../catalog/palworld/module/module.wasm")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func TestPalworldAdapterReportsBoundedFailures(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPalworldAdapterExecutesInSandbox(t *testing.T) {
|
||||
wasm, err := os.ReadFile("../../modules/palworld-rest/module.wasm")
|
||||
wasm, err := os.ReadFile("../../catalog/palworld/module/module.wasm")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -122,6 +122,10 @@ func (r *Repository) Replace(ctx context.Context, snapshots []catalog.Snapshot)
|
||||
now := r.now().UTC().Format(time.RFC3339Nano)
|
||||
ids := make([]string, 0, len(snapshots))
|
||||
for _, snapshot := range snapshots {
|
||||
moduleBundle, marshalErr := json.Marshal(snapshot.ModuleFiles)
|
||||
if marshalErr != nil {
|
||||
return fmt.Errorf("encode template module bundle: %w", marshalErr)
|
||||
}
|
||||
ids = append(ids, snapshot.Template.ID)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO templates(id, origin, trust_status, active_version, available, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, 1, ?, ?)
|
||||
@@ -130,9 +134,9 @@ func (r *Repository) Replace(ctx context.Context, snapshots []catalog.Snapshot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert catalog template: %w", err)
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO template_versions(template_id, version, schema_version, canonical_yaml, digest, game_id, game_name, description, image, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(template_id, version) DO UPDATE SET schema_version=excluded.schema_version, canonical_yaml=excluded.canonical_yaml, digest=excluded.digest, game_id=excluded.game_id, game_name=excluded.game_name, description=excluded.description, image=excluded.image`, snapshot.Template.ID, snapshot.Template.Version, snapshot.Template.SchemaVersion, snapshot.CanonicalYAML, snapshot.Digest, snapshot.Template.Game.ID, snapshot.Template.Game.Name, snapshot.Template.Game.Description, snapshot.Template.Game.Artwork.Image, now)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO template_versions(template_id, version, schema_version, canonical_yaml, digest, game_id, game_name, description, image, module_bundle, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(template_id, version) DO UPDATE SET schema_version=excluded.schema_version, canonical_yaml=excluded.canonical_yaml, digest=excluded.digest, game_id=excluded.game_id, game_name=excluded.game_name, description=excluded.description, image=excluded.image, module_bundle=excluded.module_bundle`, snapshot.Template.ID, snapshot.Template.Version, snapshot.Template.SchemaVersion, snapshot.CanonicalYAML, snapshot.Digest, snapshot.Template.Game.ID, snapshot.Template.Game.Name, snapshot.Template.Game.Description, snapshot.Template.Game.Artwork.Image, moduleBundle, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert template snapshot: %w", err)
|
||||
}
|
||||
@@ -177,8 +181,9 @@ func (r *Repository) List(ctx context.Context) ([]catalog.Summary, error) {
|
||||
|
||||
func (r *Repository) Get(ctx context.Context, id, version string) (catalog.Snapshot, error) {
|
||||
var canonical, digest, origin string
|
||||
err := r.db.QueryRowContext(ctx, `SELECT v.canonical_yaml, v.digest, t.origin FROM template_versions v JOIN templates t ON t.id=v.template_id
|
||||
WHERE v.template_id=? AND v.version=?`, id, version).Scan(&canonical, &digest, &origin)
|
||||
var moduleBundle []byte
|
||||
err := r.db.QueryRowContext(ctx, `SELECT v.canonical_yaml, v.digest, t.origin, v.module_bundle FROM template_versions v JOIN templates t ON t.id=v.template_id
|
||||
WHERE v.template_id=? AND v.version=?`, id, version).Scan(&canonical, &digest, &origin, &moduleBundle)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return catalog.Snapshot{}, catalog.ErrTemplateNotFound
|
||||
}
|
||||
@@ -189,7 +194,11 @@ func (r *Repository) Get(ctx context.Context, id, version string) (catalog.Snaps
|
||||
if err := json.Unmarshal([]byte(canonical), &template); err != nil {
|
||||
return catalog.Snapshot{}, fmt.Errorf("decode stored template snapshot: %w", err)
|
||||
}
|
||||
return catalog.Snapshot{Template: template, CanonicalYAML: canonical, Digest: digest, Origin: origin}, nil
|
||||
var moduleFiles map[string][]byte
|
||||
if len(moduleBundle) != 0 && json.Unmarshal(moduleBundle, &moduleFiles) != nil {
|
||||
return catalog.Snapshot{}, errors.New("decode stored template module bundle")
|
||||
}
|
||||
return catalog.Snapshot{Template: template, CanonicalYAML: canonical, Digest: digest, Origin: origin, ModuleFiles: moduleFiles}, nil
|
||||
}
|
||||
|
||||
func (r *Repository) CreateDraft(ctx context.Context, draft instance.Draft) error {
|
||||
|
||||
@@ -80,6 +80,9 @@ func TestCatalogSyncIsImmutableAndDraftPinsSnapshot(t *testing.T) {
|
||||
if err != nil || loaded.Digest != palworld.Digest {
|
||||
t.Fatalf("loaded snapshot = %#v, error = %v", loaded, err)
|
||||
}
|
||||
if len(loaded.ModuleFiles) == 0 || loaded.Template.Module == nil || loaded.ModuleFiles[loaded.Template.Module.Path] == nil {
|
||||
t.Fatalf("template-local module bundle was not retained: %#v", loaded.Template.Module)
|
||||
}
|
||||
|
||||
tampered := palworld
|
||||
tampered.Digest = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
|
||||
|
||||
@@ -53,6 +53,7 @@ CREATE TABLE template_versions (
|
||||
game_name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
image TEXT NOT NULL,
|
||||
module_bundle BLOB,
|
||||
created_at TEXT NOT NULL,
|
||||
PRIMARY KEY (template_id, version),
|
||||
UNIQUE (digest)
|
||||
|
||||
@@ -57,12 +57,46 @@ func initialize(ctx context.Context, db *sql.DB) error {
|
||||
return fmt.Errorf("inspect sqlite schema: %w", err)
|
||||
}
|
||||
if existing == 1 {
|
||||
return ensureDiagnosticSchema(ctx, db)
|
||||
if err := ensureDiagnosticSchema(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureTemplateModuleSchema(ctx, db)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, schema); err != nil {
|
||||
return fmt.Errorf("initialize sqlite schema: %w", err)
|
||||
}
|
||||
return ensureDiagnosticSchema(ctx, db)
|
||||
if err := ensureDiagnosticSchema(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureTemplateModuleSchema(ctx, db)
|
||||
}
|
||||
|
||||
// ensureTemplateModuleSchema preserves the complete module bundle with an
|
||||
// immutable template snapshot for stores created before template-local modules.
|
||||
func ensureTemplateModuleSchema(ctx context.Context, db *sql.DB) error {
|
||||
rows, err := db.QueryContext(ctx, "PRAGMA table_info(template_versions)")
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect template module schema: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid, notNull, primaryKey int
|
||||
var name, typ string
|
||||
var defaultValue any
|
||||
if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &primaryKey); err != nil {
|
||||
return err
|
||||
}
|
||||
if name == "module_bundle" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "ALTER TABLE template_versions ADD COLUMN module_bundle BLOB"); err != nil {
|
||||
return fmt.Errorf("migrate template module schema: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// This additive migration is safe for existing V1 databases and keeps the
|
||||
|
||||
@@ -206,7 +206,7 @@ func TestInstanceUnbanCapabilityFallback(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("palworld manual fallback validates and audits", func(t *testing.T) {
|
||||
manifest, err := os.ReadFile("../../modules/palworld-rest/manifest.yaml")
|
||||
manifest, err := os.ReadFile("../../catalog/palworld/module/manifest.yaml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -149,6 +149,14 @@
|
||||
"required": { "type": "boolean", "default": false }
|
||||
}
|
||||
},
|
||||
"module": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["path"],
|
||||
"properties": {
|
||||
"path": { "type": "string", "pattern": "^module/(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+\\.yaml$", "maxLength": 240 }
|
||||
}
|
||||
},
|
||||
"backup": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
|
||||
@@ -85,7 +85,7 @@ def validate_cross_references(template: dict, manifest: dict | None, template_pa
|
||||
return warnings
|
||||
|
||||
wasm_digest = manifest["artifacts"]["sha256"]
|
||||
wasm_path = ROOT / "modules" / manifest["id"] / manifest["artifacts"]["wasm"]
|
||||
wasm_path = template_path.parent / "module" / manifest["artifacts"]["wasm"]
|
||||
if wasm_digest == "0" * 64 and not wasm_path.exists():
|
||||
warnings.append("Palworld module is a source specification: module.wasm and its final checksum are intentionally pending.")
|
||||
elif wasm_path.is_file():
|
||||
@@ -110,8 +110,14 @@ def validate_catalog() -> list[str]:
|
||||
identities.add(identity)
|
||||
integration = template.get("integration")
|
||||
manifest = None
|
||||
module = template.get("module")
|
||||
if integration:
|
||||
manifest_path = ROOT / "modules" / integration["module_id"] / "manifest.yaml"
|
||||
assert module is not None, "Integration requires a template-local module"
|
||||
if module:
|
||||
module_path = Path(module["path"])
|
||||
assert not module_path.is_absolute() and ".." not in module_path.parts, "Module path escapes template"
|
||||
manifest_path = (template_path.parent / module_path).resolve()
|
||||
assert manifest_path.parent.is_relative_to((template_path.parent / "module").resolve()), "Module path is outside template module directory"
|
||||
assert manifest_path.is_file(), f"Declared module manifest is missing: {manifest_path}"
|
||||
manifest = validate(manifest_schema, manifest_path)
|
||||
warnings.extend(validate_cross_references(template, manifest, template_path))
|
||||
|
||||
Reference in New Issue
Block a user