fix(plugins): keep built plugin loading on one module graph (#48595)

This commit is contained in:
Harold Hunt
2026-03-16 20:58:58 -04:00
committed by GitHub
parent 4863b651c6
commit 94c27f34a1
8 changed files with 385 additions and 94 deletions

View File

@@ -3,57 +3,86 @@ import path from "node:path";
import { pathToFileURL } from "node:url";
import { removePathIfExists } from "./runtime-postbuild-shared.mjs";
function linkOrCopyFile(sourcePath, targetPath) {
try {
fs.linkSync(sourcePath, targetPath);
} catch (error) {
if (error && typeof error === "object" && "code" in error) {
const code = error.code;
if (code === "EXDEV" || code === "EPERM" || code === "EMLINK") {
fs.copyFileSync(sourcePath, targetPath);
return;
}
}
throw error;
}
function symlinkType() {
return process.platform === "win32" ? "junction" : "dir";
}
function mirrorTreeWithHardlinks(sourceRoot, targetRoot) {
fs.mkdirSync(targetRoot, { recursive: true });
const queue = [{ sourceDir: sourceRoot, targetDir: targetRoot }];
function relativeSymlinkTarget(sourcePath, targetPath) {
const relativeTarget = path.relative(path.dirname(targetPath), sourcePath);
return relativeTarget || ".";
}
while (queue.length > 0) {
const current = queue.pop();
if (!current) {
function symlinkPath(sourcePath, targetPath, type) {
fs.symlinkSync(relativeSymlinkTarget(sourcePath, targetPath), targetPath, type);
}
function shouldWrapRuntimeJsFile(sourcePath) {
return path.extname(sourcePath) === ".js";
}
function shouldCopyRuntimeFile(sourcePath) {
const relativePath = sourcePath.replace(/\\/g, "/");
return (
relativePath.endsWith("/package.json") ||
relativePath.endsWith("/openclaw.plugin.json") ||
relativePath.endsWith("/.codex-plugin/plugin.json") ||
relativePath.endsWith("/.claude-plugin/plugin.json") ||
relativePath.endsWith("/.cursor-plugin/plugin.json")
);
}
function writeRuntimeModuleWrapper(sourcePath, targetPath) {
const specifier = relativeSymlinkTarget(sourcePath, targetPath).replace(/\\/g, "/");
const normalizedSpecifier = specifier.startsWith(".") ? specifier : `./${specifier}`;
fs.writeFileSync(
targetPath,
[
`export * from ${JSON.stringify(normalizedSpecifier)};`,
`import * as module from ${JSON.stringify(normalizedSpecifier)};`,
"export default module.default;",
"",
].join("\n"),
"utf8",
);
}
function stagePluginRuntimeOverlay(sourceDir, targetDir) {
fs.mkdirSync(targetDir, { recursive: true });
for (const dirent of fs.readdirSync(sourceDir, { withFileTypes: true })) {
if (dirent.name === "node_modules") {
continue;
}
for (const dirent of fs.readdirSync(current.sourceDir, { withFileTypes: true })) {
const sourcePath = path.join(current.sourceDir, dirent.name);
const targetPath = path.join(current.targetDir, dirent.name);
const sourcePath = path.join(sourceDir, dirent.name);
const targetPath = path.join(targetDir, dirent.name);
if (dirent.isDirectory()) {
fs.mkdirSync(targetPath, { recursive: true });
queue.push({ sourceDir: sourcePath, targetDir: targetPath });
continue;
}
if (dirent.isSymbolicLink()) {
fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath);
continue;
}
if (!dirent.isFile()) {
continue;
}
linkOrCopyFile(sourcePath, targetPath);
if (dirent.isDirectory()) {
stagePluginRuntimeOverlay(sourcePath, targetPath);
continue;
}
}
}
function symlinkType() {
return process.platform === "win32" ? "junction" : "dir";
if (dirent.isSymbolicLink()) {
fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath);
continue;
}
if (!dirent.isFile()) {
continue;
}
if (shouldWrapRuntimeJsFile(sourcePath)) {
writeRuntimeModuleWrapper(sourcePath, targetPath);
continue;
}
if (shouldCopyRuntimeFile(sourcePath)) {
fs.copyFileSync(sourcePath, targetPath);
continue;
}
symlinkPath(sourcePath, targetPath);
}
}
function linkPluginNodeModules(params) {
@@ -79,15 +108,17 @@ export function stageBundledPluginRuntime(params = {}) {
}
removePathIfExists(runtimeRoot);
mirrorTreeWithHardlinks(distRoot, runtimeRoot);
fs.mkdirSync(runtimeExtensionsRoot, { recursive: true });
for (const dirent of fs.readdirSync(distExtensionsRoot, { withFileTypes: true })) {
if (!dirent.isDirectory()) {
continue;
}
const distPluginDir = path.join(distExtensionsRoot, dirent.name);
const runtimePluginDir = path.join(runtimeExtensionsRoot, dirent.name);
const sourcePluginNodeModulesDir = path.join(sourceExtensionsRoot, dirent.name, "node_modules");
stagePluginRuntimeOverlay(distPluginDir, runtimePluginDir);
linkPluginNodeModules({
runtimePluginDir,
sourcePluginNodeModulesDir,

View File

@@ -0,0 +1,58 @@
import { describe, expect, it } from "vitest";
import tsdownConfig from "../../tsdown.config.ts";
type TsdownConfigEntry = {
entry?: Record<string, string> | string[];
outDir?: string;
};
function asConfigArray(config: unknown): TsdownConfigEntry[] {
return Array.isArray(config) ? (config as TsdownConfigEntry[]) : [config as TsdownConfigEntry];
}
function entryKeys(config: TsdownConfigEntry): string[] {
if (!config.entry || Array.isArray(config.entry)) {
return [];
}
return Object.keys(config.entry);
}
describe("tsdown config", () => {
it("keeps core, plugin runtime, plugin-sdk, bundled plugins, and bundled hooks in one dist graph", () => {
const configs = asConfigArray(tsdownConfig);
const distGraphs = configs.filter((config) => {
const keys = entryKeys(config);
return (
keys.includes("index") ||
keys.includes("plugins/runtime/index") ||
keys.includes("plugin-sdk/index") ||
keys.includes("extensions/openai/index") ||
keys.includes("bundled/boot-md/handler")
);
});
expect(distGraphs).toHaveLength(1);
expect(entryKeys(distGraphs[0])).toEqual(
expect.arrayContaining([
"index",
"plugins/runtime/index",
"plugin-sdk/index",
"extensions/openai/index",
"bundled/boot-md/handler",
]),
);
});
it("does not emit plugin-sdk or hooks from a separate dist graph", () => {
const configs = asConfigArray(tsdownConfig);
expect(configs.some((config) => config.outDir === "dist/plugin-sdk")).toBe(false);
expect(
configs.some((config) =>
Array.isArray(config.entry)
? config.entry.some((entry) => entry.includes("src/hooks/"))
: false,
),
).toBe(false);
});
});

View File

@@ -69,6 +69,9 @@ function getJiti() {
const { createJiti } = require("jiti");
jitiLoader = createJiti(__filename, {
interopDefault: true,
// Prefer Node's native sync ESM loader for built dist/plugin-sdk/*.js files
// so local plugins do not create a second transpiled OpenClaw core graph.
tryNative: true,
extensions: [".ts", ".tsx", ".mts", ".cts", ".mtsx", ".ctsx", ".js", ".mjs", ".cjs", ".json"],
});
return jitiLoader;

View File

@@ -25,6 +25,7 @@ function loadRootAliasWithStubs(options?: {
}) {
let createJitiCalls = 0;
let jitiLoadCalls = 0;
let lastJitiOptions: Record<string, unknown> | undefined;
const loadedSpecifiers: string[] = [];
const monolithicExports = options?.monolithicExports ?? {
slowHelper: () => "loaded",
@@ -52,8 +53,9 @@ function loadRootAliasWithStubs(options?: {
}
if (id === "jiti") {
return {
createJiti() {
createJiti(_filename: string, jitiOptions?: Record<string, unknown>) {
createJitiCalls += 1;
lastJitiOptions = jitiOptions;
return (specifier: string) => {
jitiLoadCalls += 1;
loadedSpecifiers.push(specifier);
@@ -73,6 +75,9 @@ function loadRootAliasWithStubs(options?: {
get jitiLoadCalls() {
return jitiLoadCalls;
},
get lastJitiOptions() {
return lastJitiOptions;
},
loadedSpecifiers,
};
}
@@ -116,6 +121,7 @@ describe("plugin-sdk root alias", () => {
expect("slowHelper" in lazyRootSdk).toBe(true);
expect(lazyModule.createJitiCalls).toBe(1);
expect(lazyModule.jitiLoadCalls).toBe(1);
expect(lazyModule.lastJitiOptions?.tryNative).toBe(true);
expect((lazyRootSdk.slowHelper as () => string)()).toBe("loaded");
expect(Object.keys(lazyRootSdk)).toContain("slowHelper");
expect(Object.getOwnPropertyDescriptor(lazyRootSdk, "slowHelper")).toBeDefined();

View File

@@ -3211,6 +3211,16 @@ module.exports = {
expect(resolved).toBe(distFile);
});
it("configures the plugin loader jiti boundary to prefer native dist modules", () => {
const options = __testing.buildPluginLoaderJitiOptions({});
expect(options.tryNative).toBe(true);
expect(options.interopDefault).toBe(true);
expect(options.extensions).toContain(".js");
expect(options.extensions).toContain(".ts");
expect("alias" in options).toBe(false);
});
it("prefers src root-alias shim when loader runs from src in non-production", () => {
const { root, srcFile } = createPluginSdkAliasFixture({
srcFile: "root-alias.cjs",
@@ -3243,6 +3253,15 @@ module.exports = {
expect(resolved).toBe(srcFile);
});
it("prefers dist plugin runtime module when loader runs from dist", () => {
const { root, distFile } = createPluginRuntimeAliasFixture();
const resolved = __testing.resolvePluginRuntimeModulePath({
modulePath: path.join(root, "dist", "plugins", "loader.js"),
});
expect(resolved).toBe(distFile);
});
it("resolves plugin runtime module from package root when loader runs from transpiler cache path", () => {
const { root, srcFile } = createPluginRuntimeAliasFixture();

View File

@@ -198,6 +198,21 @@ const resolvePluginSdkAliasFile = (params: {
const resolvePluginSdkAlias = (): string | null =>
resolvePluginSdkAliasFile({ srcFile: "root-alias.cjs", distFile: "root-alias.cjs" });
function buildPluginLoaderJitiOptions(aliasMap: Record<string, string>) {
return {
interopDefault: true,
// Prefer Node's native sync ESM loader for built dist/*.js modules so
// bundled plugins and plugin-sdk subpaths stay on the canonical module graph.
tryNative: true,
extensions: [".ts", ".tsx", ".mts", ".cts", ".mtsx", ".ctsx", ".js", ".mjs", ".cjs", ".json"],
...(Object.keys(aliasMap).length > 0
? {
alias: aliasMap,
}
: {}),
};
}
function resolvePluginRuntimeModulePath(params: LoaderModuleResolveParams = {}): string | null {
try {
const modulePath = resolveLoaderModulePath(params);
@@ -273,6 +288,7 @@ const resolvePluginSdkScopedAliasMap = (): Record<string, string> => {
};
export const __testing = {
buildPluginLoaderJitiOptions,
listPluginSdkAliasCandidates,
listPluginSdkExportedSubpaths,
resolvePluginSdkAliasCandidateOrder,
@@ -839,15 +855,7 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi
...(pluginSdkAlias ? { "openclaw/plugin-sdk": pluginSdkAlias } : {}),
...resolvePluginSdkScopedAliasMap(),
};
jitiLoader = createJiti(import.meta.url, {
interopDefault: true,
extensions: [".ts", ".tsx", ".mts", ".cts", ".mtsx", ".ctsx", ".js", ".mjs", ".cjs", ".json"],
...(Object.keys(aliasMap).length > 0
? {
alias: aliasMap,
}
: {}),
});
jitiLoader = createJiti(import.meta.url, buildPluginLoaderJitiOptions(aliasMap));
return jitiLoader;
};

View File

@@ -1,8 +1,11 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { stageBundledPluginRuntime } from "../../scripts/stage-bundled-plugin-runtime.mjs";
import { discoverOpenClawPlugins } from "./discovery.js";
import { loadPluginManifestRegistry } from "./manifest-registry.js";
const tempDirs: string[] = [];
@@ -19,7 +22,7 @@ afterEach(() => {
});
describe("stageBundledPluginRuntime", () => {
it("hard-links bundled dist plugins into dist-runtime and links plugin-local node_modules", () => {
it("stages bundled dist plugins as runtime wrappers and links plugin-local node_modules", () => {
const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-");
const distPluginDir = path.join(repoRoot, "dist", "extensions", "diffs");
fs.mkdirSync(path.join(repoRoot, "dist"), { recursive: true });
@@ -39,14 +42,16 @@ describe("stageBundledPluginRuntime", () => {
const runtimePluginDir = path.join(repoRoot, "dist-runtime", "extensions", "diffs");
expect(fs.existsSync(path.join(runtimePluginDir, "index.js"))).toBe(true);
expect(fs.statSync(path.join(runtimePluginDir, "index.js")).nlink).toBeGreaterThan(1);
expect(fs.readFileSync(path.join(runtimePluginDir, "index.js"), "utf8")).toContain(
"../../../dist/extensions/diffs/index.js",
);
expect(fs.lstatSync(path.join(runtimePluginDir, "node_modules")).isSymbolicLink()).toBe(true);
expect(fs.realpathSync(path.join(runtimePluginDir, "node_modules"))).toBe(
fs.realpathSync(sourcePluginNodeModulesDir),
);
});
it("hard-links top-level dist chunks so staged bundled plugins keep relative imports working", () => {
it("writes wrappers that forward plugin entry imports into canonical dist files", async () => {
const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-chunks-");
fs.mkdirSync(path.join(repoRoot, "dist", "extensions", "diffs"), { recursive: true });
fs.writeFileSync(
@@ -62,19 +67,138 @@ describe("stageBundledPluginRuntime", () => {
stageBundledPluginRuntime({ repoRoot });
const runtimeChunkPath = path.join(repoRoot, "dist-runtime", "chunk-abc.js");
expect(fs.readFileSync(runtimeChunkPath, "utf8")).toContain("value = 1");
expect(fs.statSync(runtimeChunkPath).nlink).toBeGreaterThan(1);
expect(
fs.readFileSync(
path.join(repoRoot, "dist-runtime", "extensions", "diffs", "index.js"),
"utf8",
const runtimeEntryPath = path.join(repoRoot, "dist-runtime", "extensions", "diffs", "index.js");
expect(fs.readFileSync(runtimeEntryPath, "utf8")).toContain(
"../../../dist/extensions/diffs/index.js",
);
expect(fs.existsSync(path.join(repoRoot, "dist-runtime", "chunk-abc.js"))).toBe(false);
const runtimeModule = await import(`${pathToFileURL(runtimeEntryPath).href}?t=${Date.now()}`);
expect(runtimeModule.value).toBe(1);
});
it("copies package metadata files but symlinks other non-js plugin artifacts into the runtime overlay", () => {
const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-assets-");
const distPluginDir = path.join(repoRoot, "dist", "extensions", "diffs");
fs.mkdirSync(path.join(distPluginDir, "assets"), { recursive: true });
fs.writeFileSync(
path.join(distPluginDir, "package.json"),
JSON.stringify(
{ name: "@openclaw/diffs", openclaw: { extensions: ["./index.js"] } },
null,
2,
),
).toContain("../../chunk-abc.js");
const distChunkStats = fs.statSync(path.join(repoRoot, "dist", "chunk-abc.js"));
const runtimeChunkStats = fs.statSync(runtimeChunkPath);
expect(runtimeChunkStats.ino).toBe(distChunkStats.ino);
expect(runtimeChunkStats.dev).toBe(distChunkStats.dev);
"utf8",
);
fs.writeFileSync(path.join(distPluginDir, "openclaw.plugin.json"), "{}\n", "utf8");
fs.writeFileSync(path.join(distPluginDir, "assets", "info.txt"), "ok\n", "utf8");
stageBundledPluginRuntime({ repoRoot });
const runtimePackagePath = path.join(
repoRoot,
"dist-runtime",
"extensions",
"diffs",
"package.json",
);
const runtimeManifestPath = path.join(
repoRoot,
"dist-runtime",
"extensions",
"diffs",
"openclaw.plugin.json",
);
const runtimeAssetPath = path.join(
repoRoot,
"dist-runtime",
"extensions",
"diffs",
"assets",
"info.txt",
);
expect(fs.lstatSync(runtimePackagePath).isSymbolicLink()).toBe(false);
expect(fs.readFileSync(runtimePackagePath, "utf8")).toContain('"extensions": [');
expect(fs.lstatSync(runtimeManifestPath).isSymbolicLink()).toBe(false);
expect(fs.readFileSync(runtimeManifestPath, "utf8")).toBe("{}\n");
expect(fs.lstatSync(runtimeAssetPath).isSymbolicLink()).toBe(true);
expect(fs.readFileSync(runtimeAssetPath, "utf8")).toBe("ok\n");
});
it("preserves package metadata needed for bundled plugin discovery from dist-runtime", () => {
const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-discovery-");
const distPluginDir = path.join(repoRoot, "dist", "extensions", "demo");
const runtimeExtensionsDir = path.join(repoRoot, "dist-runtime", "extensions");
fs.mkdirSync(distPluginDir, { recursive: true });
fs.writeFileSync(
path.join(distPluginDir, "package.json"),
JSON.stringify(
{
name: "@openclaw/demo",
openclaw: {
extensions: ["./main.js"],
setupEntry: "./setup.js",
startup: {
deferConfiguredChannelFullLoadUntilAfterListen: true,
},
},
},
null,
2,
),
"utf8",
);
fs.writeFileSync(
path.join(distPluginDir, "openclaw.plugin.json"),
JSON.stringify(
{
id: "demo",
channels: ["demo"],
configSchema: { type: "object" },
},
null,
2,
),
"utf8",
);
fs.writeFileSync(path.join(distPluginDir, "main.js"), "export default {};\n", "utf8");
fs.writeFileSync(path.join(distPluginDir, "setup.js"), "export default {};\n", "utf8");
stageBundledPluginRuntime({ repoRoot });
const env = {
...process.env,
OPENCLAW_BUNDLED_PLUGINS_DIR: runtimeExtensionsDir,
};
const discovery = discoverOpenClawPlugins({
env,
cache: false,
});
const manifestRegistry = loadPluginManifestRegistry({
env,
cache: false,
candidates: discovery.candidates,
diagnostics: discovery.diagnostics,
});
const expectedRuntimeMainPath = fs.realpathSync(
path.join(runtimeExtensionsDir, "demo", "main.js"),
);
const expectedRuntimeSetupPath = fs.realpathSync(
path.join(runtimeExtensionsDir, "demo", "setup.js"),
);
expect(discovery.candidates).toHaveLength(1);
expect(fs.realpathSync(discovery.candidates[0]?.source ?? "")).toBe(expectedRuntimeMainPath);
expect(fs.realpathSync(discovery.candidates[0]?.setupSource ?? "")).toBe(
expectedRuntimeSetupPath,
);
expect(fs.realpathSync(manifestRegistry.plugins[0]?.setupSource ?? "")).toBe(
expectedRuntimeSetupPath,
);
expect(manifestRegistry.plugins[0]?.startupDeferConfiguredChannelFullLoadUntilAfterListen).toBe(
true,
);
});
it("removes stale runtime plugin directories that are no longer in dist", () => {

View File

@@ -1,13 +1,30 @@
import fs from "node:fs";
import path from "node:path";
import { defineConfig } from "tsdown";
import { defineConfig, type UserConfig } from "tsdown";
import { buildPluginSdkEntrySources } from "./scripts/lib/plugin-sdk-entries.mjs";
type InputOptionsFactory = Extract<NonNullable<UserConfig["inputOptions"]>, Function>;
type InputOptionsArg = InputOptionsFactory extends (
options: infer Options,
format: infer _Format,
context: infer _Context,
) => infer _Return
? Options
: never;
type InputOptionsReturn = InputOptionsFactory extends (
options: infer _Options,
format: infer _Format,
context: infer _Context,
) => infer Return
? Return
: never;
type OnLogFunction = InputOptionsArg extends { onLog?: infer OnLog } ? NonNullable<OnLog> : never;
const env = {
NODE_ENV: "production",
};
function buildInputOptions(options: { onLog?: unknown; [key: string]: unknown }) {
function buildInputOptions(options: InputOptionsArg): InputOptionsReturn {
if (process.env.OPENCLAW_BUILD_VERBOSE === "1") {
return undefined;
}
@@ -32,11 +49,8 @@ function buildInputOptions(options: { onLog?: unknown; [key: string]: unknown })
return {
...options,
onLog(
level: string,
log: { code?: string; message?: string; id?: string; importer?: string },
defaultHandler: (level: string, log: { code?: string }) => void,
) {
onLog(...args: Parameters<OnLogFunction>) {
const [level, log, defaultHandler] = args;
if (isSuppressedLog(log)) {
return;
}
@@ -49,7 +63,7 @@ function buildInputOptions(options: { onLog?: unknown; [key: string]: unknown })
};
}
function nodeBuildConfig(config: Record<string, unknown>) {
function nodeBuildConfig(config: UserConfig): UserConfig {
return {
...config,
env,
@@ -112,6 +126,33 @@ function listBundledPluginBuildEntries(): Record<string, string> {
const bundledPluginBuildEntries = listBundledPluginBuildEntries();
function buildBundledHookEntries(): Record<string, string> {
const hooksRoot = path.join(process.cwd(), "src", "hooks", "bundled");
const entries: Record<string, string> = {};
if (!fs.existsSync(hooksRoot)) {
return entries;
}
for (const dirent of fs.readdirSync(hooksRoot, { withFileTypes: true })) {
if (!dirent.isDirectory()) {
continue;
}
const hookName = dirent.name;
const handlerPath = path.join(hooksRoot, hookName, "handler.ts");
if (!fs.existsSync(handlerPath)) {
continue;
}
entries[`bundled/${hookName}/handler`] = handlerPath;
}
return entries;
}
const bundledHookEntries = buildBundledHookEntries();
function buildCoreDistEntries(): Record<string, string> {
return {
index: "src/index.ts",
@@ -130,33 +171,34 @@ function buildCoreDistEntries(): Record<string, string> {
"line/accounts": "src/line/accounts.ts",
"line/send": "src/line/send.ts",
"line/template-messages": "src/line/template-messages.ts",
"plugins/runtime/index": "src/plugins/runtime/index.ts",
"llm-slug-generator": "src/hooks/llm-slug-generator.ts",
};
}
const coreDistEntries = buildCoreDistEntries();
function buildUnifiedDistEntries(): Record<string, string> {
return {
...coreDistEntries,
...Object.fromEntries(
Object.entries(buildPluginSdkEntrySources()).map(([entry, source]) => [
`plugin-sdk/${entry}`,
source,
]),
),
...bundledPluginBuildEntries,
...bundledHookEntries,
};
}
export default defineConfig([
nodeBuildConfig({
// Build the root dist entrypoints together so they can share hashed chunks
// instead of emitting near-identical copies across separate builds.
entry: coreDistEntries,
}),
nodeBuildConfig({
// Bundle all plugin-sdk entries in a single build so the bundler can share
// common chunks instead of duplicating them per entry (~712MB heap saved).
entry: buildPluginSdkEntrySources(),
outDir: "dist/plugin-sdk",
}),
nodeBuildConfig({
// Bundle bundled plugin entrypoints so built gateway startup can load JS
// directly from dist/extensions instead of transpiling extensions/*.ts via Jiti.
entry: bundledPluginBuildEntries,
outDir: "dist",
// Build core entrypoints, plugin-sdk subpaths, bundled plugin entrypoints,
// and bundled hooks in one graph so runtime singletons are emitted once.
entry: buildUnifiedDistEntries(),
deps: {
neverBundle: ["@lancedb/lancedb"],
},
}),
nodeBuildConfig({
entry: ["src/hooks/bundled/*/handler.ts", "src/hooks/llm-slug-generator.ts"],
}),
]);