npm packaging for components (#29129)

Change module resolution logic to not always use `require()` resolution rules for component definitions.

GitOrigin-RevId: 6e1540240406ac60ccb3704375eb3c03267e3211

Author: thomasballinger

src/bundler/index.test.ts

@@ -175,6 +175,7 @@ test("must use isolate", () => {
   expect(mustBeIsolate("schema.js")).toBeTruthy();
   expect(mustBeIsolate("schema.jsx")).toBeTruthy();
   expect(mustBeIsolate("schema.ts")).toBeTruthy();
+  expect(mustBeIsolate("schema.js")).toBeTruthy();
 
   expect(mustBeIsolate("http.sample.js")).not.toBeTruthy();
   expect(mustBeIsolate("https.js")).not.toBeTruthy();

src/bundler/index.ts

@@ -245,13 +245,11 @@ async function externalPackageVersions(
 }
 
 export async function bundleSchema(ctx: Context, dir: string) {
-  const result = await bundle(
-    ctx,
-    dir,
-    [path.resolve(dir, "schema.ts")],
-    true,
-    "browser",
-  );
+  let target = path.resolve(dir, "schema.ts");
+  if (!ctx.fs.exists(target)) {
+    target = path.resolve(dir, "schema.js");
+  }
+  const result = await bundle(ctx, dir, [target], true, "browser");
   return result.modules;
 }
 
@@ -353,7 +351,7 @@ export async function entryPoints(
       log(chalk.yellow(`Skipping ${fpath}`));
     } else if (base === "_generated.ts") {
       log(chalk.yellow(`Skipping ${fpath}`));
-    } else if (base === "schema.ts") {
+    } else if (base === "schema.ts" || base === "schema.js") {
       log(chalk.yellow(`Skipping ${fpath}`));
     } else if ((base.match(/\./g) || []).length > 1) {
       log(chalk.yellow(`Skipping ${fpath} that contains multiple dots`));

src/cli/codegen_templates/component_api.ts

@@ -15,6 +15,7 @@ import {
 import { Identifier } from "../lib/deployApi/types.js";
 import { ComponentDefinitionPath } from "../lib/deployApi/paths.js";
 import { resolveFunctionReference } from "./component_server.js";
+import { encodeDefinitionPath } from "../lib/components/definition/bundle.js";
 
 export function componentApiJs() {
   const lines = [];
@@ -155,12 +156,14 @@ async function buildMountTree(
   definitionPath: ComponentDefinitionPath,
   attributes: string[],
 ): Promise<MountTree | null> {
-  const analysis = startPush.analysis[definitionPath];
+  // TODO make these types more precise when receiving analysis from server
+  const analysis =
+    startPush.analysis[encodeDefinitionPath(definitionPath as any)];
   if (!analysis) {
     return await ctx.crash({
       exitCode: 1,
       errorType: "fatal",
-      printedMessage: `No analysis found for component ${definitionPath}`,
+      printedMessage: `No analysis found for component ${encodeDefinitionPath(definitionPath as any)} orig: ${definitionPath}\nin\n${Object.keys(startPush.analysis).toString()}`,
     });
   }
   let current = analysis.definition.exports.branch;

src/cli/codegen_templates/component_server.ts

@@ -17,6 +17,7 @@ import { Context } from "../../bundler/context.js";
 import { CanonicalizedModulePath } from "../lib/deployApi/paths.js";
 import { Value, jsonToConvex } from "../../values/value.js";
 import { z } from "zod";
+import { encodeDefinitionPath } from "../lib/components/definition/bundle.js";
 
 export function componentServerJS(isRoot: boolean): string {
   let result = `
@@ -287,12 +288,12 @@ export async function componentServerDTS(
     componentDirectory,
   );
 
-  const analysis = startPush.analysis[definitionPath];
+  const analysis = startPush.analysis[encodeDefinitionPath(definitionPath)];
   if (!analysis) {
     return await ctx.crash({
       exitCode: 1,
       errorType: "fatal",
-      printedMessage: `No analysis found for component ${definitionPath}`,
+      printedMessage: `No analysis found for component ${encodeDefinitionPath(definitionPath as any)} orig: ${definitionPath}\nin\n${Object.keys(startPush.analysis).toString()}`,
     });
   }
   for (const childComponent of analysis.definition.childComponents) {

src/cli/lib/codegen.ts

@@ -145,6 +145,25 @@ export async function doInitialComponentCodegen(
   opts?: { dryRun?: boolean; generateCommonJSApi?: boolean; debug?: boolean },
 ) {
   const { projectConfig } = await readProjectConfig(ctx);
+
+  // This component defined in a dist directory; it is probably in a node_module
+  // directory, installed from a package. It is stuck with the files it has.
+  // Heuristics for this:
+  // - component definition has a dist/ directory as an ancestor
+  // - component definition is a .js file
+  // - presence of .js.map files
+  // We may improve this heuristic.
+  const isPublishedPackage =
+    componentDirectory.definitionPath.endsWith(".js") &&
+    !componentDirectory.isRoot;
+  if (isPublishedPackage) {
+    logMessage(
+      ctx,
+      `skipping initial codegen for installed package ${componentDirectory.path}`,
+    );
+    return;
+  }
+
   const codegenDir = await prepareForCodegen(
     ctx,
     componentDirectory.path,
@@ -207,6 +226,14 @@ export async function doFinalComponentCodegen(
   opts?: { dryRun?: boolean; debug?: boolean; generateCommonJSApi?: boolean },
 ) {
   const { projectConfig } = await readProjectConfig(ctx);
+
+  const isPublishedPackage =
+    componentDirectory.definitionPath.endsWith(".js") &&
+    !componentDirectory.isRoot;
+  if (isPublishedPackage) {
+    return;
+  }
+
   const codegenDir = path.join(componentDirectory.path, "_generated");
   ctx.fs.mkdir(codegenDir, { allowExisting: true, recursive: true });
 
@@ -308,8 +335,12 @@ async function doDataModelCodegen(
   codegenDir: string,
   opts?: { dryRun?: boolean; debug?: boolean },
 ) {
-  const schemaPath = path.join(functionsDir, "schema.ts");
-  const hasSchemaFile = ctx.fs.exists(schemaPath);
+  let schemaPath = path.join(functionsDir, "schema.ts");
+  let hasSchemaFile = ctx.fs.exists(schemaPath);
+  if (!hasSchemaFile) {
+    schemaPath = path.join(functionsDir, "schema.js");
+    hasSchemaFile = ctx.fs.exists(schemaPath);
+  }
   const schemaContent = hasSchemaFile ? dataModel : dataModelWithoutSchema;
 
   await writeFormattedFile(

src/cli/lib/components.ts

@@ -206,16 +206,13 @@ async function startComponentsPushAndCodegen(
   const componentDefinitions: ComponentDefinitionConfig[] = [];
   for (const componentDefinition of componentDefinitionSpecsWithoutImpls) {
     const impl = componentImplementations.filter(
-      (impl) =>
-        // convert from ComponentPath
-        path.resolve(rootComponent.path, impl.definitionPath) ===
-        componentDefinition.definitionPath,
+      (impl) => impl.definitionPath === componentDefinition.definitionPath,
     )[0];
     if (!impl) {
       return await ctx.crash({
         exitCode: 1,
         errorType: "fatal",
-        printedMessage: `missing! couldn't find ${componentDefinition.definitionPath} in ${componentImplementations.map((impl) => path.resolve(rootComponent.path, impl.definitionPath)).toString()}`,
+        printedMessage: `missing! couldn't find ${componentDefinition.definitionPath} in ${componentImplementations.map((impl) => impl.definitionPath).toString()}`,
       });
     }
     componentDefinitions.push({

src/cli/lib/components/constants.ts

@@ -1,2 +1,3 @@
 export const ROOT_DEFINITION_FILENAME = "convex.config.ts";
 export const DEFINITION_FILENAME = "convex.config.ts";
+export const COMPILED_DEFINITION_FILENAME = "convex.config.js";

src/cli/lib/components/definition/bundle.ts

@@ -1,11 +1,13 @@
 import path from "path";
+import crypto from "crypto";
 import {
   ComponentDirectory,
   ComponentDefinitionPath,
   buildComponentDirectory,
   isComponentDirectory,
   qualifiedDefinitionPath,
   toComponentDefinitionPath,
+  EncodedComponentDefinitionPath,
 } from "./directoryStructure.js";
 import {
   Context,
@@ -15,7 +17,6 @@ import {
 } from "../../../../bundler/context.js";
 import esbuild, { BuildOptions, Metafile, OutputFile, Plugin } from "esbuild";
 import chalk from "chalk";
-import { createRequire } from "module";
 import {
   AppDefinitionSpecWithoutImpls,
   ComponentDefinitionSpecWithoutImpls,
@@ -93,22 +94,15 @@ function componentPlugin({
         }
         let resolvedPath = undefined;
         for (const candidate of candidates) {
-          try {
-            // --experimental-import-meta-resolve is required for
-            // `import.meta.resolve` so we'll use `require.resolve`
-            // until then. Hopefully they aren't too different.
-            const require = createRequire(args.resolveDir);
-            resolvedPath = require.resolve(candidate, {
-              paths: [args.resolveDir],
-            });
+          const result = await build.resolve(candidate, {
+            // We expect this to be "import-statement" but pass 'kind' through
+            // to say honest to normal esbuild behavior.
+            kind: args.kind,
+            resolveDir: args.resolveDir,
+          });
+          if (result.path) {
+            resolvedPath = result.path;
             break;
-          } catch (e: any) {
-            if (e.code === "MODULE_NOT_FOUND") {
-              continue;
-            }
-            // We always invoke esbuild in a try/catch.
-            // eslint-disable-next-line no-restricted-syntax
-            throw e;
           }
         }
         if (resolvedPath === undefined) {
@@ -129,7 +123,12 @@ function componentPlugin({
         }
 
         verbose &&
-          logMessage(ctx, "  -> Component import! Recording it.", args.path);
+          logMessage(
+            ctx,
+            "  -> Component import! Recording it.",
+            args.path,
+            resolvedPath,
+          );
 
         if (mode === "discover") {
           return {
@@ -146,7 +145,7 @@ function componentPlugin({
             rootComponentDirectory,
             imported,
           );
-          const encodedPath = hackyMapping(componentPath);
+          const encodedPath = hackyMapping(encodeDefinitionPath(componentPath));
           return {
             path: encodedPath,
             external: true,
@@ -158,7 +157,7 @@ function componentPlugin({
 }
 
 /** The path on the deployment that identifier a component definition. */
-function hackyMapping(componentPath: ComponentDefinitionPath): string {
+function hackyMapping(componentPath: EncodedComponentDefinitionPath): string {
   return `./_componentDeps/${Buffer.from(componentPath).toString("base64").replace(/=+$/, "")}`;
 }
 
@@ -312,6 +311,24 @@ async function findComponentDependencies(
   return { components, dependencyGraph };
 }
 
+// Each path component is less than 64 bytes and escape all a-zA-Z0-9
+// This is the only version of the path the server will receive.
+export function encodeDefinitionPath(
+  s: ComponentDefinitionPath,
+): EncodedComponentDefinitionPath {
+  const components = s.split(path.sep);
+  return components
+    .map((s) => {
+      const escaped = s.replaceAll("-", "_").replaceAll("+", "_");
+      if (escaped.length <= 64) {
+        return escaped;
+      }
+      const hash = crypto.createHash("md5").update(s).digest("hex");
+      return `${escaped.slice(0, 50)}${hash.slice(0, 14)}`;
+    })
+    .join(path.sep) as EncodedComponentDefinitionPath;
+}
+
 // NB: If a directory linked to is not a member of the passed
 // componentDirectories array then there will be external links
 // with no corresponding definition bundle.
@@ -414,9 +431,15 @@ export async function bundleDefinitions(
     (out) => out.directory.path !== rootComponentDirectory.path,
   );
 
-  const componentDefinitionSpecsWithoutImpls = componentBundles.map(
-    ({ directory, outputJs, outputJsMap }) => ({
-      definitionPath: directory.path,
+  const componentDefinitionSpecsWithoutImpls: ComponentDefinitionSpecWithoutImpls[] =
+    componentBundles.map(({ directory, outputJs, outputJsMap }) => ({
+      definitionPath: encodeDefinitionPath(
+        toComponentDefinitionPath(rootComponentDirectory, directory),
+      ),
+      origDefinitionPath: toComponentDefinitionPath(
+        rootComponentDirectory,
+        directory,
+      ),
       definition: {
         path: path.relative(directory.path, outputJs.path),
         source: outputJs.text,
@@ -427,15 +450,14 @@ export async function bundleDefinitions(
         rootComponentDirectory,
         dependencyGraph,
         directory.definitionPath,
-      ),
-    }),
-  );
+      ).map(encodeDefinitionPath),
+    }));
   const appDeps = getDeps(
     rootComponentDirectory,
     dependencyGraph,
     appBundle.directory.definitionPath,
-  );
-  const appDefinitionSpecWithoutImpls = {
+  ).map(encodeDefinitionPath);
+  const appDefinitionSpecWithoutImpls: AppDefinitionSpecWithoutImpls = {
     definition: {
       path: path.relative(rootComponentDirectory.path, appBundle.outputJs.path),
       source: appBundle.outputJs.text,
@@ -465,7 +487,7 @@ export async function bundleImplementations(
   componentImplementations: {
     schema: Bundle | null;
     functions: Bundle[];
-    definitionPath: ComponentDefinitionPath;
+    definitionPath: EncodedComponentDefinitionPath;
   }[];
 }> {
   let appImplementation;
@@ -478,10 +500,12 @@ export async function bundleImplementations(
       directory.path,
     );
     let schema;
-    if (!ctx.fs.exists(path.resolve(resolvedPath, "schema.ts"))) {
-      schema = null;
-    } else {
+    if (ctx.fs.exists(path.resolve(resolvedPath, "schema.ts"))) {
+      schema = (await bundleSchema(ctx, resolvedPath))[0] || null;
+    } else if (ctx.fs.exists(path.resolve(resolvedPath, "schema.js"))) {
       schema = (await bundleSchema(ctx, resolvedPath))[0] || null;
+    } else {
+      schema = null;
     }
 
     const entryPoints = await entryPointsByEnvironment(
@@ -538,15 +562,11 @@ export async function bundleImplementations(
         externalNodeDependencies,
       };
     } else {
-      componentImplementations.push({
-        // these needs to be a componentPath when sent to the server
-        definitionPath: toComponentDefinitionPath(
-          rootComponentDirectory,
-          directory,
-        ),
-        schema,
-        functions,
-      });
+      // definitionPath is the canonical form
+      const definitionPath = encodeDefinitionPath(
+        toComponentDefinitionPath(rootComponentDirectory, directory),
+      );
+      componentImplementations.push({ definitionPath, schema, functions });
     }
     isRoot = false;
   }