perf(wasm): publish compressed binaries outside JS glue

packages/libdatadog/README.md

@@ -17,8 +17,8 @@ The package accepts Datadog v0.4 MessagePack payloads and exports them to an
 agentless intake. `sendV04()` reports completion through a callback and sends
 delivery failures to the supplied logger. It does not return a promise.
 
-The package uses a wasm-bindgen backend with the WebAssembly bytes embedded in
-JavaScript. The canonical inlined output is published as the regular
-`@datadog/libdatadog-wasm` dependency from the `wasm` workspace. The WASM
-package also contains the separate remote configuration artifact. No raw
-`.wasm` asset or native extension is published.
+The package publishes Brotli-compressed `.wasm.br` files next to the
+wasm-bindgen JavaScript loaders. Each loader reads and decompresses its file
+synchronously before instantiation. A bundled distribution must copy the
+referenced asset next to its output JavaScript file. No raw `.wasm` asset or
+native extension is published.

packages/libdatadog/package.json

@@ -53,12 +53,12 @@
     "node": ">=18"
   },
   "scripts": {
-    "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run inline:wasm && npm run inline:remote-config",
+    "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run compress:wasm && npm run compress:remote-config",
     "build:wasm:binary": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm wasm/dist",
     "build:remote-config:binary": "node ../../scripts/build-wasm.js ../../crates/remote_config wasm/dist/remote-config",
     "size:profile": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm ../../target/size --profiling",
-    "inline:wasm": "node scripts/inline-wasm.js libdatadog_wasm wasm/dist",
-    "inline:remote-config": "node scripts/inline-wasm.js remote_config wasm/dist/remote-config",
+    "compress:wasm": "node scripts/compress-wasm.js libdatadog_wasm wasm/dist",
+    "compress:remote-config": "node scripts/compress-wasm.js remote_config wasm/dist/remote-config",
     "report:wasm-size": "node scripts/report-wasm-size.js",
     "build": "npm run build:wasm",
     "test": "node scripts/run-tests.js && npm run test:types",

packages/libdatadog/scripts/inline-wasm.js → packages/libdatadog/scripts/compress-wasm.js

@@ -7,35 +7,41 @@ const { constants, brotliCompressSync } = require('node:zlib')
 const [moduleName, relativeOutputDirectory] = process.argv.slice(2)
 
 if (!moduleName || !relativeOutputDirectory) {
-  throw new Error('usage: node scripts/inline-wasm.js <module-name> <output-directory>')
+  throw new Error('usage: node scripts/compress-wasm.js <module-name> <output-directory>')
 }
 
 const outputDirectory = path.join(__dirname, '..', relativeOutputDirectory)
 const gluePath = path.join(outputDirectory, `${moduleName}.js`)
 const wasmPath = path.join(outputDirectory, `${moduleName}_bg.wasm`)
+const compressedWasmPath = `${wasmPath}.br`
 const glue = fs.readFileSync(gluePath, 'utf8')
 const wasm = fs.readFileSync(wasmPath)
-const encodedWasm = brotliCompressSync(wasm, {
+const compressedWasm = brotliCompressSync(wasm, {
   params: {
     [constants.BROTLI_PARAM_QUALITY]: 11,
   },
-}).toString('base64')
+})
 const loader = [
   `const wasmPath = \`\${__dirname}/${moduleName}_bg.wasm\`;`,
   'const wasmBytes = require(\'fs\').readFileSync(wasmPath);',
 ].join('\n')
 
 if (!glue.includes(loader)) {
-  throw new Error('wasm-bindgen loader changed; refusing to publish an external WASM asset')
+  throw new Error('wasm-bindgen loader changed; refusing to publish a compressed WASM asset')
 }
 
 fs.writeFileSync(
   gluePath,
   glue.replace(
     loader,
-    () => `const wasmBytes = require('node:zlib').brotliDecompressSync(Buffer.from('${encodedWasm}', 'base64'));`,
+    () => [
+      `const compressedWasmPath = \`\${__dirname}/${moduleName}_bg.wasm.br\`;`,
+      'const compressedWasm = require(\'node:fs\').readFileSync(compressedWasmPath);',
+      'const wasmBytes = require(\'node:zlib\').brotliDecompressSync(compressedWasm);',
+    ].join('\n'),
   ),
 )
+fs.writeFileSync(compressedWasmPath, compressedWasm)
 fs.rmSync(wasmPath)
 fs.rmSync(path.join(outputDirectory, '.gitignore'), { force: true })
 fs.rmSync(path.join(outputDirectory, 'package.json'), { force: true })

packages/libdatadog/scripts/report-wasm-size.js

@@ -307,26 +307,21 @@ function appendCrateReport (lines, profilePath) {
 
 function createReport (gluePath, profilePath) {
   const glue = fs.readFileSync(gluePath, 'utf8')
-  const match = glue.match(/Buffer\.from\('([A-Za-z0-9+/=]+)', 'base64'\)/)
-  if (!match) throw new Error('could not find the inline base64 WASM payload')
-
-  const base64Bytes = Buffer.byteLength(match[1])
-  const compressed = Buffer.from(match[1], 'base64')
+  const compressedPath = `${gluePath.slice(0, -3)}_bg.wasm.br`
+  const compressed = fs.readFileSync(compressedPath)
   const wasm = brotliDecompressSync(compressed)
-  const glueBytes = Buffer.byteLength(glue) - base64Bytes
-  const inlineBytes = Buffer.byteLength(glue)
-  const base64Overhead = base64Bytes - compressed.length
+  const glueBytes = Buffer.byteLength(glue)
+  const packagedBytes = glueBytes + compressed.length
   const sections = readSections(wasm)
   const lines = [
     '## libdatadog WASM size',
     '',
-    '| Inline artifact layer | Bytes | KiB |',
+    '| Packaged artifact layer | Bytes | KiB |',
     '| --- | ---: | ---: |',
     layerRow('Raw WASM (before Brotli)', wasm.length),
     layerRow('Brotli-compressed WASM', compressed.length),
-    layerRow('Base64 encoding overhead', base64Overhead),
     layerRow('JavaScript glue/loader', glueBytes),
-    layerRow('Final inlined JavaScript', inlineBytes, true),
+    layerRow('Final packaged artifacts', packagedBytes, true),
     '',
     '### Raw WebAssembly sections',
     '',
@@ -344,7 +339,11 @@ function createReport (gluePath, profilePath) {
 
   if (profilePath) appendCrateReport(lines, profilePath)
 
-  lines.push('', `Generated from \`${path.relative(process.cwd(), gluePath)}\`.`)
+  lines.push(
+    '',
+    `Generated from \`${path.relative(process.cwd(), gluePath)}\` and `
+    + `\`${path.relative(process.cwd(), compressedPath)}\`.`,
+  )
   return lines.join('\n')
 }
 

packages/libdatadog/test/bundlers.test.js

@@ -13,36 +13,52 @@ const webpack = require('webpack')
 const packageRoot = path.join(__dirname, '..')
 const webpackAsync = promisify(webpack)
 const entries = new Map([
-  ['package', path.join(packageRoot, 'index.js')],
-  ['WASM', path.join(packageRoot, 'wasm.js')],
-  ['remote config', path.join(packageRoot, 'remote-config.js')],
+  ['package', {
+    asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'),
+    entry: path.join(packageRoot, 'index.js'),
+  }],
+  ['WASM', {
+    asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'),
+    entry: path.join(packageRoot, 'wasm.js'),
+  }],
+  ['remote config', {
+    asset: path.join(packageRoot, 'wasm', 'dist', 'remote-config', 'remote_config_bg.wasm.br'),
+    entry: path.join(packageRoot, 'remote-config.js'),
+  }],
 ])
 
-for (const [name, entry] of entries) {
-  test(`esbuild bundles the ${name} entry point without emitting an asset`, async () => {
-    await assertBundle(entry, bundleWithEsbuild)
+for (const [name, { asset, entry }] of entries) {
+  test(`esbuild bundles the ${name} entry point with its compressed WASM asset`, async () => {
+    await assertBundle(entry, asset, bundleWithEsbuild, name !== 'remote config')
   })
 
-  test(`webpack bundles the ${name} entry point without emitting an asset`, async () => {
-    await assertBundle(entry, bundleWithWebpack)
+  test(`webpack bundles the ${name} entry point with its compressed WASM asset`, async () => {
+    await assertBundle(entry, asset, bundleWithWebpack, true)
   })
 }
 
 /**
  * @param {string} entry
+ * @param {string} asset
  * @param {(entry: string, output: string) => Promise<void>} bundle
+ * @param {boolean} loadBundle
  */
-async function assertBundle (entry, bundle) {
+async function assertBundle (entry, asset, bundle, loadBundle) {
   const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'libdatadog-bundle-'))
   const output = path.join(directory, 'bundle.cjs')
+  const copiedAsset = path.join(directory, path.basename(asset))
 
   try {
     await bundle(entry, output)
+    fs.copyFileSync(asset, copiedAsset)
     const files = fs.readdirSync(directory)
     const contents = fs.readFileSync(output, 'utf8')
 
-    assert.deepStrictEqual(files, ['bundle.cjs'])
-    assert.doesNotMatch(contents, /\.wasm(?:['"`)]|$)/m)
+    assert.equal(files.length, 2)
+    assert(files.includes('bundle.cjs'))
+    assert(files.includes(path.basename(asset)))
+    assert.match(contents, /\.wasm\.br/)
+    if (loadBundle) require(output)
   } finally {
     fs.rmSync(directory, { force: true, recursive: true })
   }

packages/libdatadog/test/package-contents.test.js

@@ -35,11 +35,16 @@ test('published packages contain only the intended artifacts', () => {
   const wasmNames = libdatadogWasm.files.map(file => file.path)
   const standaloneWasm = [...names, ...wasmNames]
     .filter(file => file.endsWith('.wasm'))
+  const compressedWasm = wasmNames.filter(file => file.endsWith('.wasm.br'))
 
   assert.deepStrictEqual(standaloneWasm, [],
-    `packages must not contain standalone WASM files: ${standaloneWasm.join(', ')}`)
+    `packages must not contain raw WASM files: ${standaloneWasm.join(', ')}`)
+  assert.deepStrictEqual(compressedWasm, [
+    'dist/libdatadog_wasm_bg.wasm.br',
+    'dist/remote-config/remote_config_bg.wasm.br',
+  ])
   assert(wasmNames.includes('dist/libdatadog_wasm.js'),
-    'WASM package must contain the inline-WASM JavaScript fallback')
+    'WASM package must contain the JavaScript loader')
   assert(wasmNames.includes('dist/remote-config/remote_config.js'),
     'WASM package must contain the dedicated remote config artifact')
   assert.strictEqual(wasmNames.includes('remote-config.js'), false)

packages/libdatadog/test/package.test.js

@@ -69,44 +69,44 @@ test('root entry point uses the WASM backend', () => {
   assert.strictEqual(libdatadog.backend(), 'wasm')
 })
 
-test('embeds a Brotli-compressed WASM fallback below the size budgets', () => {
-  const { glue, wasm } = readInlineWasm(path.join(
+test('packages a Brotli-compressed WASM fallback below the size budgets', () => {
+  const { packagedBytes, wasm } = readPackagedWasm(path.join(
     packageRoot,
     'wasm',
     'dist',
     'libdatadog_wasm.js',
   ))
 
-  assert.ok(Buffer.byteLength(glue) < 260 * 1024)
+  assert.ok(packagedBytes < 260 * 1024)
   assert.ok(wasm.length < 600 * 1024)
 })
 
-test('embeds dedicated remote config WASM below the size budgets', () => {
-  const { glue, wasm } = readInlineWasm(path.join(
+test('packages dedicated remote config WASM below the size budgets', () => {
+  const { packagedBytes, wasm } = readPackagedWasm(path.join(
     packageRoot,
     'wasm',
     'dist',
     'remote-config',
     'remote_config.js',
   ))
 
-  assert.ok(Buffer.byteLength(glue) < 450 * 1024)
+  assert.ok(packagedBytes < 450 * 1024)
   assert.ok(wasm.length < 1024 * 1024)
 })
 
-test('requires an artifact name and output directory when inlining WASM', () => {
-  const script = path.join(packageRoot, 'scripts', 'inline-wasm.js')
+test('requires an artifact name and output directory when compressing WASM', () => {
+  const script = path.join(packageRoot, 'scripts', 'compress-wasm.js')
 
   for (const scriptArguments of [[], ['fixture']]) {
     const result = spawnSync(process.execPath, [script, ...scriptArguments])
 
     assert.notStrictEqual(result.status, 0)
-    assert.match(result.stderr.toString(), /usage: node scripts\/inline-wasm\.js/)
+    assert.match(result.stderr.toString(), /usage: node scripts\/compress-wasm\.js/)
   }
 })
 
-test('inlines a named WASM artifact into its generated module', () => {
-  const outputDirectory = fs.mkdtempSync(path.join(packageRoot, '.inline-wasm-'))
+test('compresses a named WASM artifact beside its generated module', () => {
+  const outputDirectory = fs.mkdtempSync(path.join(packageRoot, '.compress-wasm-'))
   const moduleName = 'fixture'
   const wasm = Buffer.from('fixture WASM')
   const loader = [
@@ -121,14 +121,15 @@ test('inlines a named WASM artifact into its generated module', () => {
     fs.writeFileSync(path.join(outputDirectory, 'package.json'), '{}')
 
     const result = spawnSync(process.execPath, [
-      path.join(packageRoot, 'scripts', 'inline-wasm.js'),
+      path.join(packageRoot, 'scripts', 'compress-wasm.js'),
       moduleName,
       path.relative(packageRoot, outputDirectory),
     ])
 
     assert.strictEqual(result.status, 0, result.stderr.toString())
-    assert.deepStrictEqual(readInlineWasm(path.join(outputDirectory, `${moduleName}.js`)).wasm, wasm)
+    assert.deepStrictEqual(readPackagedWasm(path.join(outputDirectory, `${moduleName}.js`)).wasm, wasm)
     assert.strictEqual(fs.existsSync(path.join(outputDirectory, `${moduleName}_bg.wasm`)), false)
+    assert.strictEqual(fs.existsSync(path.join(outputDirectory, `${moduleName}_bg.wasm.br`)), true)
     assert.strictEqual(fs.existsSync(path.join(outputDirectory, '.gitignore')), false)
     assert.strictEqual(fs.existsSync(path.join(outputDirectory, 'package.json')), false)
   } finally {
@@ -139,13 +140,15 @@ test('inlines a named WASM artifact into its generated module', () => {
 /**
  * @param {string} gluePath
  */
-function readInlineWasm (gluePath) {
+function readPackagedWasm (gluePath) {
   const glue = fs.readFileSync(gluePath, 'utf8')
-  const encodedWasm = glue.match(/brotliDecompressSync\(Buffer\.from\('([^']+)', 'base64'\)\)/)?.[1]
+  const compressedWasmPath = gluePath.replace(/\.js$/, '_bg.wasm.br')
+  const compressedWasm = fs.readFileSync(compressedWasmPath)
 
-  assert.ok(encodedWasm, 'WASM must be embedded as a Brotli-compressed base64 string')
+  assert.match(glue, /brotliDecompressSync\(compressedWasm\)/)
   return {
     glue,
-    wasm: brotliDecompressSync(Buffer.from(encodedWasm, 'base64')),
+    packagedBytes: Buffer.byteLength(glue) + compressedWasm.length,
+    wasm: brotliDecompressSync(compressedWasm),
   }
 }

packages/libdatadog/test/size-report.test.js

@@ -10,12 +10,12 @@ const {
   inferCrate,
   readSections,
 } = require('../scripts/report-wasm-size')
-test('reports inline packaging and WASM section sizes', () => {
+test('reports compressed packaging and WASM section sizes', () => {
   const gluePath = path.join(__dirname, '..', 'wasm', 'dist', 'libdatadog_wasm.js')
   const report = createWasmReport(gluePath)
 
   assert.match(report, /Raw WASM \(before Brotli\)/)
-  assert.match(report, /Base64 encoding overhead/)
+  assert.match(report, /Final packaged artifacts/)
   assert.match(report, /Raw WebAssembly sections/)
   assert.match(report, /\| code \|/)
   assert.match(report, /\| data \|/)