From be0891682a8ad33497262f7d956cbaf842f22558 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Thu, 3 Sep 2026 05:14:48 +0200 Subject: [PATCH] perf(wasm): publish compressed binaries outside JS glue Base64-encoding Brotli data inside generated JavaScript expands the published fallback and forces parsers to process binary payloads as source. Keep the generated loader small and publish each compressed WASM binary beside it instead. The two release artifacts shrink from 588,918 to 454,783 packaged bytes (22.8%). Bundler and installed-package tests pin the required adjacent-asset contract. --- packages/libdatadog/README.md | 10 ++--- packages/libdatadog/package.json | 6 +-- .../{inline-wasm.js => compress-wasm.js} | 16 +++++--- .../libdatadog/scripts/report-wasm-size.js | 23 ++++++----- packages/libdatadog/test/bundlers.test.js | 38 +++++++++++++------ .../libdatadog/test/package-contents.test.js | 9 ++++- packages/libdatadog/test/package.test.js | 37 +++++++++--------- packages/libdatadog/test/size-report.test.js | 4 +- 8 files changed, 86 insertions(+), 57 deletions(-) rename packages/libdatadog/scripts/{inline-wasm.js => compress-wasm.js} (66%) diff --git a/packages/libdatadog/README.md b/packages/libdatadog/README.md index 3ba86705..66b88852 100644 --- a/packages/libdatadog/README.md +++ b/packages/libdatadog/README.md @@ -17,8 +17,8 @@ The package accepts Datadog v0.4 MessagePack payloads and exports them to an agentless intake. `sendV04()` reports completion through a callback and sends delivery failures to the supplied logger. It does not return a promise. -The package uses a wasm-bindgen backend with the WebAssembly bytes embedded in -JavaScript. The canonical inlined output is published as the regular -`@datadog/libdatadog-wasm` dependency from the `wasm` workspace. The WASM -package also contains the separate remote configuration artifact. No raw -`.wasm` asset or native extension is published. +The package publishes Brotli-compressed `.wasm.br` files next to the +wasm-bindgen JavaScript loaders. Each loader reads and decompresses its file +synchronously before instantiation. A bundled distribution must copy the +referenced asset next to its output JavaScript file. No raw `.wasm` asset or +native extension is published. diff --git a/packages/libdatadog/package.json b/packages/libdatadog/package.json index d308211a..2c69f58a 100644 --- a/packages/libdatadog/package.json +++ b/packages/libdatadog/package.json @@ -53,12 +53,12 @@ "node": ">=18" }, "scripts": { - "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run inline:wasm && npm run inline:remote-config", + "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run compress:wasm && npm run compress:remote-config", "build:wasm:binary": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm wasm/dist", "build:remote-config:binary": "node ../../scripts/build-wasm.js ../../crates/remote_config wasm/dist/remote-config", "size:profile": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm ../../target/size --profiling", - "inline:wasm": "node scripts/inline-wasm.js libdatadog_wasm wasm/dist", - "inline:remote-config": "node scripts/inline-wasm.js remote_config wasm/dist/remote-config", + "compress:wasm": "node scripts/compress-wasm.js libdatadog_wasm wasm/dist", + "compress:remote-config": "node scripts/compress-wasm.js remote_config wasm/dist/remote-config", "report:wasm-size": "node scripts/report-wasm-size.js", "build": "npm run build:wasm", "test": "node scripts/run-tests.js && npm run test:types", diff --git a/packages/libdatadog/scripts/inline-wasm.js b/packages/libdatadog/scripts/compress-wasm.js similarity index 66% rename from packages/libdatadog/scripts/inline-wasm.js rename to packages/libdatadog/scripts/compress-wasm.js index ea1634b8..474d828f 100644 --- a/packages/libdatadog/scripts/inline-wasm.js +++ b/packages/libdatadog/scripts/compress-wasm.js @@ -7,35 +7,41 @@ const { constants, brotliCompressSync } = require('node:zlib') const [moduleName, relativeOutputDirectory] = process.argv.slice(2) if (!moduleName || !relativeOutputDirectory) { - throw new Error('usage: node scripts/inline-wasm.js ') + throw new Error('usage: node scripts/compress-wasm.js ') } const outputDirectory = path.join(__dirname, '..', relativeOutputDirectory) const gluePath = path.join(outputDirectory, `${moduleName}.js`) const wasmPath = path.join(outputDirectory, `${moduleName}_bg.wasm`) +const compressedWasmPath = `${wasmPath}.br` const glue = fs.readFileSync(gluePath, 'utf8') const wasm = fs.readFileSync(wasmPath) -const encodedWasm = brotliCompressSync(wasm, { +const compressedWasm = brotliCompressSync(wasm, { params: { [constants.BROTLI_PARAM_QUALITY]: 11, }, -}).toString('base64') +}) const loader = [ `const wasmPath = \`\${__dirname}/${moduleName}_bg.wasm\`;`, 'const wasmBytes = require(\'fs\').readFileSync(wasmPath);', ].join('\n') if (!glue.includes(loader)) { - throw new Error('wasm-bindgen loader changed; refusing to publish an external WASM asset') + throw new Error('wasm-bindgen loader changed; refusing to publish a compressed WASM asset') } fs.writeFileSync( gluePath, glue.replace( loader, - () => `const wasmBytes = require('node:zlib').brotliDecompressSync(Buffer.from('${encodedWasm}', 'base64'));`, + () => [ + `const compressedWasmPath = \`\${__dirname}/${moduleName}_bg.wasm.br\`;`, + 'const compressedWasm = require(\'node:fs\').readFileSync(compressedWasmPath);', + 'const wasmBytes = require(\'node:zlib\').brotliDecompressSync(compressedWasm);', + ].join('\n'), ), ) +fs.writeFileSync(compressedWasmPath, compressedWasm) fs.rmSync(wasmPath) fs.rmSync(path.join(outputDirectory, '.gitignore'), { force: true }) fs.rmSync(path.join(outputDirectory, 'package.json'), { force: true }) diff --git a/packages/libdatadog/scripts/report-wasm-size.js b/packages/libdatadog/scripts/report-wasm-size.js index 16b5294a..b4c7a041 100644 --- a/packages/libdatadog/scripts/report-wasm-size.js +++ b/packages/libdatadog/scripts/report-wasm-size.js @@ -307,26 +307,21 @@ function appendCrateReport (lines, profilePath) { function createReport (gluePath, profilePath) { const glue = fs.readFileSync(gluePath, 'utf8') - const match = glue.match(/Buffer\.from\('([A-Za-z0-9+/=]+)', 'base64'\)/) - if (!match) throw new Error('could not find the inline base64 WASM payload') - - const base64Bytes = Buffer.byteLength(match[1]) - const compressed = Buffer.from(match[1], 'base64') + const compressedPath = `${gluePath.slice(0, -3)}_bg.wasm.br` + const compressed = fs.readFileSync(compressedPath) const wasm = brotliDecompressSync(compressed) - const glueBytes = Buffer.byteLength(glue) - base64Bytes - const inlineBytes = Buffer.byteLength(glue) - const base64Overhead = base64Bytes - compressed.length + const glueBytes = Buffer.byteLength(glue) + const packagedBytes = glueBytes + compressed.length const sections = readSections(wasm) const lines = [ '## libdatadog WASM size', '', - '| Inline artifact layer | Bytes | KiB |', + '| Packaged artifact layer | Bytes | KiB |', '| --- | ---: | ---: |', layerRow('Raw WASM (before Brotli)', wasm.length), layerRow('Brotli-compressed WASM', compressed.length), - layerRow('Base64 encoding overhead', base64Overhead), layerRow('JavaScript glue/loader', glueBytes), - layerRow('Final inlined JavaScript', inlineBytes, true), + layerRow('Final packaged artifacts', packagedBytes, true), '', '### Raw WebAssembly sections', '', @@ -344,7 +339,11 @@ function createReport (gluePath, profilePath) { if (profilePath) appendCrateReport(lines, profilePath) - lines.push('', `Generated from \`${path.relative(process.cwd(), gluePath)}\`.`) + lines.push( + '', + `Generated from \`${path.relative(process.cwd(), gluePath)}\` and ` + + `\`${path.relative(process.cwd(), compressedPath)}\`.`, + ) return lines.join('\n') } diff --git a/packages/libdatadog/test/bundlers.test.js b/packages/libdatadog/test/bundlers.test.js index 037af0ff..af0e6a0e 100644 --- a/packages/libdatadog/test/bundlers.test.js +++ b/packages/libdatadog/test/bundlers.test.js @@ -13,36 +13,52 @@ const webpack = require('webpack') const packageRoot = path.join(__dirname, '..') const webpackAsync = promisify(webpack) const entries = new Map([ - ['package', path.join(packageRoot, 'index.js')], - ['WASM', path.join(packageRoot, 'wasm.js')], - ['remote config', path.join(packageRoot, 'remote-config.js')], + ['package', { + asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'), + entry: path.join(packageRoot, 'index.js'), + }], + ['WASM', { + asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'), + entry: path.join(packageRoot, 'wasm.js'), + }], + ['remote config', { + asset: path.join(packageRoot, 'wasm', 'dist', 'remote-config', 'remote_config_bg.wasm.br'), + entry: path.join(packageRoot, 'remote-config.js'), + }], ]) -for (const [name, entry] of entries) { - test(`esbuild bundles the ${name} entry point without emitting an asset`, async () => { - await assertBundle(entry, bundleWithEsbuild) +for (const [name, { asset, entry }] of entries) { + test(`esbuild bundles the ${name} entry point with its compressed WASM asset`, async () => { + await assertBundle(entry, asset, bundleWithEsbuild, name !== 'remote config') }) - test(`webpack bundles the ${name} entry point without emitting an asset`, async () => { - await assertBundle(entry, bundleWithWebpack) + test(`webpack bundles the ${name} entry point with its compressed WASM asset`, async () => { + await assertBundle(entry, asset, bundleWithWebpack, true) }) } /** * @param {string} entry + * @param {string} asset * @param {(entry: string, output: string) => Promise} bundle + * @param {boolean} loadBundle */ -async function assertBundle (entry, bundle) { +async function assertBundle (entry, asset, bundle, loadBundle) { const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'libdatadog-bundle-')) const output = path.join(directory, 'bundle.cjs') + const copiedAsset = path.join(directory, path.basename(asset)) try { await bundle(entry, output) + fs.copyFileSync(asset, copiedAsset) const files = fs.readdirSync(directory) const contents = fs.readFileSync(output, 'utf8') - assert.deepStrictEqual(files, ['bundle.cjs']) - assert.doesNotMatch(contents, /\.wasm(?:['"`)]|$)/m) + assert.equal(files.length, 2) + assert(files.includes('bundle.cjs')) + assert(files.includes(path.basename(asset))) + assert.match(contents, /\.wasm\.br/) + if (loadBundle) require(output) } finally { fs.rmSync(directory, { force: true, recursive: true }) } diff --git a/packages/libdatadog/test/package-contents.test.js b/packages/libdatadog/test/package-contents.test.js index 9856df05..3bf1d690 100644 --- a/packages/libdatadog/test/package-contents.test.js +++ b/packages/libdatadog/test/package-contents.test.js @@ -35,11 +35,16 @@ test('published packages contain only the intended artifacts', () => { const wasmNames = libdatadogWasm.files.map(file => file.path) const standaloneWasm = [...names, ...wasmNames] .filter(file => file.endsWith('.wasm')) + const compressedWasm = wasmNames.filter(file => file.endsWith('.wasm.br')) assert.deepStrictEqual(standaloneWasm, [], - `packages must not contain standalone WASM files: ${standaloneWasm.join(', ')}`) + `packages must not contain raw WASM files: ${standaloneWasm.join(', ')}`) + assert.deepStrictEqual(compressedWasm, [ + 'dist/libdatadog_wasm_bg.wasm.br', + 'dist/remote-config/remote_config_bg.wasm.br', + ]) assert(wasmNames.includes('dist/libdatadog_wasm.js'), - 'WASM package must contain the inline-WASM JavaScript fallback') + 'WASM package must contain the JavaScript loader') assert(wasmNames.includes('dist/remote-config/remote_config.js'), 'WASM package must contain the dedicated remote config artifact') assert.strictEqual(wasmNames.includes('remote-config.js'), false) diff --git a/packages/libdatadog/test/package.test.js b/packages/libdatadog/test/package.test.js index ebb9f24d..0eec01a4 100644 --- a/packages/libdatadog/test/package.test.js +++ b/packages/libdatadog/test/package.test.js @@ -69,20 +69,20 @@ test('root entry point uses the WASM backend', () => { assert.strictEqual(libdatadog.backend(), 'wasm') }) -test('embeds a Brotli-compressed WASM fallback below the size budgets', () => { - const { glue, wasm } = readInlineWasm(path.join( +test('packages a Brotli-compressed WASM fallback below the size budgets', () => { + const { packagedBytes, wasm } = readPackagedWasm(path.join( packageRoot, 'wasm', 'dist', 'libdatadog_wasm.js', )) - assert.ok(Buffer.byteLength(glue) < 260 * 1024) + assert.ok(packagedBytes < 260 * 1024) assert.ok(wasm.length < 600 * 1024) }) -test('embeds dedicated remote config WASM below the size budgets', () => { - const { glue, wasm } = readInlineWasm(path.join( +test('packages dedicated remote config WASM below the size budgets', () => { + const { packagedBytes, wasm } = readPackagedWasm(path.join( packageRoot, 'wasm', 'dist', @@ -90,23 +90,23 @@ test('embeds dedicated remote config WASM below the size budgets', () => { 'remote_config.js', )) - assert.ok(Buffer.byteLength(glue) < 450 * 1024) + assert.ok(packagedBytes < 450 * 1024) assert.ok(wasm.length < 1024 * 1024) }) -test('requires an artifact name and output directory when inlining WASM', () => { - const script = path.join(packageRoot, 'scripts', 'inline-wasm.js') +test('requires an artifact name and output directory when compressing WASM', () => { + const script = path.join(packageRoot, 'scripts', 'compress-wasm.js') for (const scriptArguments of [[], ['fixture']]) { const result = spawnSync(process.execPath, [script, ...scriptArguments]) assert.notStrictEqual(result.status, 0) - assert.match(result.stderr.toString(), /usage: node scripts\/inline-wasm\.js/) + assert.match(result.stderr.toString(), /usage: node scripts\/compress-wasm\.js/) } }) -test('inlines a named WASM artifact into its generated module', () => { - const outputDirectory = fs.mkdtempSync(path.join(packageRoot, '.inline-wasm-')) +test('compresses a named WASM artifact beside its generated module', () => { + const outputDirectory = fs.mkdtempSync(path.join(packageRoot, '.compress-wasm-')) const moduleName = 'fixture' const wasm = Buffer.from('fixture WASM') const loader = [ @@ -121,14 +121,15 @@ test('inlines a named WASM artifact into its generated module', () => { fs.writeFileSync(path.join(outputDirectory, 'package.json'), '{}') const result = spawnSync(process.execPath, [ - path.join(packageRoot, 'scripts', 'inline-wasm.js'), + path.join(packageRoot, 'scripts', 'compress-wasm.js'), moduleName, path.relative(packageRoot, outputDirectory), ]) assert.strictEqual(result.status, 0, result.stderr.toString()) - assert.deepStrictEqual(readInlineWasm(path.join(outputDirectory, `${moduleName}.js`)).wasm, wasm) + assert.deepStrictEqual(readPackagedWasm(path.join(outputDirectory, `${moduleName}.js`)).wasm, wasm) assert.strictEqual(fs.existsSync(path.join(outputDirectory, `${moduleName}_bg.wasm`)), false) + assert.strictEqual(fs.existsSync(path.join(outputDirectory, `${moduleName}_bg.wasm.br`)), true) assert.strictEqual(fs.existsSync(path.join(outputDirectory, '.gitignore')), false) assert.strictEqual(fs.existsSync(path.join(outputDirectory, 'package.json')), false) } finally { @@ -139,13 +140,15 @@ test('inlines a named WASM artifact into its generated module', () => { /** * @param {string} gluePath */ -function readInlineWasm (gluePath) { +function readPackagedWasm (gluePath) { const glue = fs.readFileSync(gluePath, 'utf8') - const encodedWasm = glue.match(/brotliDecompressSync\(Buffer\.from\('([^']+)', 'base64'\)\)/)?.[1] + const compressedWasmPath = gluePath.replace(/\.js$/, '_bg.wasm.br') + const compressedWasm = fs.readFileSync(compressedWasmPath) - assert.ok(encodedWasm, 'WASM must be embedded as a Brotli-compressed base64 string') + assert.match(glue, /brotliDecompressSync\(compressedWasm\)/) return { glue, - wasm: brotliDecompressSync(Buffer.from(encodedWasm, 'base64')), + packagedBytes: Buffer.byteLength(glue) + compressedWasm.length, + wasm: brotliDecompressSync(compressedWasm), } } diff --git a/packages/libdatadog/test/size-report.test.js b/packages/libdatadog/test/size-report.test.js index 8adf2eaa..d76c3431 100644 --- a/packages/libdatadog/test/size-report.test.js +++ b/packages/libdatadog/test/size-report.test.js @@ -10,12 +10,12 @@ const { inferCrate, readSections, } = require('../scripts/report-wasm-size') -test('reports inline packaging and WASM section sizes', () => { +test('reports compressed packaging and WASM section sizes', () => { const gluePath = path.join(__dirname, '..', 'wasm', 'dist', 'libdatadog_wasm.js') const report = createWasmReport(gluePath) assert.match(report, /Raw WASM \(before Brotli\)/) - assert.match(report, /Base64 encoding overhead/) + assert.match(report, /Final packaged artifacts/) assert.match(report, /Raw WebAssembly sections/) assert.match(report, /\| code \|/) assert.match(report, /\| data \|/)