diff --git a/packages/libdatadog/README.md b/packages/libdatadog/README.md index 3ba86705..66b88852 100644 --- a/packages/libdatadog/README.md +++ b/packages/libdatadog/README.md @@ -17,8 +17,8 @@ The package accepts Datadog v0.4 MessagePack payloads and exports them to an agentless intake. `sendV04()` reports completion through a callback and sends delivery failures to the supplied logger. It does not return a promise. -The package uses a wasm-bindgen backend with the WebAssembly bytes embedded in -JavaScript. The canonical inlined output is published as the regular -`@datadog/libdatadog-wasm` dependency from the `wasm` workspace. The WASM -package also contains the separate remote configuration artifact. No raw -`.wasm` asset or native extension is published. +The package publishes Brotli-compressed `.wasm.br` files next to the +wasm-bindgen JavaScript loaders. Each loader reads and decompresses its file +synchronously before instantiation. A bundled distribution must copy the +referenced asset next to its output JavaScript file. No raw `.wasm` asset or +native extension is published. diff --git a/packages/libdatadog/package.json b/packages/libdatadog/package.json index d308211a..2c69f58a 100644 --- a/packages/libdatadog/package.json +++ b/packages/libdatadog/package.json @@ -53,12 +53,12 @@ "node": ">=18" }, "scripts": { - "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run inline:wasm && npm run inline:remote-config", + "build:wasm": "npm run build:wasm:binary && npm run build:remote-config:binary && npm run compress:wasm && npm run compress:remote-config", "build:wasm:binary": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm wasm/dist", "build:remote-config:binary": "node ../../scripts/build-wasm.js ../../crates/remote_config wasm/dist/remote-config", "size:profile": "node ../../scripts/build-wasm.js ../../crates/libdatadog-wasm ../../target/size --profiling", - "inline:wasm": "node scripts/inline-wasm.js libdatadog_wasm wasm/dist", - "inline:remote-config": "node scripts/inline-wasm.js remote_config wasm/dist/remote-config", + "compress:wasm": "node scripts/compress-wasm.js libdatadog_wasm wasm/dist", + "compress:remote-config": "node scripts/compress-wasm.js remote_config wasm/dist/remote-config", "report:wasm-size": "node scripts/report-wasm-size.js", "build": "npm run build:wasm", "test": "node scripts/run-tests.js && npm run test:types", diff --git a/packages/libdatadog/scripts/inline-wasm.js b/packages/libdatadog/scripts/compress-wasm.js similarity index 66% rename from packages/libdatadog/scripts/inline-wasm.js rename to packages/libdatadog/scripts/compress-wasm.js index ea1634b8..474d828f 100644 --- a/packages/libdatadog/scripts/inline-wasm.js +++ b/packages/libdatadog/scripts/compress-wasm.js @@ -7,35 +7,41 @@ const { constants, brotliCompressSync } = require('node:zlib') const [moduleName, relativeOutputDirectory] = process.argv.slice(2) if (!moduleName || !relativeOutputDirectory) { - throw new Error('usage: node scripts/inline-wasm.js ') + throw new Error('usage: node scripts/compress-wasm.js ') } const outputDirectory = path.join(__dirname, '..', relativeOutputDirectory) const gluePath = path.join(outputDirectory, `${moduleName}.js`) const wasmPath = path.join(outputDirectory, `${moduleName}_bg.wasm`) +const compressedWasmPath = `${wasmPath}.br` const glue = fs.readFileSync(gluePath, 'utf8') const wasm = fs.readFileSync(wasmPath) -const encodedWasm = brotliCompressSync(wasm, { +const compressedWasm = brotliCompressSync(wasm, { params: { [constants.BROTLI_PARAM_QUALITY]: 11, }, -}).toString('base64') +}) const loader = [ `const wasmPath = \`\${__dirname}/${moduleName}_bg.wasm\`;`, 'const wasmBytes = require(\'fs\').readFileSync(wasmPath);', ].join('\n') if (!glue.includes(loader)) { - throw new Error('wasm-bindgen loader changed; refusing to publish an external WASM asset') + throw new Error('wasm-bindgen loader changed; refusing to publish a compressed WASM asset') } fs.writeFileSync( gluePath, glue.replace( loader, - () => `const wasmBytes = require('node:zlib').brotliDecompressSync(Buffer.from('${encodedWasm}', 'base64'));`, + () => [ + `const compressedWasmPath = \`\${__dirname}/${moduleName}_bg.wasm.br\`;`, + 'const compressedWasm = require(\'node:fs\').readFileSync(compressedWasmPath);', + 'const wasmBytes = require(\'node:zlib\').brotliDecompressSync(compressedWasm);', + ].join('\n'), ), ) +fs.writeFileSync(compressedWasmPath, compressedWasm) fs.rmSync(wasmPath) fs.rmSync(path.join(outputDirectory, '.gitignore'), { force: true }) fs.rmSync(path.join(outputDirectory, 'package.json'), { force: true }) diff --git a/packages/libdatadog/scripts/report-wasm-size.js b/packages/libdatadog/scripts/report-wasm-size.js index 16b5294a..b4c7a041 100644 --- a/packages/libdatadog/scripts/report-wasm-size.js +++ b/packages/libdatadog/scripts/report-wasm-size.js @@ -307,26 +307,21 @@ function appendCrateReport (lines, profilePath) { function createReport (gluePath, profilePath) { const glue = fs.readFileSync(gluePath, 'utf8') - const match = glue.match(/Buffer\.from\('([A-Za-z0-9+/=]+)', 'base64'\)/) - if (!match) throw new Error('could not find the inline base64 WASM payload') - - const base64Bytes = Buffer.byteLength(match[1]) - const compressed = Buffer.from(match[1], 'base64') + const compressedPath = `${gluePath.slice(0, -3)}_bg.wasm.br` + const compressed = fs.readFileSync(compressedPath) const wasm = brotliDecompressSync(compressed) - const glueBytes = Buffer.byteLength(glue) - base64Bytes - const inlineBytes = Buffer.byteLength(glue) - const base64Overhead = base64Bytes - compressed.length + const glueBytes = Buffer.byteLength(glue) + const packagedBytes = glueBytes + compressed.length const sections = readSections(wasm) const lines = [ '## libdatadog WASM size', '', - '| Inline artifact layer | Bytes | KiB |', + '| Packaged artifact layer | Bytes | KiB |', '| --- | ---: | ---: |', layerRow('Raw WASM (before Brotli)', wasm.length), layerRow('Brotli-compressed WASM', compressed.length), - layerRow('Base64 encoding overhead', base64Overhead), layerRow('JavaScript glue/loader', glueBytes), - layerRow('Final inlined JavaScript', inlineBytes, true), + layerRow('Final packaged artifacts', packagedBytes, true), '', '### Raw WebAssembly sections', '', @@ -344,7 +339,11 @@ function createReport (gluePath, profilePath) { if (profilePath) appendCrateReport(lines, profilePath) - lines.push('', `Generated from \`${path.relative(process.cwd(), gluePath)}\`.`) + lines.push( + '', + `Generated from \`${path.relative(process.cwd(), gluePath)}\` and ` + + `\`${path.relative(process.cwd(), compressedPath)}\`.`, + ) return lines.join('\n') } diff --git a/packages/libdatadog/test/bundlers.test.js b/packages/libdatadog/test/bundlers.test.js index 037af0ff..af0e6a0e 100644 --- a/packages/libdatadog/test/bundlers.test.js +++ b/packages/libdatadog/test/bundlers.test.js @@ -13,36 +13,52 @@ const webpack = require('webpack') const packageRoot = path.join(__dirname, '..') const webpackAsync = promisify(webpack) const entries = new Map([ - ['package', path.join(packageRoot, 'index.js')], - ['WASM', path.join(packageRoot, 'wasm.js')], - ['remote config', path.join(packageRoot, 'remote-config.js')], + ['package', { + asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'), + entry: path.join(packageRoot, 'index.js'), + }], + ['WASM', { + asset: path.join(packageRoot, 'wasm', 'dist', 'libdatadog_wasm_bg.wasm.br'), + entry: path.join(packageRoot, 'wasm.js'), + }], + ['remote config', { + asset: path.join(packageRoot, 'wasm', 'dist', 'remote-config', 'remote_config_bg.wasm.br'), + entry: path.join(packageRoot, 'remote-config.js'), + }], ]) -for (const [name, entry] of entries) { - test(`esbuild bundles the ${name} entry point without emitting an asset`, async () => { - await assertBundle(entry, bundleWithEsbuild) +for (const [name, { asset, entry }] of entries) { + test(`esbuild bundles the ${name} entry point with its compressed WASM asset`, async () => { + await assertBundle(entry, asset, bundleWithEsbuild, name !== 'remote config') }) - test(`webpack bundles the ${name} entry point without emitting an asset`, async () => { - await assertBundle(entry, bundleWithWebpack) + test(`webpack bundles the ${name} entry point with its compressed WASM asset`, async () => { + await assertBundle(entry, asset, bundleWithWebpack, true) }) } /** * @param {string} entry + * @param {string} asset * @param {(entry: string, output: string) => Promise} bundle + * @param {boolean} loadBundle */ -async function assertBundle (entry, bundle) { +async function assertBundle (entry, asset, bundle, loadBundle) { const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'libdatadog-bundle-')) const output = path.join(directory, 'bundle.cjs') + const copiedAsset = path.join(directory, path.basename(asset)) try { await bundle(entry, output) + fs.copyFileSync(asset, copiedAsset) const files = fs.readdirSync(directory) const contents = fs.readFileSync(output, 'utf8') - assert.deepStrictEqual(files, ['bundle.cjs']) - assert.doesNotMatch(contents, /\.wasm(?:['"`)]|$)/m) + assert.equal(files.length, 2) + assert(files.includes('bundle.cjs')) + assert(files.includes(path.basename(asset))) + assert.match(contents, /\.wasm\.br/) + if (loadBundle) require(output) } finally { fs.rmSync(directory, { force: true, recursive: true }) } diff --git a/packages/libdatadog/test/package-contents.test.js b/packages/libdatadog/test/package-contents.test.js index 9856df05..3bf1d690 100644 --- a/packages/libdatadog/test/package-contents.test.js +++ b/packages/libdatadog/test/package-contents.test.js @@ -35,11 +35,16 @@ test('published packages contain only the intended artifacts', () => { const wasmNames = libdatadogWasm.files.map(file => file.path) const standaloneWasm = [...names, ...wasmNames] .filter(file => file.endsWith('.wasm')) + const compressedWasm = wasmNames.filter(file => file.endsWith('.wasm.br')) assert.deepStrictEqual(standaloneWasm, [], - `packages must not contain standalone WASM files: ${standaloneWasm.join(', ')}`) + `packages must not contain raw WASM files: ${standaloneWasm.join(', ')}`) + assert.deepStrictEqual(compressedWasm, [ + 'dist/libdatadog_wasm_bg.wasm.br', + 'dist/remote-config/remote_config_bg.wasm.br', + ]) assert(wasmNames.includes('dist/libdatadog_wasm.js'), - 'WASM package must contain the inline-WASM JavaScript fallback') + 'WASM package must contain the JavaScript loader') assert(wasmNames.includes('dist/remote-config/remote_config.js'), 'WASM package must contain the dedicated remote config artifact') assert.strictEqual(wasmNames.includes('remote-config.js'), false) diff --git a/packages/libdatadog/test/package.test.js b/packages/libdatadog/test/package.test.js index ebb9f24d..0eec01a4 100644 --- a/packages/libdatadog/test/package.test.js +++ b/packages/libdatadog/test/package.test.js @@ -69,20 +69,20 @@ test('root entry point uses the WASM backend', () => { assert.strictEqual(libdatadog.backend(), 'wasm') }) -test('embeds a Brotli-compressed WASM fallback below the size budgets', () => { - const { glue, wasm } = readInlineWasm(path.join( +test('packages a Brotli-compressed WASM fallback below the size budgets', () => { + const { packagedBytes, wasm } = readPackagedWasm(path.join( packageRoot, 'wasm', 'dist', 'libdatadog_wasm.js', )) - assert.ok(Buffer.byteLength(glue) < 260 * 1024) + assert.ok(packagedBytes < 260 * 1024) assert.ok(wasm.length < 600 * 1024) }) -test('embeds dedicated remote config WASM below the size budgets', () => { - const { glue, wasm } = readInlineWasm(path.join( +test('packages dedicated remote config WASM below the size budgets', () => { + const { packagedBytes, wasm } = readPackagedWasm(path.join( packageRoot, 'wasm', 'dist', @@ -90,23 +90,23 @@ test('embeds dedicated remote config WASM below the size budgets', () => { 'remote_config.js', )) - assert.ok(Buffer.byteLength(glue) < 450 * 1024) + assert.ok(packagedBytes < 450 * 1024) assert.ok(wasm.length < 1024 * 1024) }) -test('requires an artifact name and output directory when inlining WASM', () => { - const script = path.join(packageRoot, 'scripts', 'inline-wasm.js') +test('requires an artifact name and output directory when compressing WASM', () => { + const script = path.join(packageRoot, 'scripts', 'compress-wasm.js') for (const scriptArguments of [[], ['fixture']]) { const result = spawnSync(process.execPath, [script, ...scriptArguments]) assert.notStrictEqual(result.status, 0) - assert.match(result.stderr.toString(), /usage: node scripts\/inline-wasm\.js/) + assert.match(result.stderr.toString(), /usage: node scripts\/compress-wasm\.js/) } }) -test('inlines a named WASM artifact into its generated module', () => { - const outputDirectory = fs.mkdtempSync(path.join(packageRoot, '.inline-wasm-')) +test('compresses a named WASM artifact beside its generated module', () => { + const outputDirectory = fs.mkdtempSync(path.join(packageRoot, '.compress-wasm-')) const moduleName = 'fixture' const wasm = Buffer.from('fixture WASM') const loader = [ @@ -121,14 +121,15 @@ test('inlines a named WASM artifact into its generated module', () => { fs.writeFileSync(path.join(outputDirectory, 'package.json'), '{}') const result = spawnSync(process.execPath, [ - path.join(packageRoot, 'scripts', 'inline-wasm.js'), + path.join(packageRoot, 'scripts', 'compress-wasm.js'), moduleName, path.relative(packageRoot, outputDirectory), ]) assert.strictEqual(result.status, 0, result.stderr.toString()) - assert.deepStrictEqual(readInlineWasm(path.join(outputDirectory, `${moduleName}.js`)).wasm, wasm) + assert.deepStrictEqual(readPackagedWasm(path.join(outputDirectory, `${moduleName}.js`)).wasm, wasm) assert.strictEqual(fs.existsSync(path.join(outputDirectory, `${moduleName}_bg.wasm`)), false) + assert.strictEqual(fs.existsSync(path.join(outputDirectory, `${moduleName}_bg.wasm.br`)), true) assert.strictEqual(fs.existsSync(path.join(outputDirectory, '.gitignore')), false) assert.strictEqual(fs.existsSync(path.join(outputDirectory, 'package.json')), false) } finally { @@ -139,13 +140,15 @@ test('inlines a named WASM artifact into its generated module', () => { /** * @param {string} gluePath */ -function readInlineWasm (gluePath) { +function readPackagedWasm (gluePath) { const glue = fs.readFileSync(gluePath, 'utf8') - const encodedWasm = glue.match(/brotliDecompressSync\(Buffer\.from\('([^']+)', 'base64'\)\)/)?.[1] + const compressedWasmPath = gluePath.replace(/\.js$/, '_bg.wasm.br') + const compressedWasm = fs.readFileSync(compressedWasmPath) - assert.ok(encodedWasm, 'WASM must be embedded as a Brotli-compressed base64 string') + assert.match(glue, /brotliDecompressSync\(compressedWasm\)/) return { glue, - wasm: brotliDecompressSync(Buffer.from(encodedWasm, 'base64')), + packagedBytes: Buffer.byteLength(glue) + compressedWasm.length, + wasm: brotliDecompressSync(compressedWasm), } } diff --git a/packages/libdatadog/test/size-report.test.js b/packages/libdatadog/test/size-report.test.js index 8adf2eaa..d76c3431 100644 --- a/packages/libdatadog/test/size-report.test.js +++ b/packages/libdatadog/test/size-report.test.js @@ -10,12 +10,12 @@ const { inferCrate, readSections, } = require('../scripts/report-wasm-size') -test('reports inline packaging and WASM section sizes', () => { +test('reports compressed packaging and WASM section sizes', () => { const gluePath = path.join(__dirname, '..', 'wasm', 'dist', 'libdatadog_wasm.js') const report = createWasmReport(gluePath) assert.match(report, /Raw WASM \(before Brotli\)/) - assert.match(report, /Base64 encoding overhead/) + assert.match(report, /Final packaged artifacts/) assert.match(report, /Raw WebAssembly sections/) assert.match(report, /\| code \|/) assert.match(report, /\| data \|/)