Commit ac1b9018d2163f1c4583fb8409e946f3a3be4ca3
Release ssb-pkg 1.0.0
cel committed on 5/12/2020, 12:43:35 AMParent: 999e6c5d97246e80be3475a0418e31ab178cf9c3
Files changed
LICENSE | changed |
README.md | changed |
lib/help.js | changed |
lib/index.js | changed |
lib/packer.js | changed |
lib/producer.js | changed |
lib/refiner.js | changed |
lib/walker.js | changed |
lib/add-blobs.js | added |
package.json | changed |
prelude/bootstrap.js | changed |
prelude/install.js | added |
prelude/install.sh | added |
LICENSE | ||
---|---|---|
@@ -1,7 +1,8 @@ | ||
1 | 1 … | The MIT License (MIT) |
2 | 2 … | |
3 | 3 … | Copyright (c) 2016 Zeit, Inc. |
4 … | +Copyright (c) 2020 Charles E. Lehner | |
4 | 5 … | |
5 | 6 … | Permission is hereby granted, free of charge, to any person obtaining a copy |
6 | 7 … | of this software and associated documentation files (the "Software"), to deal |
7 | 8 … | in the Software without restriction, including without limitation the rights |
README.md | ||
---|---|---|
@@ -1,37 +1,38 @@ | ||
1 | -**Disclaimer: `pkg` was created for use within containers and is not intended for use in serverless environments. For those using ZEIT Now, this means that there is no requirement to use `pkg` in your projects as the benefits it provides are not applicable to the platform.** | |
1 … | +# ssb-pkg | |
2 | 2 … | |
3 | ---- | |
3 … | +Single-command Node.js binary compiler and installer generator for Secure | |
4 … | +Scuttlebutt (SSB). Based on [pkg](https://github.com/zeit/pkg) by Zeit. | |
4 | 5 … | |
5 | -![](https://res.cloudinary.com/zeit-inc/image/upload/v1509936789/repositories/pkg/pkg-repo-banner-new.png) | |
6 … | +This command line interface enables you to package your Node.js project into an | |
7 … | +installer that runs on SSB and generate an executable that can be run even on | |
8 … | +devices without Node.js installed. | |
6 | 9 … | |
7 | -[![Build Status](https://travis-ci.org/zeit/pkg.svg?branch=master)](https://travis-ci.org/zeit/pkg) | |
8 | -[![Coverage Status](https://coveralls.io/repos/github/zeit/pkg/badge.svg?branch=master)](https://coveralls.io/github/zeit/pkg?branch=master) | |
9 | -[![Dependency Status](https://david-dm.org/zeit/pkg/status.svg)](https://david-dm.org/zeit/pkg) | |
10 | -[![devDependency Status](https://david-dm.org/zeit/pkg/dev-status.svg)](https://david-dm.org/zeit/pkg?type=dev) | |
11 | -[![Join the community on Spectrum](https://withspectrum.github.io/badge/badge.svg)](https://spectrum.chat/zeit) | |
12 | - | |
13 | -This command line interface enables you to package your Node.js project into an executable that can be run even on devices without Node.js installed. | |
14 | - | |
15 | 10 … | ## Use Cases |
16 | 11 … | |
17 | -* Make a commercial version of your application without sources | |
18 | -* Make a demo/evaluation/trial version of your app without sources | |
19 | -* Instantly make executables for other platforms (cross-compilation) | |
20 | -* Make some kind of self-extracting archive or installer | |
12 … | +* Build SSB apps for distribution via SSB | |
13 … | +* Minimize blob footprint of incremental releases, reusing existing blobs | |
14 … | +* Make executables for other platforms (cross-compilation) | |
21 | 15 … | * No need to install Node.js and npm to run the packaged application |
22 | 16 … | * No need to download hundreds of files via `npm install` to deploy |
23 | -your application. Deploy it as a single file | |
17 … | +your application. Deploy it as a single installer producing a single file | |
24 | 18 … | * Put your assets inside the executable to make it even more portable |
25 | 19 … | * Test your app against new Node.js version without installing it |
20 … | +* Package additional data from SSB blobs at install time - including | |
21 … | + architecture-dependend data | |
22 … | +* Single installer for multiple platforms | |
26 | 23 … | |
27 | 24 … | ## Usage |
28 | 25 … | |
26 … | +Requires: [sbotc](%133ulDgs/oC1DXjoK04vDFy6DgVBB/Zok15YJmuhD5Q=.sha256) | |
27 … | + | |
28 … | +Install using [ssb-npm](%iqhz/sQCZCSp91JYAqfQPzHuDYrjw1geKPf1wJ1CvlA=.sha256): | |
29 … | + | |
29 | 30 … | ```sh |
30 | -npm install -g pkg | |
31 … | +ssb-npm install -g ssb-pkg | |
31 | 32 … | ``` |
32 | 33 … | |
33 | -After installing it, run `pkg --help` without arguments to see list of options. | |
34 … | +After installing it, run `ssb-pkg --help` without arguments to see list of options. | |
34 | 35 … | |
35 | 36 … | The entrypoint of your project is a mandatory CLI argument. It may be: |
36 | 37 … | |
37 | 38 … | * Path to entry file. Suppose it is `/path/app.js`, then |
@@ -108,8 +109,46 @@ | ||
108 | 109 … | See also |
109 | 110 … | [Detecting assets in source code](#detecting-assets-in-source-code) and |
110 | 111 … | [Snapshot filesystem](#snapshot-filesystem). |
111 | 112 … | |
113 … | +### Blob Assets | |
114 … | + | |
115 … | +`blobAssets` is a [glob](https://github.com/sindresorhus/globby) | |
116 … | +or list of globs, or object mapping targets to lists of globs. Files | |
117 … | +specified in `blobAssets` will be packaged into the executable via SSB blobs | |
118 … | +fetched during install, rather than being included in the main payload blob | |
119 … | +(snapshot) with the rest of the assets and scripts. This allows the payload | |
120 … | +size to be decreased by factoring out large infrequently changed files. If | |
121 … | +`blobAssets` is organized by target, each set of files is only added during | |
122 … | +install time if its target name matches the system platform and architecture or | |
123 … | +is `"*"`; this is to allow for packing additional binaries so that they are | |
124 … | +only fetched for the systems that need them, and the payload blob can be reused | |
125 … | +on all the systems. | |
126 … | + | |
127 … | +Example: | |
128 … | +``` | |
129 … | + "blobAssets": { | |
130 … | + "linux-x64": [ | |
131 … | + "node_modules/sodium-native/prebuilds/linux-x64/libsodium.so.23", | |
132 … | + "node_modules/sodium-native/prebuilds/linux-x64/node.abi64.node" | |
133 … | + ] | |
134 … | + "linux-arm": [ | |
135 … | + "node_modules/sodium-native/prebuilds/linux-arm/libsodium.so.23", | |
136 … | + "node_modules/sodium-native/prebuilds/linux-arm/node.abi64.node" | |
137 … | + ], | |
138 … | + "darwin-x64": [ | |
139 … | + "node_modules/sodium-native/prebuilds/darwin-x64/libsodium.dylib", | |
140 … | + "node_modules/sodium-native/prebuilds/darwin-x64/node.abi64.node" | |
141 … | + ], | |
142 … | + } | |
143 … | +``` | |
144 … | + | |
145 … | +#### Special handling of modules for Blob Assets | |
146 … | + | |
147 … | +Since `sodium-native` is commonly required for SSB Node.js applications, | |
148 … | +`ssb-pkg` detects it in the dependency tree and automatically adds `blobAssets` entries | |
149 … | +for the module's shared libraries corresponding to the current targets. | |
150 … | + | |
112 | 151 … | ### Options |
113 | 152 … | |
114 | 153 … | Node.js application can be called with runtime options |
115 | 154 … | (belonging to Node.js or V8). To list them type `node --help` or |
@@ -122,10 +161,10 @@ | ||
122 | 161 … | ``` |
123 | 162 … | |
124 | 163 … | ### Output |
125 | 164 … | |
126 | -You may specify `--output` if you create only one executable | |
127 | -or `--out-path` to place executables for multiple targets. | |
165 … | +You may specify `--output` to place the generated installer; otherwise a | |
166 … | +location is picked for you. | |
128 | 167 … | |
129 | 168 … | ### Debug |
130 | 169 … | |
131 | 170 … | Pass `--debug` to `pkg` to get a log of packaging process. |
@@ -207,29 +246,36 @@ | ||
207 | 246 … | add the `.node` file directly in the `assets` field in `package.json`. |
208 | 247 … | |
209 | 248 … | The way NodeJS requires native addon is different from a classic JS |
210 | 249 … | file. It needs to have a file on disk to load it but `pkg` only generate |
211 | -one file. To circumvent this, `pkg` will create a temporary file on the | |
250 … | +one file. To circumvent this, `pkg` will create a file on the | |
212 | 251 … | disk. These files will stay on the disk after the process has exited |
213 | -and will be used again on the next process launch. | |
252 … | +and will be used again on the next process launch. They are stored under | |
253 … | +`~/.cache/ssb-pkg/`. | |
214 | 254 … | |
215 | 255 … | When a package, that contains a native module, is being installed, |
216 | 256 … | the native module is compiled against current system-wide Node.js |
217 | 257 … | version. Then, when you compile your project with `pkg`, pay attention |
218 | 258 … | to `--target` option. You should specify the same Node.js version |
219 | 259 … | as your system-wide Node.js to make compiled executable compatible |
220 | 260 … | with `.node` files. |
221 | 261 … | |
262 … | +If a native module package contains multiple `.node` files for different | |
263 … | +platforms/architectures/Node.js versions (e.g. prebuilds), you may specify them | |
264 … | +with the `blobAssets` config option as described in [Blob Assets](#blob-assets). | |
265 … | +For some packges this is done automatically, as described in | |
266 … | +[Special handling of modules for Blob Assets](#special-handling-of-modules-for-blob-assets). | |
267 … | + | |
222 | 268 … | ## API |
223 | 269 … | |
224 | 270 … | `const { exec } = require('pkg')` |
225 | 271 … | |
226 | 272 … | `exec(args)` takes an array of command line arguments and returns |
227 | 273 … | a promise. For example: |
228 | 274 … | |
229 | 275 … | ```js |
230 | -await exec([ 'app.js', '--target', 'host', '--output', 'app.exe' ]) | |
231 | -// do something with app.exe, run, test, upload, deploy, etc | |
276 … | +await exec([ 'app.js', '--target', 'host', '--output', 'install.sh' ]) | |
277 … | +// do something with install.sh, run, test, upload, deploy, etc | |
232 | 278 … | ``` |
233 | 279 … | |
234 | 280 … | ## Troubleshooting |
235 | 281 … |
lib/help.js | ||
---|---|---|
@@ -10,13 +10,13 @@ | ||
10 | 10 … | -v, --version output pkg version |
11 | 11 … | -t, --targets comma-separated list of targets (see examples) |
12 | 12 … | -c, --config package.json or any json file with top-level config |
13 | 13 … | --options bake v8 options into executable to run with them on |
14 | - -o, --output output file name or template for several files | |
15 | - --out-path path to save output one or more executables | |
14 … | + -o, --output output file name | |
16 | 15 … | -d, --debug show more information during packaging process [off] |
17 | 16 … | -b, --build don't download prebuilt base binaries, build them |
18 | 17 … | --public speed up and disclose the sources of top-level project |
18 … | + --bin-name name of binary to install | |
19 | 19 … | |
20 | 20 … | ${chalk.dim('Examples:')} |
21 | 21 … | |
22 | 22 … | ${chalk.gray('–')} Makes executables for Linux, macOS and Windows |
lib/index.js | ||
---|---|---|
@@ -124,11 +124,11 @@ | ||
124 | 124 … | const argv = minimist(argv2, { |
125 | 125 … | boolean: [ 'b', 'build', 'bytecode', 'd', 'debug', |
126 | 126 … | 'h', 'help', 'public', 'v', 'version' ], |
127 | 127 … | string: [ '_', 'c', 'config', 'o', 'options', 'output', |
128 | - 'outdir', 'out-dir', 'out-path', 'public-packages', | |
129 | - 't', 'target', 'targets' ], | |
130 | - default: { bytecode: true } | |
128 … | + 'public-packages', | |
129 … | + 't', 'target', 'targets', 'bin-name' ], | |
130 … | + default: { bytecode: true, public: true } | |
131 | 131 … | }); |
132 | 132 … | |
133 | 133 … | if (argv.h || argv.help) { |
134 | 134 … | help(); |
@@ -141,9 +141,9 @@ | ||
141 | 141 … | console.log(version); |
142 | 142 … | return; |
143 | 143 … | } |
144 | 144 … | |
145 | - log.info(`pkg@${version}`); | |
145 … | + log.info(`ssb-pkg@${version}`); | |
146 | 146 … | |
147 | 147 … | // debug |
148 | 148 … | |
149 | 149 … | log.debugMode = argv.d || argv.debug; |
@@ -187,21 +187,32 @@ | ||
187 | 187 … | inputJsonName = inputJsonName.split('/').pop(); // @org/foo |
188 | 188 … | } |
189 | 189 … | } |
190 | 190 … | |
191 | - // inputBin | |
191 … | + // inputBin, binName, otherBins | |
192 | 192 … | |
193 | - let inputBin; | |
193 … | + let inputBin, binName, otherBins; | |
194 | 194 … | |
195 | 195 … | if (inputJson) { |
196 | 196 … | let bin = inputJson.bin; |
197 | 197 … | if (bin) { |
198 | 198 … | if (typeof bin === 'object') { |
199 | 199 … | if (bin[inputJsonName]) { |
200 … | + binName = inputJsonName | |
200 | 201 … | bin = bin[inputJsonName]; |
201 | 202 … | } else { |
202 | - bin = bin[Object.keys(bin)[0]]; // TODO multiple inputs to pkg them all? | |
203 … | + binName = Object.keys(bin)[0] | |
204 … | + bin = bin[binName] | |
203 | 205 … | } |
206 … | + otherBins = {} | |
207 … | + var dir = path.dirname(input) | |
208 … | + for (var name in inputJson.bin) { | |
209 … | + if (name !== binName) { | |
210 … | + otherBins[name] = path.resolve(dir, inputJson.bin[name]) | |
211 … | + } | |
212 … | + } | |
213 … | + } else { | |
214 … | + binName = inputJsonName | |
204 | 215 … | } |
205 | 216 … | inputBin = path.resolve(path.dirname(input), bin); |
206 | 217 … | if (!await exists(inputBin)) { |
207 | 218 … | throw wasReported('Bin file does not exist (taken from package.json ' + |
@@ -209,8 +220,16 @@ | ||
209 | 220 … | } |
210 | 221 … | } |
211 | 222 … | } |
212 | 223 … | |
224 … | + if (!binName) { | |
225 … | + if (argv['bin-name']) { | |
226 … | + binName = argv['bin-name'] | |
227 … | + } else { | |
228 … | + throw wasReported('Missing binary name. Use --bin-name <name> or use package.json with \'bin\' property'); | |
229 … | + } | |
230 … | + } | |
231 … | + | |
213 | 232 … | if (inputJson && !inputBin) { |
214 | 233 … | throw wasReported('Property \'bin\' does not exist in', [ input ]); |
215 | 234 … | } |
216 | 235 … | |
@@ -241,18 +260,13 @@ | ||
241 | 260 … | configJson = { pkg: configJson }; |
242 | 261 … | } |
243 | 262 … | } |
244 | 263 … | |
245 | - // output, outputPath | |
264 … | + // output | |
246 | 265 … | |
247 | 266 … | let output = argv.o || argv.output; |
248 | - const outputPath = argv['out-path'] || argv.outdir || argv['out-dir']; | |
249 | 267 … | let autoOutput = false; |
250 | 268 … | |
251 | - if (output && outputPath) { | |
252 | - throw wasReported('Specify either \'output\' or \'out-path\'. Not both'); | |
253 | - } | |
254 | - | |
255 | 269 … | if (!output) { |
256 | 270 … | let name; |
257 | 271 … | if (inputJson) { |
258 | 272 … | name = inputJsonName; |
@@ -268,9 +282,8 @@ | ||
268 | 282 … | } |
269 | 283 … | autoOutput = true; |
270 | 284 … | const ext = path.extname(name); |
271 | 285 … | output = name.slice(0, -ext.length || undefined); |
272 | - output = path.resolve(outputPath || '', output); | |
273 | 286 … | } else { |
274 | 287 … | output = path.resolve(output); |
275 | 288 … | } |
276 | 289 … | |
@@ -324,33 +337,21 @@ | ||
324 | 337 … | file = stringifyTargetForOutput(output, target, different); |
325 | 338 … | } |
326 | 339 … | if (target.platform === 'win' && |
327 | 340 … | path.extname(file) !== '.exe') file += '.exe'; |
328 | - target.output = file; | |
329 | 341 … | } |
330 | 342 … | |
331 | 343 … | // bakes |
332 | 344 … | |
333 | 345 … | const bakes = (argv.options || '').split(',') |
334 | 346 … | .filter((bake) => bake).map((bake) => '--' + bake); |
335 | 347 … | |
336 | - // check if input is going | |
337 | - // to be overwritten by output | |
338 | - | |
339 | - for (const target of targets) { | |
340 | - if (target.output === inputFin) { | |
341 | - if (autoOutput) { | |
342 | - target.output += '-' + target.platform; | |
343 | - } else { | |
344 | - throw wasReported('Refusing to overwrite input file', [ inputFin ]); | |
345 | - } | |
346 | - } | |
347 | - } | |
348 | - | |
349 | 348 … | // fetch targets |
350 | 349 … | |
351 | 350 … | const { bytecode } = argv; |
352 | 351 … | |
352 … | + let fabricator; | |
353 … | + | |
353 | 354 … | for (const target of targets) { |
354 | 355 … | target.forceBuild = forceBuild; |
355 | 356 … | await needWithDryRun(target); |
356 | 357 … | const f = target.fabricator = fabricatorForTarget(target); |
@@ -367,13 +368,20 @@ | ||
367 | 368 … | for (const target of targets) { |
368 | 369 … | target.binaryPath = await needViaCache(target); |
369 | 370 … | const f = target.fabricator; |
370 | 371 … | if (bytecode) { |
372 … | + if (f && fabricator && ( | |
373 … | + f.nodeRange !== fabricator.nodeRange || | |
374 … | + f.arch !== fabricator.arch | |
375 … | + )) { | |
376 … | + throw new Error('If using bytecode, Node.js version and architecture must be same for all targets') | |
377 … | + } | |
371 | 378 … | f.binaryPath = await needViaCache(f); |
372 | 379 … | if (f.platform !== 'win') { |
373 | 380 … | await plusx(f.binaryPath); |
374 | 381 … | } |
375 | 382 … | } |
383 … | + fabricator = f | |
376 | 384 … | } |
377 | 385 … | |
378 | 386 … | // marker |
379 | 387 … | |
@@ -399,8 +407,9 @@ | ||
399 | 407 … | |
400 | 408 … | const params = {}; |
401 | 409 … | if (argv.public) { |
402 | 410 … | params.publicToplevel = true; |
411 … | + params.publicPackages = [ '*' ]; | |
403 | 412 … | } |
404 | 413 … | if (argv['public-packages']) { |
405 | 414 … | params.publicPackages = argv['public-packages'].split(','); |
406 | 415 … | if (params.publicPackages.indexOf('*') !== -1) { |
@@ -411,38 +420,43 @@ | ||
411 | 420 … | // records |
412 | 421 … | |
413 | 422 … | let records; |
414 | 423 … | let entrypoint = inputFin; |
424 … | + let otherEntrypoints = otherBins | |
415 | 425 … | const addition = isConfiguration(input) ? input : undefined; |
416 | 426 … | |
417 | - const walkResult = await walk(marker, entrypoint, addition, params); | |
427 … | + const walkResult = await walk(marker, entrypoint, addition, params, targets, otherEntrypoints); | |
418 | 428 … | entrypoint = walkResult.entrypoint; |
419 | 429 … | records = walkResult.records; |
420 | 430 … | |
421 | - const refineResult = refine(records, entrypoint); | |
431 … | + const refineResult = refine(records, entrypoint, otherEntrypoints); | |
422 | 432 … | entrypoint = refineResult.entrypoint; |
433 … | + otherEntrypoints = refineResult.otherEntrypoints; | |
423 | 434 … | records = refineResult.records; |
424 | 435 … | |
425 | - const backpack = packer({ records, entrypoint, bytecode }); | |
436 … | + const backpack = packer({ records, entrypoint, bytecode, otherEntrypoints }); | |
426 | 437 … | |
427 | 438 … | log.debug('Targets:', JSON.stringify(targets, null, 2)); |
428 | 439 … | |
429 | - for (const target of targets) { | |
430 | - if (await exists(target.output)) { | |
431 | - if ((await stat(target.output)).isFile()) { | |
432 | - await remove(target.output); | |
433 | - } else { | |
434 | - throw wasReported('Refusing to overwrite non-file output', [ target.output ]); | |
435 | - } | |
440 … | + if (await exists(output)) { | |
441 … | + if ((await stat(output)).isFile()) { | |
442 … | + await remove(output); | |
436 | 443 … | } else { |
437 | - await mkdirp(path.dirname(target.output)); | |
444 … | + throw wasReported('Refusing to overwrite non-file output', [ output ]); | |
438 | 445 … | } |
446 … | + } else { | |
447 … | + await mkdirp(path.dirname(output)); | |
448 … | + } | |
439 | 449 … | |
440 | - const slash = target.platform === 'win' ? '\\' : '/'; | |
441 | - await producer({ backpack, bakes, slash, target }); | |
442 | - if (target.platform !== 'win') { | |
443 | - await plusx(target.output); | |
444 | - } | |
450 … | + // TODO | |
451 … | + // const slash = target.platform === 'win' ? '\\' : '/'; | |
452 … | + const slash = '/'; | |
453 … | + | |
454 … | + await producer({ backpack, bakes, slash, targets, fabricator, output, binName }); | |
455 … | + | |
456 … | + if (hostPlatform !== 'win') { | |
457 … | + await plusx(output); | |
445 | 458 … | } |
446 | 459 … | |
447 | 460 … | shutdown(); |
461 … | + | |
448 | 462 … | } |
lib/packer.js | ||
---|---|---|
@@ -31,13 +31,13 @@ | ||
31 | 31 … | } |
32 | 32 … | return false; |
33 | 33 … | } |
34 | 34 … | |
35 | -export default function ({ records, entrypoint, bytecode }) { | |
35 … | +export default function ({ records, entrypoint, bytecode, otherEntrypoints }) { | |
36 | 36 … | const stripes = []; |
37 | 37 … | for (const snap in records) { |
38 | 38 … | const record = records[snap]; |
39 | - const { file } = record; | |
39 … | + const { file, blobGroup } = record; | |
40 | 40 … | if (!hasAnyStore(record)) continue; |
41 | 41 … | assert(record[STORE_STAT], 'packer: no STORE_STAT'); |
42 | 42 … | |
43 | 43 … | assert(record[STORE_BLOB] || record[STORE_CONTENT] || record[STORE_LINKS]); |
@@ -58,9 +58,9 @@ | ||
58 | 58 … | |
59 | 59 … | if (store === STORE_BLOB || |
60 | 60 … | store === STORE_CONTENT) { |
61 | 61 … | if (record.body === undefined) { |
62 | - stripes.push({ snap, store, file }); | |
62 … | + stripes.push({ snap, store, file, blobGroup: blobGroup }); | |
63 | 63 … | } else |
64 | 64 … | if (Buffer.isBuffer(record.body)) { |
65 | 65 … | stripes.push({ snap, store, buffer: record.body }); |
66 | 66 … | } else |
@@ -137,6 +137,6 @@ | ||
137 | 137 … | '\n,\n' + |
138 | 138 … | '%DEFAULT_ENTRYPOINT%' + |
139 | 139 … | '\n);'; |
140 | 140 … | |
141 | - return { prelude, entrypoint, stripes }; | |
141 … | + return { prelude, entrypoint, otherEntrypoints, stripes }; | |
142 | 142 … | } |
lib/producer.js | ||
---|---|---|
@@ -3,88 +3,91 @@ | ||
3 | 3 … | import Multistream from 'multistream'; |
4 | 4 … | import assert from 'assert'; |
5 | 5 … | import { fabricateTwice } from './fabricator.js'; |
6 | 6 … | import fs from 'fs'; |
7 … | +import path from 'path'; | |
8 … | +import zlib from 'zlib'; | |
7 | 9 … | import intoStream from 'into-stream'; |
8 | 10 … | import streamMeter from 'stream-meter'; |
11 … | +import addBlobs from './add-blobs.js'; | |
9 | 12 … | |
10 | -function discoverPlaceholder (binaryBuffer, searchString, padder) { | |
11 | - const placeholder = Buffer.from(searchString); | |
12 | - const position = binaryBuffer.indexOf(placeholder); | |
13 | - if (position === -1) return { notFound: true }; | |
14 | - return { position, size: placeholder.length, padder }; | |
13 … | +function snapshotifyMap(files, slash) { | |
14 … | + var o = {} | |
15 … | + for (var name in files) { | |
16 … | + var file = files[name] | |
17 … | + o[name] = snapshotify(file, slash) | |
18 … | + } | |
19 … | + return o | |
15 | 20 … | } |
16 | 21 … | |
17 | -function injectPlaceholder (fd, placeholder, value, cb) { | |
18 | - const { notFound, position, size, padder } = placeholder; | |
19 | - if (notFound) assert(false, 'Placeholder for not found'); | |
20 | - if (typeof value === 'number') value = value.toString(); | |
21 | - if (typeof value === 'string') value = Buffer.from(value); | |
22 | - const padding = Buffer.from(padder.repeat(size - value.length)); | |
23 | - value = Buffer.concat([ value, padding ]); | |
24 | - fs.write(fd, value, 0, value.length, position, cb); | |
22 … | +function substitute(template, vars) { | |
23 … | + return Object.keys(vars).reduce(function (str, key) { | |
24 … | + return str.replace(key, vars[key]) | |
25 … | + }, template) | |
25 | 26 … | } |
26 | 27 … | |
27 | -function discoverPlaceholders (binaryBuffer) { | |
28 | - return { | |
29 | - BAKERY: discoverPlaceholder(binaryBuffer, '\0' + '// BAKERY '.repeat(20), '\0'), | |
30 | - PAYLOAD_POSITION: discoverPlaceholder(binaryBuffer, '// PAYLOAD_POSITION //', ' '), | |
31 | - PAYLOAD_SIZE: discoverPlaceholder(binaryBuffer, '// PAYLOAD_SIZE //', ' '), | |
32 | - PRELUDE_POSITION: discoverPlaceholder(binaryBuffer, '// PRELUDE_POSITION //', ' '), | |
33 | - PRELUDE_SIZE: discoverPlaceholder(binaryBuffer, '// PRELUDE_SIZE //', ' ') | |
34 | - }; | |
28 … | +function addGzBlobs(file, cb) { | |
29 … | + var idFile = file + '.gzids' | |
30 … | + fs.readFile(idFile, 'utf8', function (err, contents) { | |
31 … | + if (!err) return cb(null, contents.trim().split('\n')) | |
32 … | + if (err.code !== 'ENOENT') return cb(err) | |
33 … | + fs.createReadStream(file) | |
34 … | + .pipe(zlib.createGzip()) | |
35 … | + .pipe(addBlobs(function (err, ids) { | |
36 … | + if (err) return cb(err) | |
37 … | + fs.writeFile(idFile, ids.join('\n')+'\n', function (err) { | |
38 … | + if (err) console.trace(err) | |
39 … | + cb(null, ids) | |
40 … | + }) | |
41 … | + })) | |
42 … | + }) | |
35 | 43 … | } |
36 | 44 … | |
37 | -function injectPlaceholders (fd, placeholders, values, cb) { | |
38 | - injectPlaceholder(fd, placeholders.BAKERY, values.BAKERY, (error) => { | |
39 | - if (error) return cb(error); | |
40 | - injectPlaceholder(fd, placeholders.PAYLOAD_POSITION, values.PAYLOAD_POSITION, (error2) => { | |
41 | - if (error2) return cb(error2); | |
42 | - injectPlaceholder(fd, placeholders.PAYLOAD_SIZE, values.PAYLOAD_SIZE, (error3) => { | |
43 | - if (error3) return cb(error3); | |
44 | - injectPlaceholder(fd, placeholders.PRELUDE_POSITION, values.PRELUDE_POSITION, (error4) => { | |
45 | - if (error4) return cb(error4); | |
46 | - injectPlaceholder(fd, placeholders.PRELUDE_SIZE, values.PRELUDE_SIZE, cb); | |
47 | - }); | |
48 | - }); | |
49 | - }); | |
50 | - }); | |
45 … | +function caseNodeBlobs(ids, pattern) { | |
46 … | + return ' ' + pattern + ') set -- \\\n ' + ids.map(function (id) { | |
47 … | + return "'" + id + "'" | |
48 … | + }).join(' \\\n ') + ';;\n' | |
51 | 49 … | } |
52 | 50 … | |
53 | -function makeBakeryValueFromBakes (bakes) { | |
54 | - const parts = []; | |
55 | - if (bakes.length) { | |
56 | - for (let i = 0; i < bakes.length; i += 1) { | |
57 | - parts.push(Buffer.from(bakes[i])); | |
58 | - parts.push(Buffer.alloc(1)); | |
59 | - } | |
60 | - parts.push(Buffer.alloc(1)); | |
61 | - } | |
62 | - return Buffer.concat(parts); | |
51 … | +function switchNodeBlobs(nodeBlobIds, targets) { | |
52 … | + return Object.keys(targets).map(function (t) { | |
53 … | + var target = targets[t] | |
54 … | + var ids = nodeBlobIds[target.platform + '-' + target.arch] | |
55 … | + if (!ids) throw new Error('target without node blob ids: ' + t) | |
56 … | + var system = | |
57 … | + target.platform === 'linux' ? 'Linux' : | |
58 … | + target.platform === 'macos' ? 'Darwin*' : | |
59 … | + null | |
60 … | + var machine = | |
61 … | + target.arch === 'x64' ? 'x86_64' : | |
62 … | + target.arch === 'x86' ? 'i686' : | |
63 … | + target.arch === 'armv7' ? 'armv7l' : | |
64 … | + target.arch === 'arm64' ? 'aarch64' : | |
65 … | + null | |
66 … | + if (!system) throw new Error('Unknown system: ' + target.platform) | |
67 … | + if (!machine) throw new Error('Unknown machine: ' + target.arch) | |
68 … | + var pattern = system + '\\ ' + machine | |
69 … | + return caseNodeBlobs(ids, pattern) | |
70 … | + }).filter(Boolean).join('\n') | |
63 | 71 … | } |
64 | 72 … | |
65 | -function replaceDollarWise (s, sf, st) { | |
66 | - return s.replace(sf, () => st); | |
67 | -} | |
68 | - | |
69 | -function makePreludeBufferFromPrelude (prelude) { | |
70 | - return Buffer.from( | |
71 | - '(function(process, require, console, EXECPATH_FD, PAYLOAD_POSITION, PAYLOAD_SIZE) { ' + | |
72 | - prelude + | |
73 | - '\n})' // dont remove \n | |
74 | - ); | |
75 | -} | |
76 | - | |
77 | -export default function ({ backpack, bakes, slash, target }) { | |
73 … | +export default function ({ backpack, bakes, slash, targets, fabricator, output, binName }) { | |
78 | 74 … | return new Promise((resolve, reject) => { |
79 | 75 … | if (!Buffer.alloc) { |
80 | 76 … | throw wasReported('Your node.js does not have Buffer.alloc. Please upgrade!'); |
81 | 77 … | } |
82 | 78 … | |
83 | - let { prelude, entrypoint, stripes } = backpack; | |
79 … | + var nodeBlobIds = {} | |
80 … | + var preludeBlobIds, payloadBlobIds, vfsBlobIds, installJsBlobId | |
81 … | + var waiting = 3 | |
82 … | + | |
83 … | + let { prelude, entrypoint, otherEntrypoints, stripes } = backpack; | |
84 | 84 … | entrypoint = snapshotify(entrypoint, slash); |
85 … | + otherEntrypoints = snapshotifyMap(otherEntrypoints, slash) | |
85 | 86 … | stripes = stripes.slice(); |
86 | 87 … | |
88 … | + var fileBlobs = {/* <blobGroup>: {<snap>: <id>} */} | |
89 … | + | |
87 | 90 … | const vfs = {}; |
88 | 91 … | for (const stripe of stripes) { |
89 | 92 … | let { snap } = stripe; |
90 | 93 … | snap = snapshotify(snap, slash); |
@@ -98,15 +101,20 @@ | ||
98 | 101 … | meter = streamMeter(); |
99 | 102 … | return s.pipe(meter); |
100 | 103 … | } |
101 | 104 … | |
102 | - function next (s) { | |
103 | - count += 1; | |
104 | - return pipeToNewMeter(s); | |
105 … | + for (const target of targets) { | |
106 … | + waiting++ | |
107 … | + addGzBlobs(target.binaryPath, function (err, ids) { | |
108 … | + if (err) return reject(err) | |
109 … | + nodeBlobIds[target.platform + '-' + target.arch] = ids | |
110 … | + if (!--waiting) next() | |
111 … | + }) | |
105 | 112 … | } |
106 | 113 … | |
107 | - const binaryBuffer = fs.readFileSync(target.binaryPath); | |
108 | - const placeholders = discoverPlaceholders(binaryBuffer); | |
114 … | + var installJs = path.join(__dirname, '../prelude/install.js') | |
115 … | + fs.createReadStream(installJs) | |
116 … | + .pipe(addBlobs(onAddInstallJsBlobs)); | |
109 | 117 … | |
110 | 118 … | let track = 0; |
111 | 119 … | let prevStripe; |
112 | 120 … | |
@@ -115,99 +123,124 @@ | ||
115 | 123 … | let preludePosition; |
116 | 124 … | let preludeSize; |
117 | 125 … | |
118 | 126 … | new Multistream((cb) => { |
119 | - if (count === 0) { | |
120 | - return cb(undefined, next( | |
121 | - intoStream(binaryBuffer) | |
122 | - )); | |
123 | - } else | |
124 | - if (count === 1) { | |
125 | - payloadPosition = meter.bytes; | |
126 | - return cb(undefined, next( | |
127 | - intoStream(Buffer.alloc(0)) | |
128 | - )); | |
129 | - } else | |
130 | - if (count === 2) { | |
131 | - if (prevStripe && !prevStripe.skip) { | |
132 | - let { snap, store } = prevStripe; | |
133 | - snap = snapshotify(snap, slash); | |
134 | - vfs[snap][store] = [ track, meter.bytes ]; | |
135 | - track += meter.bytes; | |
136 | - } | |
127 … | + if (prevStripe && !prevStripe.skip) { | |
128 … | + let { snap, store } = prevStripe; | |
129 … | + snap = snapshotify(snap, slash); | |
130 … | + vfs[snap][store] = [ track, meter.bytes ]; | |
131 … | + track += meter.bytes; | |
132 … | + } | |
137 | 133 … | |
138 | - if (stripes.length) { | |
139 | - // clone to prevent 'skip' propagate | |
140 | - // to other targets, since same stripe | |
141 | - // is used for several targets | |
142 | - const stripe = Object.assign({}, stripes.shift()); | |
143 | - prevStripe = stripe; | |
134 … | + if (!stripes.length) { | |
135 … | + return cb() | |
136 … | + } | |
144 | 137 … | |
145 | - if (stripe.buffer) { | |
146 | - if (stripe.store === STORE_BLOB) { | |
147 | - const snap = snapshotify(stripe.snap, slash); | |
148 | - return fabricateTwice(bakes, target.fabricator, snap, stripe.buffer, (error, buffer) => { | |
149 | - if (error) { | |
150 | - log.warn(error.message); | |
151 | - stripe.skip = true; | |
152 | - return cb(undefined, intoStream(Buffer.alloc(0))); | |
153 | - } | |
138 … | + // clone to prevent 'skip' propagate | |
139 … | + // to other targets, since same stripe | |
140 … | + // is used for several targets | |
141 … | + const stripe = Object.assign({}, stripes.shift()); | |
142 … | + prevStripe = stripe; | |
154 | 143 … | |
155 | - cb(undefined, pipeToNewMeter(intoStream(buffer))); | |
156 | - }); | |
157 | - } else { | |
158 | - return cb(undefined, pipeToNewMeter(intoStream(stripe.buffer))); | |
144 … | + if (stripe.buffer) { | |
145 … | + if (stripe.store === STORE_BLOB) { | |
146 … | + const snap = snapshotify(stripe.snap, slash); | |
147 … | + return fabricateTwice(bakes, fabricator, snap, stripe.buffer, (error, buffer) => { | |
148 … | + if (error) { | |
149 … | + log.warn(error.message); | |
150 … | + stripe.skip = true; | |
151 … | + return cb(undefined, intoStream(Buffer.alloc(0))); | |
159 | 152 … | } |
160 | - } else | |
161 | - if (stripe.file) { | |
162 | - if (stripe.file === target.output) { | |
163 | - return cb(wasReported( | |
164 | - 'Trying to take executable into executable', stripe.file | |
165 | - )); | |
166 | - } | |
167 | 153 … | |
168 | - assert.equal(stripe.store, STORE_CONTENT); // others must be buffers from walker | |
169 | - return cb(undefined, pipeToNewMeter(fs.createReadStream(stripe.file))); | |
170 | - } else { | |
171 | - assert(false, 'producer: bad stripe'); | |
172 | - } | |
154 … | + cb(undefined, pipeToNewMeter(intoStream(buffer))); | |
155 … | + }); | |
173 | 156 … | } else { |
174 | - payloadSize = track; | |
175 | - preludePosition = payloadPosition + payloadSize; | |
176 | - return cb(undefined, next( | |
177 | - intoStream(makePreludeBufferFromPrelude( | |
178 | - replaceDollarWise( | |
179 | - replaceDollarWise(prelude, '%VIRTUAL_FILESYSTEM%', JSON.stringify(vfs)), | |
180 | - '%DEFAULT_ENTRYPOINT%', JSON.stringify(entrypoint)) | |
181 | - )) | |
157 … | + return cb(undefined, pipeToNewMeter(intoStream(stripe.buffer))); | |
158 … | + } | |
159 … | + } else | |
160 … | + if (stripe.file) { | |
161 … | + if (stripe.file === output) { | |
162 … | + return cb(wasReported( | |
163 … | + 'Trying to take executable into executable', stripe.file | |
182 | 164 … | )); |
183 | 165 … | } |
166 … | + | |
167 … | + var blobGroup = stripe.blobGroup | |
168 … | + if (blobGroup) { | |
169 … | + var readFile = fs.createReadStream(stripe.file) | |
170 … | + return readFile.pipe(addBlobs(function (err, ids) { | |
171 … | + if (err) return cb(err) | |
172 … | + var snap = snapshotify(stripe.snap, slash); | |
173 … | + var blobs = fileBlobs[blobGroup] || (fileBlobs[blobGroup] = {}) | |
174 … | + blobs[snap] = ids | |
175 … | + // vfs entry will be added during install | |
176 … | + stripe.skip = true | |
177 … | + cb(null, intoStream(Buffer.alloc(0))) | |
178 … | + })) | |
179 … | + } | |
180 … | + | |
181 … | + assert.equal(stripe.store, STORE_CONTENT); // others must be buffers from walker | |
182 … | + return cb(undefined, pipeToNewMeter(fs.createReadStream(stripe.file))); | |
184 | 183 … | } else { |
185 | - return cb(); | |
184 … | + assert(false, 'producer: bad stripe'); | |
186 | 185 … | } |
187 | 186 … | }).on('error', (error) => { |
188 | 187 … | reject(error); |
189 | 188 … | }).pipe( |
190 | - fs.createWriteStream(target.output) | |
189 … | + zlib.createGzip() | |
190 … | + ).pipe( | |
191 … | + addBlobs(onAddPayloadBlobs) | |
191 | 192 … | ).on('error', (error) => { |
192 | 193 … | reject(error); |
193 | - }).on('close', () => { | |
194 | - preludeSize = meter.bytes; | |
195 | - fs.open(target.output, 'r+', (error, fd) => { | |
196 | - if (error) return reject(error); | |
197 | - injectPlaceholders(fd, placeholders, { | |
198 | - BAKERY: makeBakeryValueFromBakes(bakes), | |
199 | - PAYLOAD_POSITION: payloadPosition, | |
200 | - PAYLOAD_SIZE: payloadSize, | |
201 | - PRELUDE_POSITION: preludePosition, | |
202 | - PRELUDE_SIZE: preludeSize | |
203 | - }, (error2) => { | |
204 | - if (error2) return reject(error2); | |
205 | - fs.close(fd, (error3) => { | |
206 | - if (error3) return reject(error3); | |
207 | - resolve(); | |
208 | - }); | |
209 | - }); | |
210 | - }); | |
211 | 194 … | }); |
195 … | + | |
196 … | + function onAddInstallJsBlobs(err, ids) { | |
197 … | + if (err) return reject(err) | |
198 … | + if (ids.length > 1) { | |
199 … | + return reject(new Error('install.js must fit in one blob')) | |
200 … | + } | |
201 … | + installJsBlobId = ids | |
202 … | + if (!--waiting) next() | |
203 … | + } | |
204 … | + | |
205 … | + addBlobs(function (err, ids) { | |
206 … | + if (err) return reject(err) | |
207 … | + preludeBlobIds = ids | |
208 … | + if (!--waiting) next() | |
209 … | + }).end(prelude) | |
210 … | + | |
211 … | + function onAddPayloadBlobs(err, ids) { | |
212 … | + if (err) return reject(err) | |
213 … | + payloadBlobIds = ids | |
214 … | + addBlobs(function (err, ids) { | |
215 … | + vfsBlobIds = ids | |
216 … | + if (err) return reject(err) | |
217 … | + if (!--waiting) next() | |
218 … | + }).end(JSON.stringify(vfs)) | |
219 … | + } | |
220 … | + | |
221 … | + function next() { | |
222 … | + var installSh = path.join(__dirname, '../prelude/install.sh') | |
223 … | + var installScriptTemplate = fs.readFileSync(installSh, 'utf8') | |
224 … | + var settings = { | |
225 … | + '%INSTALL_JS_BLOB%': installJsBlobId, | |
226 … | + '%SWITCH_NODE_BLOBS%': switchNodeBlobs(nodeBlobIds, targets), | |
227 … | + '%SETTINGS%': JSON.stringify({ | |
228 … | + preludeBlobs: preludeBlobIds, | |
229 … | + payloadBlobs: payloadBlobIds, | |
230 … | + vfsBlobs: vfsBlobIds, | |
231 … | + binName: binName, | |
232 … | + entrypoint: entrypoint, | |
233 … | + otherEntrypoints: otherEntrypoints, | |
234 … | + bakes: bakes, | |
235 … | + fileBlobs: fileBlobs, | |
236 … | + }, null, 2) | |
237 … | + } | |
238 … | + var installScript = substitute(installScriptTemplate, settings) | |
239 … | + fs.writeFile(output, installScript, function (err) { | |
240 … | + if (err) return reject(err) | |
241 … | + resolve() | |
242 … | + }) | |
243 … | + } | |
244 … | + | |
212 | 245 … | }); |
213 | 246 … | } |
lib/refiner.js | ||
---|---|---|
@@ -38,9 +38,9 @@ | ||
38 | 38 … | if (!found) break; |
39 | 39 … | } |
40 | 40 … | } |
41 | 41 … | |
42 | -function denominate (records, entrypoint, denominator) { | |
42 … | +function denominate (records, entrypoint, otherEntrypoints, denominator) { | |
43 | 43 … | const newRecords = {}; |
44 | 44 … | |
45 | 45 … | for (const file in records) { |
46 | 46 … | let snap = substituteDenominator(file, denominator); |
@@ -53,15 +53,22 @@ | ||
53 | 53 … | |
54 | 54 … | newRecords[snap] = records[file]; |
55 | 55 … | } |
56 | 56 … | |
57 … | + var otherEntrypoints2 = {} | |
58 … | + for (var name in otherEntrypoints) { | |
59 … | + var file = otherEntrypoints[name] | |
60 … | + otherEntrypoints2[name] = substituteDenominator(file, denominator) | |
61 … | + } | |
62 … | + | |
57 | 63 … | return { |
58 | 64 … | records: newRecords, |
59 | - entrypoint: substituteDenominator(entrypoint, denominator) | |
65 … | + entrypoint: substituteDenominator(entrypoint, denominator), | |
66 … | + otherEntrypoints: otherEntrypoints2 | |
60 | 67 … | }; |
61 | 68 … | } |
62 | 69 … | |
63 | -export default function (records, entrypoint) { | |
70 … | +export default function (records, entrypoint, otherEntrypoints) { | |
64 | 71 … | purgeTopDirectories(records); |
65 | 72 … | const denominator = retrieveDenominator(Object.keys(records)); |
66 | - return denominate(records, entrypoint, denominator); | |
73 … | + return denominate(records, entrypoint, otherEntrypoints, denominator); | |
67 | 74 … | } |
lib/walker.js | ||
---|---|---|
@@ -11,9 +11,12 @@ | ||
11 | 11 … | import detector from './detector.js'; |
12 | 12 … | import fs from 'fs-extra'; |
13 | 13 … | import globby from 'globby'; |
14 | 14 … | import path from 'path'; |
15 … | +import { system } from 'pkg-fetch' | |
15 | 16 … | |
17 … | +const { abiToNodeRange } = system; | |
18 … | + | |
16 | 19 … | const win32 = process.platform === 'win32'; |
17 | 20 … | |
18 | 21 … | function unlikelyJavascript (file) { |
19 | 22 … | return [ '.css', '.html', '.json' ].includes(path.extname(file)); |
@@ -142,9 +145,9 @@ | ||
142 | 145 … | } |
143 | 146 … | } |
144 | 147 … | } |
145 | 148 … | |
146 | - let { assets } = pkgConfig; | |
149 … | + let { assets, blobAssets } = pkgConfig; | |
147 | 150 … | |
148 | 151 … | if (assets) { |
149 | 152 … | assets = expandFiles(assets, base); |
150 | 153 … | for (const asset of assets) { |
@@ -158,8 +161,28 @@ | ||
158 | 161 … | }); |
159 | 162 … | } |
160 | 163 … | } |
161 | 164 … | } |
165 … | + | |
166 … | + if (blobAssets) { | |
167 … | + if (Array.isArray(blobAssets)) blobAssets = {'*': blobAssets} | |
168 … | + for (const group in blobAssets) { | |
169 … | + const theseBlobAssets = expandFiles(blobAssets[group], base); | |
170 … | + for (const asset of theseBlobAssets) { | |
171 … | + const stat = await fs.stat(asset); | |
172 … | + if (stat.isFile()) { | |
173 … | + this.append({ | |
174 … | + file: asset, | |
175 … | + marker, | |
176 … | + store: STORE_CONTENT, | |
177 … | + reason: configPath, | |
178 … | + blobGroup: group | |
179 … | + }); | |
180 … | + } | |
181 … | + } | |
182 … | + } | |
183 … | + } | |
184 … | + | |
162 | 185 … | } else { |
163 | 186 … | let { files } = config; |
164 | 187 … | |
165 | 188 … | if (files) { |
@@ -512,17 +535,98 @@ | ||
512 | 535 … | } |
513 | 536 … | } |
514 | 537 … | } |
515 | 538 … | |
516 | - async step_STORE_ANY (record, marker, store) { // eslint-disable-line camelcase | |
539 … | + async stepAddPrebuildsForTarget (record, marker, target) { | |
540 … | + const majorRange = target.nodeRange.replace(/\..*/, '') | |
541 … | + const prebuildPlatform = | |
542 … | + target.platform === 'linux' ? 'linux' : | |
543 … | + target.platform === 'macos' ? 'darwin' : | |
544 … | + null | |
545 … | + if (!prebuildPlatform) { | |
546 … | + throw new Error('Unknown platform for prebuilds: ' + target.platform) | |
547 … | + } | |
548 … | + const prebuildArch = | |
549 … | + target.arch === 'x64' ? 'x64' : | |
550 … | + target.arch === 'x86' ? 'ia32' : | |
551 … | + target.arch === 'armv7' ? 'arm' : | |
552 … | + target.arch === 'arm64' ? 'arm64' : | |
553 … | + null | |
554 … | + if (!prebuildArch) { | |
555 … | + throw new Error('Unknown architecture for prebuilds: ' + target.platform) | |
556 … | + } | |
557 … | + const platformArch = prebuildPlatform + '-' + prebuildArch | |
558 … | + const dir = path.resolve(record.file, '../prebuilds', platformArch) | |
559 … | + let files | |
560 … | + try { files = fs.readdirSync(dir) } | |
561 … | + catch(e) { return } | |
562 … | + | |
563 … | + // sodium-native needs *.node and libsodium.so*/dylib/dll | |
564 … | + var foundNativeModule, foundSharedLib | |
565 … | + var filesToAdd = files.filter(function (file) { | |
566 … | + if (file === 'libsodium.dylib' | |
567 … | + || /^libsodium\.so\.\d+$/.test(file)) { | |
568 … | + return foundSharedLib = true | |
569 … | + } | |
570 … | + var m | |
571 … | + if (file === 'node.napi.node' | |
572 … | + || ((m = /^node\.abi(\d+)\.node$/.exec(file)) | |
573 … | + && abiToNodeRange(m[1]) === majorRange)) { | |
574 … | + return foundNativeModule = true | |
575 … | + } | |
576 … | + }) | |
577 … | + if (!foundNativeModule) { | |
578 … | + throw new Error('Missing native module for ' | |
579 … | + + platformArch + ' (' + Object.keys(nodeRanges).join(', ') + ')') | |
580 … | + } | |
581 … | + if (!foundSharedLib) { | |
582 … | + throw new Error('Missing shared library for ' | |
583 … | + + platformArch + ' (' + Object.keys(nodeRanges).join(', ') + ')') | |
584 … | + } | |
585 … | + filesToAdd.map(function (name) { | |
586 … | + return path.join(dir, name) | |
587 … | + }).filter(function (file) { | |
588 … | + return fs.existsSync(file) | |
589 … | + }).map((file) => { | |
590 … | + this.append({ | |
591 … | + file: file, | |
592 … | + marker, | |
593 … | + store: STORE_CONTENT, | |
594 … | + reason: 'prebuilds', | |
595 … | + blobGroup: platformArch | |
596 … | + }) | |
597 … | + }) | |
598 … | + } | |
599 … | + | |
600 … | + async stepAddPrebuilds (record, marker) { | |
601 … | + for (const target of this.targets) { | |
602 … | + this.stepAddPrebuildsForTarget(record, marker, target) | |
603 … | + } | |
604 … | + } | |
605 … | + | |
606 … | + async step_STORE_ANY (record, marker, store, blobGroup) { // eslint-disable-line camelcase | |
517 | 607 … | if (record[store] !== undefined) return; |
518 | 608 … | record[store] = false; // default is discard |
609 … | + record.blobGroup = blobGroup | |
519 | 610 … | |
520 | 611 … | this.append({ |
521 | 612 … | file: record.file, |
522 | 613 … | store: STORE_STAT |
523 | 614 … | }); |
524 | 615 … | |
616 … | + // detect sodium-native | |
617 … | + if (store === STORE_BLOB | |
618 … | + && path.basename(record.file) === 'index.js' | |
619 … | + && path.basename(path.dirname(record.file)) === 'sodium-native' | |
620 … | + && await fs.exists(path.resolve(record.file, '../prebuilds'))) { | |
621 … | + if (this.sodiumNative) { | |
622 … | + log.error('sodium-native found multiple times:', | |
623 … | + [ '%1: ' + this.sodiumNative, '%2: ' + record.file ]); | |
624 … | + this.sodiumNative = record.file | |
625 … | + } | |
626 … | + await this.stepAddPrebuilds(record, marker); | |
627 … | + } | |
628 … | + | |
525 | 629 … | const derivatives1 = []; |
526 | 630 … | await this.stepActivate(marker, derivatives1); |
527 | 631 … | await this.stepDerivatives(record, marker, derivatives1); |
528 | 632 … | if (store === STORE_BLOB) { |
@@ -602,9 +706,9 @@ | ||
602 | 706 … | const { file, store, data } = task; |
603 | 707 … | const record = this.records[file]; |
604 | 708 … | if (store === STORE_BLOB || |
605 | 709 … | store === STORE_CONTENT) { |
606 | - await this.step_STORE_ANY(record, task.marker, store); | |
710 … | + await this.step_STORE_ANY(record, task.marker, store, task.blobGroup); | |
607 | 711 … | } else |
608 | 712 … | if (store === STORE_LINKS) { |
609 | 713 … | this.step_STORE_LINKS(record, data); |
610 | 714 … | } else |
@@ -627,14 +731,16 @@ | ||
627 | 731 … | } |
628 | 732 … | } |
629 | 733 … | } |
630 | 734 … | |
631 | - async start (marker, entrypoint, addition, params) { | |
735 … | + async start (marker, entrypoint, addition, params, targets, otherEntrypoints) { | |
632 | 736 … | this.tasks = []; |
633 | 737 … | this.records = {}; |
634 | 738 … | this.dictionary = {}; |
635 | 739 … | this.patches = {}; |
636 | 740 … | this.params = params; |
741 … | + this.targets = targets; | |
742 … | + this.otherEntrypoints = otherEntrypoints | |
637 | 743 … | |
638 | 744 … | await this.readDictionary(); |
639 | 745 … | |
640 | 746 … | this.append({ |
@@ -650,8 +756,17 @@ | ||
650 | 756 … | store: STORE_CONTENT |
651 | 757 … | }); |
652 | 758 … | } |
653 | 759 … | |
760 … | + if (otherEntrypoints) for (const name in otherEntrypoints) { | |
761 … | + const file = otherEntrypoints[name] | |
762 … | + this.append({ | |
763 … | + file, | |
764 … | + marker, | |
765 … | + store: STORE_BLOB | |
766 … | + }) | |
767 … | + } | |
768 … | + | |
654 | 769 … | const tasks = this.tasks; |
655 | 770 … | for (let i = 0; i < tasks.length; i += 1) { |
656 | 771 … | // NO MULTIPLE WORKERS! THIS WILL LEAD TO NON-DETERMINISTIC |
657 | 772 … | // ORDER. one-by-one fifo is the only way to iterate tasks |
lib/add-blobs.js | ||
---|---|---|
@@ -1,0 +1,65 @@ | ||
1 … | +const proc = require('child_process'); | |
2 … | +const Transform = require('stream').Transform; | |
3 … | + | |
4 … | +module.exports = function (cb) { | |
5 … | + var ids = [] | |
6 … | + var maxBlobSize = 5242879 | |
7 … | + var add, len = 0 | |
8 … | + var onAddDone | |
9 … | + function startAdd() { | |
10 … | + return proc.execFile('sbotc', ['blobs.add'], (err, stdout, stderr) => { | |
11 … | + if (err) return console.error(stderr), cb(err) | |
12 … | + var id = stdout.trim() | |
13 … | + ids.push(id) | |
14 … | + if (onAddDone) onAddDone(id), onAddDone = null | |
15 … | + else console.trace(id) | |
16 … | + }) | |
17 … | + } | |
18 … | + return new Transform({ | |
19 … | + transform: function transform(data, encoding, cb) { | |
20 … | + if (!add) add = startAdd() | |
21 … | + var buf = Buffer.from(data, encoding) | |
22 … | + if (buf.length === 0) return cb() | |
23 … | + var remainingSpace = maxBlobSize - len | |
24 … | + if (buf.length < remainingSpace) { | |
25 … | + len += buf.length | |
26 … | + return add.stdin.write(buf, cb) | |
27 … | + } | |
28 … | + var endBuf = buf.slice(0, remainingSpace) | |
29 … | + var nextBuf = buf.slice(remainingSpace) | |
30 … | + len += endBuf.length | |
31 … | + var waiting = 2 | |
32 … | + onAddDone = function (id) { | |
33 … | + if (!--waiting) next() | |
34 … | + } | |
35 … | + var stdin = add.stdin | |
36 … | + add = null | |
37 … | + len = 0 | |
38 … | + stdin.end(endBuf, function (err) { | |
39 … | + if (err) return cb(err) | |
40 … | + if (!--waiting) next() | |
41 … | + }) | |
42 … | + function next() { | |
43 … | + transform(nextBuf, null, cb) | |
44 … | + } | |
45 … | + }, | |
46 … | + flush: (_cb) => { | |
47 … | + if (!add) { | |
48 … | + _cb() | |
49 … | + return cb(null, ids) | |
50 … | + } | |
51 … | + var waiting = 2 | |
52 … | + onAddDone = function (id) { | |
53 … | + if (!--waiting) next() | |
54 … | + } | |
55 … | + add.stdin.end(function (err) { | |
56 … | + if (err) return cb(err) | |
57 … | + if (!--waiting) next() | |
58 … | + }) | |
59 … | + function next() { | |
60 … | + _cb() | |
61 … | + cb(null, ids) | |
62 … | + } | |
63 … | + } | |
64 … | + }) | |
65 … | +} |
package.json | ||
---|---|---|
@@ -1,13 +1,16 @@ | ||
1 | 1 … | { |
2 | - "name": "pkg", | |
3 | - "version": "4.4.8", | |
4 | - "description": "Package your Node.js project into an executable", | |
2 … | + "name": "ssb-pkg", | |
3 … | + "version": "1.0.0", | |
4 … | + "description": "Package your Node.js project into an executable installer for SSB", | |
5 | 5 … | "main": "lib-es5/index.js", |
6 | 6 … | "license": "MIT", |
7 | - "repository": "zeit/pkg", | |
7 … | + "repository": { | |
8 … | + "type": "git", | |
9 … | + "url": "ssb://%ZqSrybpvuDshsFFTMCM+vcZS7ZP52Y+BfRT+6UIQ8Ns=.sha256" | |
10 … | + }, | |
8 | 11 … | "bin": { |
9 | - "pkg": "lib-es5/bin.js" | |
12 … | + "ssb-pkg": "lib-es5/bin.js" | |
10 | 13 … | }, |
11 | 14 … | "files": [ |
12 | 15 … | "lib-es5/*.js", |
13 | 16 … | "dictionary/*.js", |
@@ -59,24 +62,15 @@ | ||
59 | 62 … | "rimraf": "^3.0.1" |
60 | 63 … | }, |
61 | 64 … | "scripts": { |
62 | 65 … | "babel": "node test/rimraf-es5.js && babel lib --out-dir lib-es5", |
63 | - "lint": "eslint-klopov . || true", | |
64 | - "prepare": "npm run babel", | |
65 | - "prepublishOnly": "eslint-klopov . && npm run test:no-npm", | |
66 | - "test": "eslint-klopov . && npm run babel && node test/test.js node14 no-npm && node test/test.js node12 no-npm && node test/test.js node10 no-npm && node test/test.js host only-npm", | |
67 | - "test:no-npm": "node test/test.js host no-npm" | |
66 … | + "babel-watch": "babel lib --out-dir lib-es5 -w --skip-initial-build", | |
67 … | + "prepare": "npm run babel" | |
68 | 68 … | }, |
69 | 69 … | "eslintConfig": { |
70 | 70 … | "extends": "klopov", |
71 | 71 … | "parser": "babel-eslint", |
72 | 72 … | "rules": { |
73 | 73 … | "wrap-iife": "off" |
74 | 74 … | } |
75 | - }, | |
76 | - "greenkeeper": { | |
77 | - "emails": false, | |
78 | - "ignore": [ | |
79 | - "pkg-fetch" | |
80 | - ] | |
81 | 75 … | } |
82 | 76 … | } |
prelude/bootstrap.js | ||
---|---|---|
@@ -67,8 +67,13 @@ | ||
67 | 67 … | |
68 | 68 … | ENTRYPOINT = process.argv[1]; |
69 | 69 … | delete process.env.PKG_EXECPATH; |
70 | 70 … | |
71 … | +if (process.env.PKG_ENTRYPOINT) { | |
72 … | + ENTRYPOINT = process.env.PKG_ENTRYPOINT; | |
73 … | + delete process.env.PKG_ENTRYPOINT; | |
74 … | +} | |
75 … | + | |
71 | 76 … | // ///////////////////////////////////////////////////////////////// |
72 | 77 … | // EXECSTAT //////////////////////////////////////////////////////// |
73 | 78 … | // ///////////////////////////////////////////////////////////////// |
74 | 79 … | |
@@ -1580,19 +1585,37 @@ | ||
1580 | 1585 … | const fs = require('fs'); |
1581 | 1586 … | var ancestor = {}; |
1582 | 1587 … | ancestor.dlopen = process.dlopen; |
1583 | 1588 … | |
1589 … | + function getLibCacheDir() { | |
1590 … | + var path = require('path') | |
1591 … | + var os = require('os') | |
1592 … | + var cacheDir = process.env.SSB_PKG_CACHE_DIR | |
1593 … | + || path.join(os.homedir(), '.cache', 'ssb-pkg', | |
1594 … | + os.platform() + '-' + os.arch()) | |
1595 … | + try { | |
1596 … | + fs.accessSync(cacheDir) | |
1597 … | + } catch(e) { | |
1598 … | + try { fs.mkdirSync(path.dirname(path.dirname(cacheDir))) } catch(e) {} | |
1599 … | + try { fs.mkdirSync(path.dirname(cacheDir)) } catch(e) {} | |
1600 … | + fs.mkdirSync(cacheDir) | |
1601 … | + } | |
1602 … | + return cacheDir | |
1603 … | + } | |
1604 … | + | |
1584 | 1605 … | process.dlopen = function () { |
1585 | 1606 … | const args = cloneArgs(arguments); |
1586 | 1607 … | const modulePath = args[1]; |
1587 | - const moduleDirname = require('path').dirname(modulePath); | |
1608 … | + const path = require('path') | |
1609 … | + const moduleDirname = path.dirname(modulePath); | |
1588 | 1610 … | if (insideSnapshot(modulePath)) { |
1589 | 1611 … | // Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path |
1590 | 1612 … | // we need to write the file somewhere on disk first and then load it |
1591 | 1613 … | const moduleContent = fs.readFileSync(modulePath); |
1592 | - const moduleBaseName = require('path').basename(modulePath); | |
1614 … | + const moduleBaseName = path.basename(modulePath); | |
1593 | 1615 … | const hash = require('crypto').createHash('sha256').update(moduleContent).digest('hex'); |
1594 | - const tmpModulePath = `${require('os').tmpdir()}/${hash}_${moduleBaseName}`; | |
1616 … | + const tmpModulePath = path.join(getLibCacheDir(), | |
1617 … | + hash + '_' + moduleBaseName); | |
1595 | 1618 … | try { |
1596 | 1619 … | fs.statSync(tmpModulePath); |
1597 | 1620 … | } catch (e) { |
1598 | 1621 … | // Most likely this means the module is not on disk yet |
@@ -1618,10 +1641,10 @@ | ||
1618 | 1641 … | const m = e.message.match(unknownModuleErrorRegex) |
1619 | 1642 … | const moduleName = m[1] || m[2]; |
1620 | 1643 … | const modulePath = `${moduleDirname}/${moduleName}`; |
1621 | 1644 … | const moduleContent = fs.readFileSync(modulePath); |
1622 | - const moduleBaseName = require('path').basename(modulePath); | |
1623 | - const tmpModulePath = `${require('os').tmpdir()}/${moduleBaseName}`; | |
1645 … | + const moduleBaseName = path.basename(modulePath); | |
1646 … | + const tmpModulePath = path.join(getLibCacheDir(), moduleBaseName); | |
1624 | 1647 … | try { |
1625 | 1648 … | fs.statSync(tmpModulePath); |
1626 | 1649 … | } catch (e) { |
1627 | 1650 … | fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 }); |
prelude/install.js | ||
---|---|---|
@@ -1,0 +1,315 @@ | ||
1 … | +var os = require('os') | |
2 … | +var fs = require('fs') | |
3 … | +var zlib = require('zlib') | |
4 … | +var http = require('http') | |
5 … | +var stream = require('stream') | |
6 … | +var path = require('path') | |
7 … | +var STORE_CONTENT = 1 | |
8 … | + | |
9 … | +var settings = JSON.parse(fs.readFileSync(3)) | |
10 … | + | |
11 … | +function ReadBlobs(ids) { | |
12 … | + if (!Array.isArray(ids)) throw new TypeError('blob ids should be array') | |
13 … | + this.ids = ids | |
14 … | + stream.Readable.call(this, ids) | |
15 … | +} | |
16 … | +ReadBlobs.prototype = new stream.Readable() | |
17 … | +ReadBlobs.prototype.constructor = ReadBlobs | |
18 … | +ReadBlobs.prototype.i = 0 | |
19 … | +ReadBlobs.prototype.blobsBase = process.env.SSB_BLOBS_BASE | |
20 … | + || 'http://localhost:8989/blobs/get/' | |
21 … | +ReadBlobs.prototype._read = function (size) { | |
22 … | + if (!this.req) this._makeReq(this) | |
23 … | + else if (this.res) this.res.resume() | |
24 … | +} | |
25 … | +ReadBlobs.prototype._destroy = function (err, cb) { | |
26 … | + if (this.req) this.req.destroy(), this.req = null | |
27 … | + if (this.res) this.res.destroy(), this.res = null | |
28 … | + cb(err) | |
29 … | +} | |
30 … | +ReadBlobs.prototype._makeReq = function () { | |
31 … | + if (this.i >= this.ids.length) return this.push(null) | |
32 … | + var id = this.ids[this.i++] | |
33 … | + var url = this.blobsBase + id | |
34 … | + console.error(id) | |
35 … | + var onResp = (res) => { | |
36 … | + this.res = res | |
37 … | + if (res.statusCode !== 200) { | |
38 … | + var reqStatus = res.statusCode + ' ' + res.statusMessage | |
39 … | + return this.destroy(new Error(reqStatus)) | |
40 … | + } | |
41 … | + res.on('data', (data) => { | |
42 … | + if (this.push(data) === false) res.pause() | |
43 … | + }) | |
44 … | + res.on('end', () => { | |
45 … | + this._makeReq() | |
46 … | + }) | |
47 … | + } | |
48 … | + this.req = http.get(url, onResp).on('error', (err) => { | |
49 … | + // sometimes localhost resolves to 127.0.0.1 but ssb-ws listens on ::1 | |
50 … | + if (err.code === 'ECONNREFUSED' && err.address === '127.0.0.1') { | |
51 … | + this.blobsBase = this.blobsBase.replace('localhost', '[::1]') | |
52 … | + url = this.blobsBase + id | |
53 … | + http.get(url, onResp).on('error', (err) => { | |
54 … | + this.destroy(err) | |
55 … | + }) | |
56 … | + } else { | |
57 … | + this.destroy(err) | |
58 … | + } | |
59 … | + }) | |
60 … | +} | |
61 … | + | |
62 … | +function Meter() { | |
63 … | + stream.Transform.call(this) | |
64 … | +} | |
65 … | +Meter.prototype = new stream.Transform() | |
66 … | +Meter.prototype.constructor = Meter | |
67 … | +Meter.prototype.length = 0 | |
68 … | +Meter.prototype._transform = function (buf, encoding, cb) { | |
69 … | + this.length += buf.length | |
70 … | + cb(null, buf) | |
71 … | +} | |
72 … | + | |
73 … | +function collect(cb) { | |
74 … | + var bufs = [] | |
75 … | + return new stream.Writable({ | |
76 … | + write: function (chunk, encoding, cb) { | |
77 … | + bufs.push(Buffer.from(chunk, encoding)) | |
78 … | + cb() | |
79 … | + } | |
80 … | + }).on('finish', function () { | |
81 … | + cb(null, Buffer.concat(bufs)) | |
82 … | + }).on('error', cb) | |
83 … | +} | |
84 … | + | |
85 … | +function substitute(template, vars) { | |
86 … | + return Object.keys(vars).reduce(function (str, key) { | |
87 … | + return str.replace(key, vars[key]) | |
88 … | + }, template) | |
89 … | +} | |
90 … | + | |
91 … | +function discoverPlaceholder (binaryBuffer, searchString, padder) { | |
92 … | + const placeholder = Buffer.from(searchString); | |
93 … | + const position = binaryBuffer.indexOf(placeholder); | |
94 … | + if (position === -1) return { notFound: true }; | |
95 … | + return { position, size: placeholder.length, padder }; | |
96 … | +} | |
97 … | + | |
98 … | +function injectPlaceholder (fd, placeholder, value, cb) { | |
99 … | + const { notFound, position, size, padder } = placeholder; | |
100 … | + if (notFound) assert(false, 'Placeholder for not found'); | |
101 … | + if (typeof value === 'number') value = value.toString(); | |
102 … | + if (typeof value === 'string') value = Buffer.from(value); | |
103 … | + const padding = Buffer.from(padder.repeat(size - value.length)); | |
104 … | + value = Buffer.concat([ value, padding ]); | |
105 … | + fs.write(fd, value, 0, value.length, position, cb); | |
106 … | +} | |
107 … | + | |
108 … | +function discoverPlaceholders (binaryBuffer) { | |
109 … | + return { | |
110 … | + BAKERY: discoverPlaceholder(binaryBuffer, '\0' + '// BAKERY '.repeat(20), '\0'), | |
111 … | + PAYLOAD_POSITION: discoverPlaceholder(binaryBuffer, '// PAYLOAD_POSITION //', ' '), | |
112 … | + PAYLOAD_SIZE: discoverPlaceholder(binaryBuffer, '// PAYLOAD_SIZE //', ' '), | |
113 … | + PRELUDE_POSITION: discoverPlaceholder(binaryBuffer, '// PRELUDE_POSITION //', ' '), | |
114 … | + PRELUDE_SIZE: discoverPlaceholder(binaryBuffer, '// PRELUDE_SIZE //', ' ') | |
115 … | + }; | |
116 … | +} | |
117 … | + | |
118 … | +function injectPlaceholders (fd, placeholders, values, cb) { | |
119 … | + injectPlaceholder(fd, placeholders.BAKERY, values.BAKERY, (error) => { | |
120 … | + if (error) return cb(error); | |
121 … | + injectPlaceholder(fd, placeholders.PAYLOAD_POSITION, values.PAYLOAD_POSITION, (error2) => { | |
122 … | + if (error2) return cb(error2); | |
123 … | + injectPlaceholder(fd, placeholders.PAYLOAD_SIZE, values.PAYLOAD_SIZE, (error3) => { | |
124 … | + if (error3) return cb(error3); | |
125 … | + injectPlaceholder(fd, placeholders.PRELUDE_POSITION, values.PRELUDE_POSITION, (error4) => { | |
126 … | + if (error4) return cb(error4); | |
127 … | + injectPlaceholder(fd, placeholders.PRELUDE_SIZE, values.PRELUDE_SIZE, cb); | |
128 … | + }); | |
129 … | + }); | |
130 … | + }); | |
131 … | + }); | |
132 … | +} | |
133 … | + | |
134 … | +function makeBakeryValueFromBakes (bakes) { | |
135 … | + const parts = []; | |
136 … | + if (bakes.length) { | |
137 … | + for (let i = 0; i < bakes.length; i += 1) { | |
138 … | + parts.push(Buffer.from(bakes[i])); | |
139 … | + parts.push(Buffer.alloc(1)); | |
140 … | + } | |
141 … | + parts.push(Buffer.alloc(1)); | |
142 … | + } | |
143 … | + return Buffer.concat(parts); | |
144 … | +} | |
145 … | + | |
146 … | +function makePreludeBufferFromPrelude (prelude) { | |
147 … | + return Buffer.from( | |
148 … | + '(function(process, require, console, EXECPATH_FD, PAYLOAD_POSITION, PAYLOAD_SIZE) { ' + | |
149 … | + prelude + | |
150 … | + '\n})' // dont remove \n | |
151 … | + ); | |
152 … | +} | |
153 … | + | |
154 … | +function plusx(file) { | |
155 … | + const s = fs.statSync(file) | |
156 … | + const newMode = s.mode | 64 | 8 | 1 | |
157 … | + if (s.mode === newMode) return | |
158 … | + const base8 = newMode.toString(8).slice(-3) | |
159 … | + fs.chmodSync(file, base8) | |
160 … | +} | |
161 … | + | |
162 … | +var prefix = process.env.PREFIX || path.join(os.homedir(), '.local') | |
163 … | +var binDir = process.env.BIN_DIR || path.join(prefix, 'bin') | |
164 … | +var binPath = path.join(binDir, settings.binName) | |
165 … | +var binTmp = binPath + '~' | |
166 … | + | |
167 … | +// assume the node binary is the pkg one to use | |
168 … | +var nodeBuffer = fs.readFileSync(process.execPath) | |
169 … | + | |
170 … | +var placeholders = discoverPlaceholders(nodeBuffer) | |
171 … | +var payloadPosition = nodeBuffer.length, payloadSize | |
172 … | +var preludePosition, preludeSize | |
173 … | +var vfs | |
174 … | +var binFd, writeStream | |
175 … | + | |
176 … | +try { | |
177 … | + fs.statSync(binDir) | |
178 … | +} catch(e) { | |
179 … | + try { fs.mkdirSync(path.dirname(path.dirname(binDir))) } catch(e) {} | |
180 … | + try { fs.mkdirSync(path.dirname(binDir)) } catch(e) {} | |
181 … | + fs.mkdirSync(binDir) | |
182 … | +} | |
183 … | + | |
184 … | +function writeNodeBinary() { | |
185 … | + fs.open(binTmp, 'w+', function (err, fd) { | |
186 … | + if (err) throw err | |
187 … | + binFd = fd | |
188 … | + writeStream = fs.createWriteStream(null, {fd: fd}) | |
189 … | + writeStream.write(nodeBuffer, function (err) { | |
190 … | + if (err) throw err | |
191 … | + getVfs() | |
192 … | + }) | |
193 … | + }) | |
194 … | +} | |
195 … | + | |
196 … | +function getVfs() { | |
197 … | + new ReadBlobs(settings.vfsBlobs) | |
198 … | + .pipe(collect(function (err, vfsData) { | |
199 … | + if (err) throw err | |
200 … | + vfs = JSON.parse(vfsData) | |
201 … | + getPayload() | |
202 … | + })) | |
203 … | +} | |
204 … | + | |
205 … | +function getPayload() { | |
206 … | + var payloadMeter = new Meter() | |
207 … | + var readPayload = new ReadBlobs(settings.payloadBlobs) | |
208 … | + .pipe(zlib.createGunzip()) | |
209 … | + .pipe(payloadMeter) | |
210 … | + readPayload.pipe(writeStream, {end: false}) | |
211 … | + readPayload.on('end', function () { | |
212 … | + payloadSize = payloadMeter.length | |
213 … | + addFileBlobs() | |
214 … | + }) | |
215 … | +} | |
216 … | + | |
217 … | +function addFileBlobs() { | |
218 … | + var offset = payloadSize | |
219 … | + var fileBlobs = settings.fileBlobs | |
220 … | + var queue = [] | |
221 … | + function enqueue(fileBlobsToAdd) { | |
222 … | + for (var snap in fileBlobsToAdd) { | |
223 … | + var ids = fileBlobsToAdd[snap] | |
224 … | + queue.push({snap: snap, ids: ids}) | |
225 … | + } | |
226 … | + } | |
227 … | + if (Array.isArray(fileBlobs)) { | |
228 … | + enqueue(fileBlobs) | |
229 … | + } else [ | |
230 … | + os.platform() + '-' + os.arch(), | |
231 … | + '*' | |
232 … | + ].forEach(function (group) { | |
233 … | + enqueue(fileBlobs[group]) | |
234 … | + }) | |
235 … | + ;(function next() { | |
236 … | + if (!queue.length) return getPrelude() | |
237 … | + var item = queue.shift() | |
238 … | + var snap = item.snap, ids = item.ids | |
239 … | + | |
240 … | + var fileMeter = new Meter() | |
241 … | + var readFile = new ReadBlobs(ids) | |
242 … | + .pipe(fileMeter) | |
243 … | + readFile.pipe(writeStream, {end: false}) | |
244 … | + readFile.on('end', function () { | |
245 … | + var fileSize = fileMeter.length | |
246 … | + vfs[snap][STORE_CONTENT] = [offset, fileSize] | |
247 … | + offset += fileSize | |
248 … | + payloadSize += fileSize | |
249 … | + next() | |
250 … | + }) | |
251 … | + })() | |
252 … | +} | |
253 … | + | |
254 … | +function getPrelude() { | |
255 … | + preludePosition = payloadPosition + payloadSize | |
256 … | + new ReadBlobs(settings.preludeBlobs) | |
257 … | + .pipe(collect(function (err, preludeData) { | |
258 … | + if (err) throw err | |
259 … | + preludeData = makePreludeBufferFromPrelude( | |
260 … | + substitute(preludeData.toString('utf8'), { | |
261 … | + '%VIRTUAL_FILESYSTEM%': JSON.stringify(vfs), | |
262 … | + '%DEFAULT_ENTRYPOINT%': JSON.stringify(settings.entrypoint) | |
263 … | + }) | |
264 … | + ) | |
265 … | + preludeSize = preludeData.length | |
266 … | + writeStream.write(preludeData, function (err) { | |
267 … | + if (err) throw err | |
268 … | + inject() | |
269 … | + }) | |
270 … | + })) | |
271 … | +} | |
272 … | + | |
273 … | +function inject() { | |
274 … | + injectPlaceholders(binFd, placeholders, { | |
275 … | + BAKERY: makeBakeryValueFromBakes(settings.bakes), | |
276 … | + PAYLOAD_POSITION: payloadPosition, | |
277 … | + PAYLOAD_SIZE: payloadSize, | |
278 … | + PRELUDE_POSITION: preludePosition, | |
279 … | + PRELUDE_SIZE: preludeSize | |
280 … | + }, function (err) { | |
281 … | + if (err) throw err | |
282 … | + fs.close(binFd, function (err) { | |
283 … | + if (err) throw err | |
284 … | + putBins() | |
285 … | + }) | |
286 … | + }) | |
287 … | +} | |
288 … | + | |
289 … | +function addOtherBin(name, snap) { | |
290 … | + var script = `#!/bin/sh | |
291 … | +PKG_ENTRYPOINT=${snap} exec ${settings.binName} "$@" | |
292 … | +` | |
293 … | + var binPath = path.join(binDir, name) | |
294 … | + var binTmp = binPath + '~' | |
295 … | + fs.writeFileSync(binTmp, script) | |
296 … | + plusx(binTmp) | |
297 … | + fs.renameSync(binTmp, binPath) | |
298 … | + console.log(binPath) | |
299 … | +} | |
300 … | + | |
301 … | +function putBins() { | |
302 … | + plusx(binTmp) | |
303 … | + fs.renameSync(binTmp, binPath) | |
304 … | + | |
305 … | + var others = settings.otherEntrypoints | |
306 … | + if (others) for (var name in others) { | |
307 … | + var snap = others[name] | |
308 … | + addOtherBin(name, snap) | |
309 … | + } | |
310 … | + | |
311 … | + console.log(binPath) | |
312 … | + process.exit(0) | |
313 … | +} | |
314 … | + | |
315 … | +writeNodeBinary() |
prelude/install.sh | ||
---|---|---|
@@ -1,0 +1,33 @@ | ||
1 … | +#!/bin/sh | |
2 … | +{ | |
3 … | +blobs_base=${SSB_BLOBS_BASE:-http://localhost:8989/blobs/get/} | |
4 … | + | |
5 … | +system=$(uname -sm) || return 1 | |
6 … | +case "$system" in | |
7 … | +%SWITCH_NODE_BLOBS% | |
8 … | + *) | |
9 … | + printf 'System not recognized: %s\n' "$system" >&2 | |
10 … | + exit 1 | |
11 … | + ;; | |
12 … | +esac | |
13 … | + | |
14 … | +install_js_blob='%INSTALL_JS_BLOB%' | |
15 … | +exec 3<<EOF | |
16 … | +%SETTINGS% | |
17 … | +EOF | |
18 … | + | |
19 … | +tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/ssb-pkg.XXXXXXXXX") || return 1 | |
20 … | +node_bin="$tmp_dir/ssb-pkg-node-$$" | |
21 … | +install_js="$tmp_dir/ssb-pkg-install-$$" | |
22 … | + | |
23 … | +while ! for id; do | |
24 … | + echo "$id" >&2 | |
25 … | + curl -sS "$blobs_base$id" | |
26 … | +done | gunzip > "$node_bin"; do sleep 2; done | |
27 … | +chmod +x "$node_bin" | |
28 … | + | |
29 … | +while ! curl -sS "$blobs_base$install_js_blob" -o "$install_js" | |
30 … | +do sleep 1; done | |
31 … | +"$node_bin" "$install_js" | |
32 … | +rm -rf "$tmp_dir" | |
33 … | +} |
Built with git-ssb-web