git ssb - Add support for Node native addons · cel-desktop/ssb-pkg@65191f9a

Commit 65191f9a9664f013ebe726bb95a1a54b086e1bd5

Add support for Node native addons

Author: Guillaume Besson <guillaume@besson.co>
Source: https://github.com/zeit/pkg/pull/837

cel committed on 5/11/2020, 3:58:47 PM
Parent: 938b7ea4cfaa3cef6e1690b9ef3ea8bd7a0d2de1

Files changed

README.md	changed
lib/packer.js	changed
lib/walker.js	changed
prelude/bootstrap.js	changed
test/test-50-native-addon-2/node_modules/dependency/time.node	changed
test/test-50-native-addon-3/lib/community/time-y.node	changed
test/test-50-native-addon-3/lib/enterprise/time-z.node	changed
test/test-50-native-addon-3/lib/time-x.node	changed
test/test-50-native-addon-3/node_modules/dependency/time-d.node	changed
test/test-50-native-addon/lib/time.node	changed
test/test-50-can-include-addon/main.js	added
test/test-50-can-include-addon/test-x-index.js	added
test/test-50-can-include-addon/time.node	added
test/test-50-cannot-include-addon/main.js	deleted
test/test-50-cannot-include-addon/test-x-index.js	deleted
test/test-50-cannot-include-addon/time.node	deleted
test/test-50-native-addon-4/lib/time.node	added
test/test-50-native-addon-4/main.js	added
test/test-50-native-addon-4/test-x-index.js	added

README.mdView
		@@ -199,13 +199,20 @@
199	199 …	This way you may even avoid creating `pkg` config for your project.
200	200 …
201	201 …	## Native addons
202	202 …
203		-Native addons (`.node` files) use is supported, but packaging
204		-`.node` files inside the executable is not resolved yet. You have
205		-to deploy native addons used by your project to the same directory
206		-as the executable.
	203 …	+Native addons (`.node` files) use is supported. When `pkg` encounters
	204 …	+a `.node` file in a `require` call, it will package this like an asset.
	205 …	+In some cases (like with the `bindings` package), the module path is generated
	206 …	+dynamicaly and `pkg` won't be able to detect it. In this case, you should
	207 …	+add the `.node` file directly in the `assets` field in `package.json`.
207	208 …
	209 …	+The way NodeJS requires native addon is different from a classic JS
	210 …	+file. It needs to have a file on disk to load it but `pkg` only generate
	211 …	+one file. To circumvent this, `pkg` will create a temporary file on the
	212 …	+disk. These files will stay on the disk after the process has exited
	213 …	+and will be used again on the next process launch.
	214 …	+
208	215 …	When a package, that contains a native module, is being installed,
209	216 …	the native module is compiled against current system-wide Node.js
210	217 …	version. Then, when you compile your project with `pkg`, pay attention
211	218 …	to `--target` option. You should specify the same Node.js version

lib/packer.jsView
		@@ -1,9 +1,9 @@
1	1 …	/* eslint-disable complexity */
2	2 …
3	3 …	import {
4	4 …	STORE_BLOB, STORE_CONTENT, STORE_LINKS,
5		- STORE_STAT, isDotJS, isDotJSON, isDotNODE
	5 …	+ STORE_STAT, isDotJS, isDotJSON
6	6 …	} from '../prelude/common.js';
7	7 …
8	8 …	import { log, wasReported } from './log.js';
9	9 …	import assert from 'assert';
		@@ -33,20 +33,15 @@
33	33 …	}
34	34 …
35	35 …	export default function ({ records, entrypoint, bytecode }) {
36	36 …	const stripes = [];
37		-
38	37 …	for (const snap in records) {
39	38 …	const record = records[snap];
40	39 …	const { file } = record;
41	40 …	if (!hasAnyStore(record)) continue;
42	41 …	assert(record[STORE_STAT], 'packer: no STORE_STAT');
43	42 …
44		- if (isDotNODE(file)) {
45		- continue;
46		- } else {
47		- assert(record[STORE_BLOB] \|\| record[STORE_CONTENT] \|\| record[STORE_LINKS]);
48		- }
	43 …	+ assert(record[STORE_BLOB] \|\| record[STORE_CONTENT] \|\| record[STORE_LINKS]);
49	44 …
50	45 …	if (record[STORE_BLOB] && !bytecode) {
51	46 …	delete record[STORE_BLOB];
52	47 …	if (!record[STORE_CONTENT]) {

lib/walker.jsView
		@@ -521,26 +521,13 @@
521	521 …	file: record.file,
522	522 …	store: STORE_STAT
523	523 …	});
524	524 …
525		- if (isDotNODE(record.file)) {
526		- // provide explicit deployFiles to override
527		- // native addon deployment place. see 'sharp'
528		- if (!marker.hasDeployFiles) {
529		- log.warn('Cannot include addon %1 into executable.', [
530		- 'The addon must be distributed with executable as %2.',
531		- '%1: ' + record.file,
532		- '%2: path-to-executable/' + path.basename(record.file) ]);
533		- }
534		- return; // discard
535		- }
536		-
537	525 …	const derivatives1 = [];
538	526 …	await this.stepActivate(marker, derivatives1);
539	527 …	await this.stepDerivatives(record, marker, derivatives1);
540		-
541	528 …	if (store === STORE_BLOB) {
542		- if (unlikelyJavascript(record.file)) {
	529 …	+ if (unlikelyJavascript(record.file) \|\| isDotNODE(record.file)) {
543	530 …	this.append({
544	531 …	file: record.file,
545	532 …	marker,
546	533 …	store: STORE_CONTENT

prelude/bootstrap.jsView
		@@ -1570,4 +1570,66 @@
1570	1570 …	value: customPromiseExecFunction(require('child_process').execFile)
1571	1571 …	});
1572	1572 …	}
1573	1573 …	}());
	1574 …	+
	1575 …	+// /////////////////////////////////////////////////////////////////
	1576 …	+// PATCH PROCESS ///////////////////////////////////////////////////
	1577 …	+// /////////////////////////////////////////////////////////////////
	1578 …	+
	1579 …	+(function() {
	1580 …	+ const fs = require('fs');
	1581 …	+ var ancestor = {};
	1582 …	+ ancestor.dlopen = process.dlopen;
	1583 …	+
	1584 …	+ process.dlopen = function () {
	1585 …	+ const args = cloneArgs(arguments);
	1586 …	+ const modulePath = args[1];
	1587 …	+ const moduleDirname = require('path').dirname(modulePath);
	1588 …	+ if (insideSnapshot(modulePath)) {
	1589 …	+ // Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path
	1590 …	+ // we need to write the file somewhere on disk first and then load it
	1591 …	+ const moduleContent = fs.readFileSync(modulePath);
	1592 …	+ const moduleBaseName = require('path').basename(modulePath);
	1593 …	+ const hash = require('crypto').createHash('sha256').update(moduleContent).digest('hex');
	1594 …	+ const tmpModulePath = `${require('os').tmpdir()}/${hash}_${moduleBaseName}`;
	1595 …	+ try {
	1596 …	+ fs.statSync(tmpModulePath);
	1597 …	+ } catch (e) {
	1598 …	+ // Most likely this means the module is not on disk yet
	1599 …	+ fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 });
	1600 …	+ }
	1601 …	+ args[1] = tmpModulePath;
	1602 …	+ }
	1603 …	+
	1604 …	+ const unknownModuleErrorRegex = /([^:]+): cannot open shared object file: No such file or directory/;
	1605 …	+ const tryImporting = function (previousErrorMessage) {
	1606 …	+ try {
	1607 …	+ const res = ancestor.dlopen.apply(process, args);
	1608 …	+ return res;
	1609 …	+ } catch (e) {
	1610 …	+ if (e.message === previousErrorMessage) {
	1611 …	+ // we already tried to fix this and it didn't work, give up
	1612 …	+ throw e;
	1613 …	+ }
	1614 …	+ if (e.message.match(unknownModuleErrorRegex)) {
	1615 …	+ // some modules are packaged with dynamic linking and needs to open other files that should be in
	1616 …	+ // the same directory, in this case, we write this file in the same /tmp directory and try to
	1617 …	+ // import the module again
	1618 …	+ const moduleName = e.message.match(unknownModuleErrorRegex)[1];
	1619 …	+ const modulePath = `${moduleDirname}/${moduleName}`;
	1620 …	+ const moduleContent = fs.readFileSync(modulePath);
	1621 …	+ const moduleBaseName = require('path').basename(modulePath);
	1622 …	+ const tmpModulePath = `${require('os').tmpdir()}/${moduleBaseName}`;
	1623 …	+ try {
	1624 …	+ fs.statSync(tmpModulePath);
	1625 …	+ } catch (e) {
	1626 …	+ fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 });
	1627 …	+ }
	1628 …	+ return tryImporting(e.message);
	1629 …	+ }
	1630 …	+ throw e;
	1631 …	+ }
	1632 …	+ }
	1633 …	+ tryImporting();
	1634 …	+ }
	1635 …	+}());

test/test-50-native-addon-2/node_modules/dependency/time.nodeView
		@@ -1,1 +1,1 @@
1		-module.exports = require('path').basename(__filename);
	1 …	+module.exports = 'test';

test/test-50-native-addon-3/lib/community/time-y.nodeView
		@@ -1,1 +1,1 @@
1		-module.exports = require('path').basename(__filename);
	1 …	+module.exports = 'time-y';

test/test-50-native-addon-3/lib/enterprise/time-z.nodeView
		@@ -1,1 +1,1 @@
1		-module.exports = require('path').basename(__filename);
	1 …	+module.exports = 'time-z';

test/test-50-native-addon-3/lib/time-x.nodeView
		@@ -1,1 +1,1 @@
1		-module.exports = require('path').basename(__filename);
	1 …	+module.exports = 'test';

test/test-50-native-addon-3/node_modules/dependency/time-d.nodeView
		@@ -1,1 +1,1 @@
1		-module.exports = require('path').basename(__filename);
	1 …	+module.exports = 'time-d';

test/test-50-native-addon/lib/time.nodeView
		@@ -1,1 +1,1 @@
1		-module.exports = require('path').basename(__filename);
	1 …	+module.exports = 'test';

test/test-50-can-include-addon/main.jsView
		@@ -1,0 +1,31 @@
	1 …	+#!/usr/bin/env node
	2 …	+
	3 …	+'use strict';
	4 …	+
	5 …	+const assert = require('assert');
	6 …	+const utils = require('../utils.js');
	7 …	+
	8 …	+assert(!module.parent);
	9 …	+assert(__dirname === process.cwd());
	10 …	+
	11 …	+const target = process.argv[2] \|\| 'host';
	12 …	+const input = './test-x-index.js';
	13 …	+const output = './test-output.exe';
	14 …	+const standard = 'stdout';
	15 …	+
	16 …	+let right;
	17 …	+
	18 …	+const inspect = (standard === 'stdout')
	19 …	+ ? [ 'inherit', 'pipe', 'inherit' ]
	20 …	+ : [ 'inherit', 'inherit', 'pipe' ];
	21 …	+
	22 …	+right = utils.pkg.sync([
	23 …	+ '--target', target,
	24 …	+ '--output', output, input
	25 …	+], inspect);
	26 …	+
	27 …	+assert(right.indexOf('\x1B\x5B') < 0, 'colors detected');
	28 …	+right = right.replace(/\\/g, '/');
	29 …	+assert(right.indexOf('test-50-can-include-addon/time.node') === -1);
	30 …	+assert(right.indexOf('path-to-executable/time.node') === -1);
	31 …	+utils.vacuum.sync(output);

test/test-50-can-include-addon/test-x-index.jsView
		@@ -1,0 +1,3 @@
	1 …	+'use strict';
	2 …	+
	3 …	+require('./time.node');

test/test-50-can-include-addon/time.nodeView
		@@ -1,0 +1,1 @@
	1 …	+module.exports = 'test';

test/test-50-cannot-include-addon/main.jsView
		@@ -1,31 +1,0 @@
1		-#!/usr/bin/env node
2		-
3		-'use strict';
4		-
5		-const assert = require('assert');
6		-const utils = require('../utils.js');
7		-
8		-assert(!module.parent);
9		-assert(__dirname === process.cwd());
10		-
11		-const target = process.argv[2] \|\| 'host';
12		-const input = './test-x-index.js';
13		-const output = './test-output.exe';
14		-const standard = 'stdout';
15		-
16		-let right;
17		-
18		-const inspect = (standard === 'stdout')
19		- ? [ 'inherit', 'pipe', 'inherit' ]
20		- : [ 'inherit', 'inherit', 'pipe' ];
21		-
22		-right = utils.pkg.sync([
23		- '--target', target,
24		- '--output', output, input
25		-], inspect);
26		-
27		-assert(right.indexOf('\x1B\x5B') < 0, 'colors detected');
28		-right = right.replace(/\\/g, '/');
29		-assert(right.indexOf('test-50-cannot-include-addon/time.node') >= 0);
30		-assert(right.indexOf('path-to-executable/time.node') >= 0);
31		-utils.vacuum.sync(output);

test/test-50-cannot-include-addon/test-x-index.jsView
		@@ -1,3 +1,0 @@
1		-'use strict';
2		-
3		-require('./time.node');

test/test-50-cannot-include-addon/time.nodeView
		@@ -1,1 +1,0 @@
1		-module.exports = require('path').basename(__filename);

test/test-50-native-addon-4/lib/time.nodeView
		@@ -1,0 +1,1 @@
	1 …	+module.exports = 'test';

test/test-50-native-addon-4/main.jsView
		@@ -1,0 +1,36 @@
	1 …	+#!/usr/bin/env node
	2 …	+
	3 …	+'use strict';
	4 …	+
	5 …	+const path = require('path');
	6 …	+const assert = require('assert');
	7 …	+const utils = require('../utils.js');
	8 …	+
	9 …	+assert(!module.parent);
	10 …	+assert(__dirname === process.cwd());
	11 …	+
	12 …	+const host = 'node' + process.version.match(/^v(\d+)/)[1];
	13 …	+const target = process.argv[2] \|\| host;
	14 …	+const input = './test-x-index.js';
	15 …	+const output = './run-time/test-output.exe';
	16 …	+
	17 …	+let left, right;
	18 …	+utils.mkdirp.sync(path.dirname(output));
	19 …	+
	20 …	+left = utils.spawn.sync(
	21 …	+ 'node', [ path.basename(input) ],
	22 …	+ { cwd: path.dirname(input) }
	23 …	+);
	24 …	+
	25 …	+utils.pkg.sync([
	26 …	+ '--target', target,
	27 …	+ '--output', output, input
	28 …	+]);
	29 …	+
	30 …	+right = utils.spawn.sync(
	31 …	+ './' + path.basename(output), [],
	32 …	+ { cwd: path.dirname(output) }
	33 …	+);
	34 …	+
	35 …	+assert.equal(left, right);
	36 …	+utils.vacuum.sync(path.dirname(output));

test/test-50-native-addon-4/test-x-index.jsView
		@@ -1,0 +1,10 @@
	1 …	+/* eslint-disable no-underscore-dangle */
	2 …	+
	3 …	+'use strict';
	4 …	+
	5 …	+var fs = require('fs');
	6 …	+var path = require('path');
	7 …	+var Module = require('module');
	8 …	+Module._extensions['.node'] = Module._extensions['.js'];
	9 …	+console.log(fs.existsSync(path.join(__dirname, 'lib/time.node')));
	10 …	+console.log(require('./lib/time.node'));

Built with git-ssb-web

cel-desktop / ssb-pkg

Commit 65191f9a9664f013ebe726bb95a1a54b086e1bd5

Add support for Node native addons

Files changed