import { STORE_BLOB, STORE_CONTENT, snapshotify } from '../prelude/common.js'; import { log, wasReported } from './log.js'; import Multistream from 'multistream'; import assert from 'assert'; import { fabricateTwice } from './fabricator.js'; import fs from 'fs'; import path from 'path'; import zlib from 'zlib'; import intoStream from 'into-stream'; import streamMeter from 'stream-meter'; function snapshotifyMap(files, slash) { var o = {} for (var name in files) { var file = files[name] o[name] = snapshotify(file, slash) } return o } function substitute(template, vars) { return Object.keys(vars).reduce(function (str, key) { return str.replace(key, vars[key]) }, template) } function getJson(file, cb) { fs.readFile(file, 'utf8', function (err, data) { if (err) return cb(err) var obj try { obj = JSON.parse(data) } catch(e) { return cb(e) } cb(null, obj) }) } function addGzBlobs(ssb, file, cb) { var linksFile = file + '.gzlinks' getJson(linksFile, function (err, links) { if (!err) return cb(null, links) if (err.code !== 'ENOENT' && err.message !== 'SyntaxError') return cb(err) fs.createReadStream(file) .pipe(zlib.createGzip()) .pipe(ssb.addBlobs(function (err, links) { if (err) return cb(err) fs.writeFile(linksFile, JSON.stringify(links, null, 2), function (err) { if (err) console.trace(err) cb(null, links) }) })) }) } function caseNodeBlobs(links, pattern) { return ' ' + pattern + ') set -- \\\n ' + links.map(function (link) { return "'" + link.link + "'" }).join(' \\\n ') + ';;\n' } function switchNodeBlobs(nodeBlobs, targets) { return Object.keys(targets).map(function (t) { var target = targets[t] var links = nodeBlobs[target.platform + '-' + target.arch] if (!links) throw new Error('target without node blobs: ' + t) var system = target.platform === 'linux' ? 'Linux' : target.platform === 'macos' ? 'Darwin*' : null var machine = target.arch === 'x64' ? 'x86_64' : target.arch === 'x86' ? 'i686' : target.arch === 'armv7' ? 'armv7l' : target.arch === 'arm64' ? 'aarch64' : null if (!system) throw new Error('Unknown system: ' + target.platform) if (!machine) throw new Error('Unknown machine: ' + target.arch) var pattern = system + '\\ ' + machine return caseNodeBlobs(links, pattern) }).filter(Boolean).join('\n') } export default function ({ backpack, bakes, slash, targets, fabricator, output, binName, ssb }) { return new Promise((resolve, reject) => { if (!Buffer.alloc) { throw wasReported('Your node.js does not have Buffer.alloc. Please upgrade!'); } var nodeBlobs = {} var preludeBlobs, payloadBlobs, vfsBlobs, installJsBlobId var waiting = 3 let { prelude, entrypoint, otherEntrypoints, stripes } = backpack; entrypoint = snapshotify(entrypoint, slash); otherEntrypoints = snapshotifyMap(otherEntrypoints, slash) stripes = stripes.slice(); var fileBlobs = {/* : {: } */} const vfs = {}; for (const stripe of stripes) { let { snap } = stripe; snap = snapshotify(snap, slash); if (!vfs[snap]) vfs[snap] = {}; } let meter; let count = 0; function pipeToNewMeter (s) { meter = streamMeter(); return s.pipe(meter); } for (const target of targets) { waiting++ addGzBlobs(ssb, target.binaryPath, function (err, links) { if (err) return reject(err) nodeBlobs[target.platform + '-' + target.arch] = links if (!--waiting) next() }) } var installJs = path.join(__dirname, '../prelude/install.js') fs.createReadStream(installJs) .pipe(ssb.addBlobs(onAddInstallJsBlobs)); let track = 0; let prevStripe; let payloadPosition; let payloadSize; let preludePosition; let preludeSize; new Multistream((cb) => { if (prevStripe && !prevStripe.skip) { let { snap, store } = prevStripe; snap = snapshotify(snap, slash); vfs[snap][store] = [ track, meter.bytes ]; track += meter.bytes; } if (!stripes.length) { return cb() } // clone to prevent 'skip' propagate // to other targets, since same stripe // is used for several targets const stripe = Object.assign({}, stripes.shift()); prevStripe = stripe; if (stripe.buffer) { if (stripe.store === STORE_BLOB) { const snap = snapshotify(stripe.snap, slash); return fabricateTwice(bakes, fabricator, snap, stripe.buffer, (error, buffer) => { if (error) { log.warn(error.message); stripe.skip = true; return cb(undefined, intoStream(Buffer.alloc(0))); } cb(undefined, pipeToNewMeter(intoStream(buffer))); }); } else { return cb(undefined, pipeToNewMeter(intoStream(stripe.buffer))); } } else if (stripe.file) { if (stripe.file === output) { return cb(wasReported( 'Trying to take executable into executable', stripe.file )); } var blobGroup = stripe.blobGroup if (blobGroup) { var readFile = fs.createReadStream(stripe.file) return readFile.pipe(ssb.addBlobs(function (err, links) { if (err) return cb(err) var snap = snapshotify(stripe.snap, slash); var blobs = fileBlobs[blobGroup] || (fileBlobs[blobGroup] = {}) blobs[snap] = links // vfs entry will be added during install stripe.skip = true cb(null, intoStream(Buffer.alloc(0))) })) } assert.equal(stripe.store, STORE_CONTENT); // others must be buffers from walker return cb(undefined, pipeToNewMeter(fs.createReadStream(stripe.file))); } else { assert(false, 'producer: bad stripe'); } }).on('error', (error) => { reject(error); }).pipe( zlib.createGzip() ).pipe( ssb.addBlobs(onAddPayloadBlobs) ).on('error', (error) => { reject(error); }); function onAddInstallJsBlobs(err, links) { if (err) return reject(err) if (links.length > 1) { return reject(new Error('install.js must fit in one blob')) } installJsBlobId = links[0].link if (!--waiting) next() } var writable = ssb.addBlobs(function (err, links) { if (err) return reject(err) preludeBlobs = links if (!--waiting) next() }) writable.write(prelude) writable.end() // pull-stream-to-stream writable doesn't support .end(data) function onAddPayloadBlobs(err, links) { if (err) return reject(err) payloadBlobs = links var writable = ssb.addBlobs(function (err, links) { vfsBlobs = links if (err) return reject(err) if (!--waiting) next() }) writable.write(JSON.stringify(vfs)) writable.end() } function next() { var installSh = path.join(__dirname, '../prelude/install.sh') var installScriptTemplate = fs.readFileSync(installSh, 'utf8') var settings = { '%INSTALL_JS_BLOB%': installJsBlobId, '%SWITCH_NODE_BLOBS%': switchNodeBlobs(nodeBlobs, targets), '%SETTINGS%': JSON.stringify({ preludeBlobs: preludeBlobs, payloadBlobs: payloadBlobs, vfsBlobs: vfsBlobs, binName: binName, entrypoint: entrypoint, otherEntrypoints: otherEntrypoints, bakes: bakes, fileBlobs: fileBlobs, }, null, 2) } var installScript = substitute(installScriptTemplate, settings) fs.writeFile(output, installScript, function (err) { if (err) return reject(err) resolve() }) } }); }