git ssb

1
import { STORE_BLOB, STORE_CONTENT, snapshotify } from '../prelude/common.js';
2
import { log, wasReported } from './log.js';
3
import Multistream from 'multistream';
4
import assert from 'assert';
5
import { fabricateTwice } from './fabricator.js';
6
import fs from 'fs';
7
import path from 'path';
8
import zlib from 'zlib';
9
import intoStream from 'into-stream';
10
import streamMeter from 'stream-meter';
11
12
function snapshotifyMap(files, slash) {
13
  var o = {}
14
  for (var name in files) {
15
    var file = files[name]
16
    o[name] = snapshotify(file, slash)
17
  }
18
  return o
19
}
20
21
function substitute(template, vars) {
22
  return Object.keys(vars).reduce(function (str, key) {
23
    return str.replace(key, vars[key])
24
  }, template)
25
}
26
27
function getJson(file, cb) {
28
  fs.readFile(file, 'utf8', function (err, data) {
29
    if (err) return cb(err)
30
    var obj
31
    try { obj = JSON.parse(data) }
32
    catch(e) { return cb(e) }
33
    cb(null, obj)
34
  })
35
}
36
37
function addGzBlobs(ssb, file, cb) {
38
  var linksFile = file + '.gzlinks'
39
  getJson(linksFile, function (err, links) {
40
    if (!err) return cb(null, links)
41
    if (err.code !== 'ENOENT' && err.message !== 'SyntaxError') return cb(err)
42
    fs.createReadStream(file)
43
      .pipe(zlib.createGzip())
44
      .pipe(ssb.addBlobs(function (err, links) {
45
        if (err) return cb(err)
46
        fs.writeFile(linksFile, JSON.stringify(links, null, 2), function (err) {
47
          if (err) console.trace(err)
48
          cb(null, links)
49
        })
50
      }))
51
  })
52
}
53
54
function caseNodeBlobs(links, pattern) {
55
  return '  ' + pattern + ') set -- \\\n    ' + links.map(function (link) {
56
    return "'" + link.link + "'"
57
  }).join(' \\\n    ') + ';;\n'
58
}
59
60
function switchNodeBlobs(nodeBlobs, targets) {
61
  return Object.keys(targets).map(function (t) {
62
    var target = targets[t]
63
    var links = nodeBlobs[target.platform + '-' + target.arch]
64
    if (!links) throw new Error('target without node blobs: ' + t)
65
    var system =
66
      target.platform === 'linux' ? 'Linux' :
67
      target.platform === 'macos' ? 'Darwin*' :
68
      null
69
    var machine =
70
      target.arch === 'x64' ? 'x86_64' :
71
      target.arch === 'x86' ? 'i686' :
72
      target.arch === 'armv7' ? 'armv7l' :
73
      target.arch === 'arm64' ? 'aarch64' :
74
      null
75
    if (!system) throw new Error('Unknown system: ' + target.platform)
76
    if (!machine) throw new Error('Unknown machine: ' + target.arch)
77
    var pattern = system + '\\ ' + machine
78
    return caseNodeBlobs(links, pattern)
79
  }).filter(Boolean).join('\n')
80
}
81
82
export default function ({ backpack, bakes, slash, targets, fabricator, output, binName, ssb }) {
83
  return new Promise((resolve, reject) => {
84
    if (!Buffer.alloc) {
85
      throw wasReported('Your node.js does not have Buffer.alloc. Please upgrade!');
86
    }
87
88
    var nodeBlobs = {}
89
    var preludeBlobs, payloadBlobs, vfsBlobs, installJsBlobId
90
    var waiting = 3
91
92
    let { prelude, entrypoint, otherEntrypoints, stripes } = backpack;
93
    entrypoint = snapshotify(entrypoint, slash);
94
    otherEntrypoints = snapshotifyMap(otherEntrypoints, slash)
95
    stripes = stripes.slice();
96
97
    var fileBlobs = {/* <blobGroup>: {<snap>: <id>} */}
98
99
    const vfs = {};
100
    for (const stripe of stripes) {
101
      let { snap } = stripe;
102
      snap = snapshotify(snap, slash);
103
      if (!vfs[snap]) vfs[snap] = {};
104
    }
105
106
    let meter;
107
    let count = 0;
108
109
    function pipeToNewMeter (s) {
110
      meter = streamMeter();
111
      return s.pipe(meter);
112
    }
113
114
    for (const target of targets) {
115
      waiting++
116
      addGzBlobs(ssb, target.binaryPath, function (err, links) {
117
        if (err) return reject(err)
118
        nodeBlobs[target.platform + '-' + target.arch] = links
119
        if (!--waiting) next()
120
      })
121
    }
122
123
    var installJs = path.join(__dirname, '../prelude/install.js')
124
    fs.createReadStream(installJs)
125
      .pipe(ssb.addBlobs(onAddInstallJsBlobs));
126
127
    let track = 0;
128
    let prevStripe;
129
130
    let payloadPosition;
131
    let payloadSize;
132
    let preludePosition;
133
    let preludeSize;
134
135
    new Multistream((cb) => {
136
      if (prevStripe && !prevStripe.skip) {
137
        let { snap, store } = prevStripe;
138
        snap = snapshotify(snap, slash);
139
        vfs[snap][store] = [ track, meter.bytes ];
140
        track += meter.bytes;
141
      }
142
143
      if (!stripes.length) {
144
        return cb()
145
      }
146
147
      // clone to prevent 'skip' propagate
148
      // to other targets, since same stripe
149
      // is used for several targets
150
      const stripe = Object.assign({}, stripes.shift());
151
      prevStripe = stripe;
152
153
      if (stripe.buffer) {
154
        if (stripe.store === STORE_BLOB) {
155
          const snap = snapshotify(stripe.snap, slash);
156
          return fabricateTwice(bakes, fabricator, snap, stripe.buffer, (error, buffer) => {
157
            if (error) {
158
              log.warn(error.message);
159
              stripe.skip = true;
160
              return cb(undefined, intoStream(Buffer.alloc(0)));
161
            }
162
163
            cb(undefined, pipeToNewMeter(intoStream(buffer)));
164
          });
165
        } else {
166
          return cb(undefined, pipeToNewMeter(intoStream(stripe.buffer)));
167
        }
168
      } else
169
      if (stripe.file) {
170
        if (stripe.file === output) {
171
          return cb(wasReported(
172
            'Trying to take executable into executable', stripe.file
173
          ));
174
        }
175
176
        var blobGroup = stripe.blobGroup
177
        if (blobGroup) {
178
          var readFile = fs.createReadStream(stripe.file)
179
          return readFile.pipe(ssb.addBlobs(function (err, links) {
180
            if (err) return cb(err)
181
            var snap = snapshotify(stripe.snap, slash);
182
            var blobs = fileBlobs[blobGroup] || (fileBlobs[blobGroup] = {})
183
            blobs[snap] = links
184
            // vfs entry will be added during install
185
            stripe.skip = true
186
            cb(null, intoStream(Buffer.alloc(0)))
187
          }))
188
        }
189
190
        assert.equal(stripe.store, STORE_CONTENT); // others must be buffers from walker
191
        return cb(undefined, pipeToNewMeter(fs.createReadStream(stripe.file)));
192
      } else {
193
        assert(false, 'producer: bad stripe');
194
      }
195
    }).on('error', (error) => {
196
      reject(error);
197
    }).pipe(
198
      zlib.createGzip()
199
    ).pipe(
200
      ssb.addBlobs(onAddPayloadBlobs)
201
    ).on('error', (error) => {
202
      reject(error);
203
    });
204
205
    function onAddInstallJsBlobs(err, links) {
206
      if (err) return reject(err)
207
      if (links.length > 1) {
208
        return reject(new Error('install.js must fit in one blob'))
209
      }
210
      installJsBlobId = links[0].link
211
      if (!--waiting) next()
212
    }
213
214
    var writable = ssb.addBlobs(function (err, links) {
215
      if (err) return reject(err)
216
      preludeBlobs = links
217
      if (!--waiting) next()
218
    })
219
    writable.write(prelude)
220
    writable.end()
221
    // pull-stream-to-stream writable doesn't support .end(data)
222
223
    function onAddPayloadBlobs(err, links) {
224
      if (err) return reject(err)
225
      payloadBlobs = links
226
      var writable = ssb.addBlobs(function (err, links) {
227
        vfsBlobs = links
228
        if (err) return reject(err)
229
        if (!--waiting) next()
230
      })
231
      writable.write(JSON.stringify(vfs))
232
      writable.end()
233
    }
234
235
    function next() {
236
      var installSh = path.join(__dirname, '../prelude/install.sh')
237
      var installScriptTemplate = fs.readFileSync(installSh, 'utf8')
238
      var settings = {
239
        '%INSTALL_JS_BLOB%': installJsBlobId,
240
        '%SWITCH_NODE_BLOBS%': switchNodeBlobs(nodeBlobs, targets),
241
        '%SETTINGS%': JSON.stringify({
242
          preludeBlobs: preludeBlobs,
243
          payloadBlobs: payloadBlobs,
244
          vfsBlobs: vfsBlobs,
245
          binName: binName,
246
          entrypoint: entrypoint,
247
          otherEntrypoints: otherEntrypoints,
248
          bakes: bakes,
249
          fileBlobs: fileBlobs,
250
        }, null, 2)
251
      }
252
      var installScript = substitute(installScriptTemplate, settings)
253
      fs.writeFile(output, installScript, function (err) {
254
        if (err) return reject(err)
255
        resolve()
256
      })
257
    }
258
259
  });
260
}
261