git ssb

1
import { STORE_BLOB, STORE_CONTENT, snapshotify } from '../prelude/common.js';
2
import { log, wasReported } from './log.js';
3
import Multistream from 'multistream';
4
import assert from 'assert';
5
import { fabricateTwice } from './fabricator.js';
6
import fs from 'fs';
7
import path from 'path';
8
import zlib from 'zlib';
9
import intoStream from 'into-stream';
10
import streamMeter from 'stream-meter';
11
12
function snapshotifyMap(files, slash) {
13
  var o = {}
14
  for (var name in files) {
15
    var file = files[name]
16
    o[name] = snapshotify(file, slash)
17
  }
18
  return o
19
}
20
21
function substitute(template, vars) {
22
  return Object.keys(vars).reduce(function (str, key) {
23
    return str.replace(key, vars[key])
24
  }, template)
25
}
26
27
function addGzBlobs(ssb, file, cb) {
28
  var idFile = file + '.gzids'
29
  fs.readFile(idFile, 'utf8', function (err, contents) {
30
    if (!err) return cb(null, contents.trim().split('\n'))
31
    if (err.code !== 'ENOENT') return cb(err)
32
    fs.createReadStream(file)
33
      .pipe(zlib.createGzip())
34
      .pipe(ssb.addBlobs(function (err, ids) {
35
        if (err) return cb(err)
36
        fs.writeFile(idFile, ids.join('\n')+'\n', function (err) {
37
          if (err) console.trace(err)
38
          cb(null, ids)
39
        })
40
      }))
41
  })
42
}
43
44
function caseNodeBlobs(ids, pattern) {
45
  return '  ' + pattern + ') set -- \\\n    ' + ids.map(function (id) {
46
    return "'" + id + "'"
47
  }).join(' \\\n    ') + ';;\n'
48
}
49
50
function switchNodeBlobs(nodeBlobIds, targets) {
51
  return Object.keys(targets).map(function (t) {
52
    var target = targets[t]
53
    var ids = nodeBlobIds[target.platform + '-' + target.arch]
54
    if (!ids) throw new Error('target without node blob ids: ' + t)
55
    var system =
56
      target.platform === 'linux' ? 'Linux' :
57
      target.platform === 'macos' ? 'Darwin*' :
58
      null
59
    var machine =
60
      target.arch === 'x64' ? 'x86_64' :
61
      target.arch === 'x86' ? 'i686' :
62
      target.arch === 'armv7' ? 'armv7l' :
63
      target.arch === 'arm64' ? 'aarch64' :
64
      null
65
    if (!system) throw new Error('Unknown system: ' + target.platform)
66
    if (!machine) throw new Error('Unknown machine: ' + target.arch)
67
    var pattern = system + '\\ ' + machine
68
    return caseNodeBlobs(ids, pattern)
69
  }).filter(Boolean).join('\n')
70
}
71
72
export default function ({ backpack, bakes, slash, targets, fabricator, output, binName, ssb }) {
73
  return new Promise((resolve, reject) => {
74
    if (!Buffer.alloc) {
75
      throw wasReported('Your node.js does not have Buffer.alloc. Please upgrade!');
76
    }
77
78
    var nodeBlobIds = {}
79
    var preludeBlobIds, payloadBlobIds, vfsBlobIds, installJsBlobId
80
    var waiting = 3
81
82
    let { prelude, entrypoint, otherEntrypoints, stripes } = backpack;
83
    entrypoint = snapshotify(entrypoint, slash);
84
    otherEntrypoints = snapshotifyMap(otherEntrypoints, slash)
85
    stripes = stripes.slice();
86
87
    var fileBlobs = {/* <blobGroup>: {<snap>: <id>} */}
88
89
    const vfs = {};
90
    for (const stripe of stripes) {
91
      let { snap } = stripe;
92
      snap = snapshotify(snap, slash);
93
      if (!vfs[snap]) vfs[snap] = {};
94
    }
95
96
    let meter;
97
    let count = 0;
98
99
    function pipeToNewMeter (s) {
100
      meter = streamMeter();
101
      return s.pipe(meter);
102
    }
103
104
    for (const target of targets) {
105
      waiting++
106
      addGzBlobs(ssb, target.binaryPath, function (err, ids) {
107
        if (err) return reject(err)
108
        nodeBlobIds[target.platform + '-' + target.arch] = ids
109
        if (!--waiting) next()
110
      })
111
    }
112
113
    var installJs = path.join(__dirname, '../prelude/install.js')
114
    fs.createReadStream(installJs)
115
      .pipe(ssb.addBlobs(onAddInstallJsBlobs));
116
117
    let track = 0;
118
    let prevStripe;
119
120
    let payloadPosition;
121
    let payloadSize;
122
    let preludePosition;
123
    let preludeSize;
124
125
    new Multistream((cb) => {
126
      if (prevStripe && !prevStripe.skip) {
127
        let { snap, store } = prevStripe;
128
        snap = snapshotify(snap, slash);
129
        vfs[snap][store] = [ track, meter.bytes ];
130
        track += meter.bytes;
131
      }
132
133
      if (!stripes.length) {
134
        return cb()
135
      }
136
137
      // clone to prevent 'skip' propagate
138
      // to other targets, since same stripe
139
      // is used for several targets
140
      const stripe = Object.assign({}, stripes.shift());
141
      prevStripe = stripe;
142
143
      if (stripe.buffer) {
144
        if (stripe.store === STORE_BLOB) {
145
          const snap = snapshotify(stripe.snap, slash);
146
          return fabricateTwice(bakes, fabricator, snap, stripe.buffer, (error, buffer) => {
147
            if (error) {
148
              log.warn(error.message);
149
              stripe.skip = true;
150
              return cb(undefined, intoStream(Buffer.alloc(0)));
151
            }
152
153
            cb(undefined, pipeToNewMeter(intoStream(buffer)));
154
          });
155
        } else {
156
          return cb(undefined, pipeToNewMeter(intoStream(stripe.buffer)));
157
        }
158
      } else
159
      if (stripe.file) {
160
        if (stripe.file === output) {
161
          return cb(wasReported(
162
            'Trying to take executable into executable', stripe.file
163
          ));
164
        }
165
166
        var blobGroup = stripe.blobGroup
167
        if (blobGroup) {
168
          var readFile = fs.createReadStream(stripe.file)
169
          return readFile.pipe(ssb.addBlobs(function (err, ids) {
170
            if (err) return cb(err)
171
            var snap = snapshotify(stripe.snap, slash);
172
            var blobs = fileBlobs[blobGroup] || (fileBlobs[blobGroup] = {})
173
            blobs[snap] = ids
174
            // vfs entry will be added during install
175
            stripe.skip = true
176
            cb(null, intoStream(Buffer.alloc(0)))
177
          }))
178
        }
179
180
        assert.equal(stripe.store, STORE_CONTENT); // others must be buffers from walker
181
        return cb(undefined, pipeToNewMeter(fs.createReadStream(stripe.file)));
182
      } else {
183
        assert(false, 'producer: bad stripe');
184
      }
185
    }).on('error', (error) => {
186
      reject(error);
187
    }).pipe(
188
      zlib.createGzip()
189
    ).pipe(
190
      ssb.addBlobs(onAddPayloadBlobs)
191
    ).on('error', (error) => {
192
      reject(error);
193
    });
194
195
    function onAddInstallJsBlobs(err, ids) {
196
      if (err) return reject(err)
197
      if (ids.length > 1) {
198
        return reject(new Error('install.js must fit in one blob'))
199
      }
200
      installJsBlobId = ids
201
      if (!--waiting) next()
202
    }
203
204
    var writable = ssb.addBlobs(function (err, ids) {
205
      if (err) return reject(err)
206
      preludeBlobIds = ids
207
      if (!--waiting) next()
208
    })
209
    writable.write(prelude)
210
    writable.end()
211
    // pull-stream-to-stream writable doesn't support .end(data)
212
213
    function onAddPayloadBlobs(err, ids) {
214
      if (err) return reject(err)
215
      payloadBlobIds = ids
216
      var writable = ssb.addBlobs(function (err, ids) {
217
        vfsBlobIds = ids
218
        if (err) return reject(err)
219
        if (!--waiting) next()
220
      })
221
      writable.write(JSON.stringify(vfs))
222
      writable.end()
223
    }
224
225
    function next() {
226
      var installSh = path.join(__dirname, '../prelude/install.sh')
227
      var installScriptTemplate = fs.readFileSync(installSh, 'utf8')
228
      var settings = {
229
        '%INSTALL_JS_BLOB%': installJsBlobId,
230
        '%SWITCH_NODE_BLOBS%': switchNodeBlobs(nodeBlobIds, targets),
231
        '%SETTINGS%': JSON.stringify({
232
          preludeBlobs: preludeBlobIds,
233
          payloadBlobs: payloadBlobIds,
234
          vfsBlobs: vfsBlobIds,
235
          binName: binName,
236
          entrypoint: entrypoint,
237
          otherEntrypoints: otherEntrypoints,
238
          bakes: bakes,
239
          fileBlobs: fileBlobs,
240
        }, null, 2)
241
      }
242
      var installScript = substitute(installScriptTemplate, settings)
243
      fs.writeFile(output, installScript, function (err) {
244
        if (err) return reject(err)
245
        resolve()
246
      })
247
    }
248
249
  });
250
}
251