git ssb

1
import { STORE_BLOB, STORE_CONTENT, snapshotify } from '../prelude/common.js';
2
import { log, wasReported } from './log.js';
3
import Multistream from 'multistream';
4
import assert from 'assert';
5
import { fabricateTwice } from './fabricator.js';
6
import fs from 'fs';
7
import path from 'path';
8
import zlib from 'zlib';
9
import intoStream from 'into-stream';
10
import streamMeter from 'stream-meter';
11
import addBlobs from './add-blobs.js';
12
13
function snapshotifyMap(files, slash) {
14
  var o = {}
15
  for (var name in files) {
16
    var file = files[name]
17
    o[name] = snapshotify(file, slash)
18
  }
19
  return o
20
}
21
22
function substitute(template, vars) {
23
  return Object.keys(vars).reduce(function (str, key) {
24
    return str.replace(key, vars[key])
25
  }, template)
26
}
27
28
function addGzBlobs(file, cb) {
29
  var idFile = file + '.gzids'
30
  fs.readFile(idFile, 'utf8', function (err, contents) {
31
    if (!err) return cb(null, contents.trim().split('\n'))
32
    if (err.code !== 'ENOENT') return cb(err)
33
    fs.createReadStream(file)
34
      .pipe(zlib.createGzip())
35
      .pipe(addBlobs(function (err, ids) {
36
        if (err) return cb(err)
37
        fs.writeFile(idFile, ids.join('\n')+'\n', function (err) {
38
          if (err) console.trace(err)
39
          cb(null, ids)
40
        })
41
      }))
42
  })
43
}
44
45
function caseNodeBlobs(ids, pattern) {
46
  return '  ' + pattern + ') set -- \\\n    ' + ids.map(function (id) {
47
    return "'" + id + "'"
48
  }).join(' \\\n    ') + ';;\n'
49
}
50
51
function switchNodeBlobs(nodeBlobIds, targets) {
52
  return Object.keys(targets).map(function (t) {
53
    var target = targets[t]
54
    var ids = nodeBlobIds[target.platform + '-' + target.arch]
55
    if (!ids) throw new Error('target without node blob ids: ' + t)
56
    var system =
57
      target.platform === 'linux' ? 'Linux' :
58
      target.platform === 'macos' ? 'Darwin*' :
59
      null
60
    var machine =
61
      target.arch === 'x64' ? 'x86_64' :
62
      target.arch === 'x86' ? 'i686' :
63
      target.arch === 'armv7' ? 'armv7l' :
64
      target.arch === 'arm64' ? 'aarch64' :
65
      null
66
    if (!system) throw new Error('Unknown system: ' + target.platform)
67
    if (!machine) throw new Error('Unknown machine: ' + target.arch)
68
    var pattern = system + '\\ ' + machine
69
    return caseNodeBlobs(ids, pattern)
70
  }).filter(Boolean).join('\n')
71
}
72
73
export default function ({ backpack, bakes, slash, targets, fabricator, output, binName }) {
74
  return new Promise((resolve, reject) => {
75
    if (!Buffer.alloc) {
76
      throw wasReported('Your node.js does not have Buffer.alloc. Please upgrade!');
77
    }
78
79
    var nodeBlobIds = {}
80
    var preludeBlobIds, payloadBlobIds, vfsBlobIds, installJsBlobId
81
    var waiting = 3
82
83
    let { prelude, entrypoint, otherEntrypoints, stripes } = backpack;
84
    entrypoint = snapshotify(entrypoint, slash);
85
    otherEntrypoints = snapshotifyMap(otherEntrypoints, slash)
86
    stripes = stripes.slice();
87
88
    var fileBlobs = {/* <blobGroup>: {<snap>: <id>} */}
89
90
    const vfs = {};
91
    for (const stripe of stripes) {
92
      let { snap } = stripe;
93
      snap = snapshotify(snap, slash);
94
      if (!vfs[snap]) vfs[snap] = {};
95
    }
96
97
    let meter;
98
    let count = 0;
99
100
    function pipeToNewMeter (s) {
101
      meter = streamMeter();
102
      return s.pipe(meter);
103
    }
104
105
    for (const target of targets) {
106
      waiting++
107
      addGzBlobs(target.binaryPath, function (err, ids) {
108
        if (err) return reject(err)
109
        nodeBlobIds[target.platform + '-' + target.arch] = ids
110
        if (!--waiting) next()
111
      })
112
    }
113
114
    var installJs = path.join(__dirname, '../prelude/install.js')
115
    fs.createReadStream(installJs)
116
      .pipe(addBlobs(onAddInstallJsBlobs));
117
118
    let track = 0;
119
    let prevStripe;
120
121
    let payloadPosition;
122
    let payloadSize;
123
    let preludePosition;
124
    let preludeSize;
125
126
    new Multistream((cb) => {
127
      if (prevStripe && !prevStripe.skip) {
128
        let { snap, store } = prevStripe;
129
        snap = snapshotify(snap, slash);
130
        vfs[snap][store] = [ track, meter.bytes ];
131
        track += meter.bytes;
132
      }
133
134
      if (!stripes.length) {
135
        return cb()
136
      }
137
138
      // clone to prevent 'skip' propagate
139
      // to other targets, since same stripe
140
      // is used for several targets
141
      const stripe = Object.assign({}, stripes.shift());
142
      prevStripe = stripe;
143
144
      if (stripe.buffer) {
145
        if (stripe.store === STORE_BLOB) {
146
          const snap = snapshotify(stripe.snap, slash);
147
          return fabricateTwice(bakes, fabricator, snap, stripe.buffer, (error, buffer) => {
148
            if (error) {
149
              log.warn(error.message);
150
              stripe.skip = true;
151
              return cb(undefined, intoStream(Buffer.alloc(0)));
152
            }
153
154
            cb(undefined, pipeToNewMeter(intoStream(buffer)));
155
          });
156
        } else {
157
          return cb(undefined, pipeToNewMeter(intoStream(stripe.buffer)));
158
        }
159
      } else
160
      if (stripe.file) {
161
        if (stripe.file === output) {
162
          return cb(wasReported(
163
            'Trying to take executable into executable', stripe.file
164
          ));
165
        }
166
167
        var blobGroup = stripe.blobGroup
168
        if (blobGroup) {
169
          var readFile = fs.createReadStream(stripe.file)
170
          return readFile.pipe(addBlobs(function (err, ids) {
171
            if (err) return cb(err)
172
            var snap = snapshotify(stripe.snap, slash);
173
            var blobs = fileBlobs[blobGroup] || (fileBlobs[blobGroup] = {})
174
            blobs[snap] = ids
175
            // vfs entry will be added during install
176
            stripe.skip = true
177
            cb(null, intoStream(Buffer.alloc(0)))
178
          }))
179
        }
180
181
        assert.equal(stripe.store, STORE_CONTENT); // others must be buffers from walker
182
        return cb(undefined, pipeToNewMeter(fs.createReadStream(stripe.file)));
183
      } else {
184
        assert(false, 'producer: bad stripe');
185
      }
186
    }).on('error', (error) => {
187
      reject(error);
188
    }).pipe(
189
      zlib.createGzip()
190
    ).pipe(
191
      addBlobs(onAddPayloadBlobs)
192
    ).on('error', (error) => {
193
      reject(error);
194
    });
195
196
    function onAddInstallJsBlobs(err, ids) {
197
      if (err) return reject(err)
198
      if (ids.length > 1) {
199
        return reject(new Error('install.js must fit in one blob'))
200
      }
201
      installJsBlobId = ids
202
      if (!--waiting) next()
203
    }
204
205
    addBlobs(function (err, ids) {
206
      if (err) return reject(err)
207
      preludeBlobIds = ids
208
      if (!--waiting) next()
209
    }).end(prelude)
210
211
    function onAddPayloadBlobs(err, ids) {
212
      if (err) return reject(err)
213
      payloadBlobIds = ids
214
      addBlobs(function (err, ids) {
215
        vfsBlobIds = ids
216
        if (err) return reject(err)
217
        if (!--waiting) next()
218
      }).end(JSON.stringify(vfs))
219
    }
220
221
    function next() {
222
      var installSh = path.join(__dirname, '../prelude/install.sh')
223
      var installScriptTemplate = fs.readFileSync(installSh, 'utf8')
224
      var settings = {
225
        '%INSTALL_JS_BLOB%': installJsBlobId,
226
        '%SWITCH_NODE_BLOBS%': switchNodeBlobs(nodeBlobIds, targets),
227
        '%SETTINGS%': JSON.stringify({
228
          preludeBlobs: preludeBlobIds,
229
          payloadBlobs: payloadBlobIds,
230
          vfsBlobs: vfsBlobIds,
231
          binName: binName,
232
          entrypoint: entrypoint,
233
          otherEntrypoints: otherEntrypoints,
234
          bakes: bakes,
235
          fileBlobs: fileBlobs,
236
        }, null, 2)
237
      }
238
      var installScript = substitute(installScriptTemplate, settings)
239
      fs.writeFile(output, installScript, function (err) {
240
        if (err) return reject(err)
241
        resolve()
242
      })
243
    }
244
245
  });
246
}
247