git ssb

1
#!/usr/bin/env node
2
3
var http = require('http')
4
var fs = require('fs')
5
var proc = require('child_process')
6
var path = require('path')
7
var URL = require('url')
8
var http = require('http')
9
var https = require('https')
10
var crypto = require('crypto')
11
var Transform = require('stream').Transform
12
var SsbNpmRegistry = require('../')
13
var pullFile = require('pull-file')
14
var lru = require('hashlru')
15
var memo = require('asyncmemo')
16
17
var ssbAppname = process.env.ssb_appname || 'ssb'
18
var ssbPath = process.env.ssb_path || path.join(process.env.HOME, '.' + ssbAppname)
19
var blobsPath = path.join(ssbPath, 'blobs')
20
var configPath = path.join(ssbPath, 'config')
21
var blobsTmpPath = path.join(blobsPath, 'tmp')
22
var numTmpBlobs = 0
23
24
var host = 'localhost'
25
var port = null
26
var msgsUrl = null
27
var blobsUrl = null
28
var server
29
var listenHostname
30
var branches = []
31
32
function shift(args) {
33
  if (!args.length) return usage(1)
34
  return args.shift()
35
}
36
37
function version() {
38
  var pkg = require('../package')
39
  console.log(pkg.name + ' ' + pkg.version)
40
}
41
42
function usage(code) {
43
  (code ? process.stderr : process.stdout)
44
    .write(fs.readFileSync(path.join(__dirname, 'usage.txt'), 'utf8'))
45
  process.exit(code)
46
}
47
48
function main(args) {
49
  var viewerUrl = null
50
  var wsUrl = null
51
52
  var cmd
53
  var cmdArgs = []
54
  while (args.length) {
55
    var arg = args.shift()
56
    switch (arg) {
57
      case '--help': return usage(0)
58
      case '--version': return version()
59
      case '--host': host = shift(args); break
60
      case '--port': port = shift(args); break
61
      case '--msgs-url': msgsUrl = shift(args); break
62
      case '--blobs-url': blobsUrl = shift(args); break
63
      case '--ws-url': wsUrl = shift(args); break
64
      case '--viewer-url': viewerUrl = shift(args); break
65
      case '--branch': branches.push(shift(args)); break
66
      case '--exec': cmd = shift(args); break
67
      case '--': cmdArgs.push.apply(cmdArgs, args.splice(0)); break
68
      default: cmdArgs.push(arg); break
69
    }
70
  }
71
72
  if (wsUrl && viewerUrl) {
73
    throw '--ws-url and --viewer-url options conflict'
74
  }
75
  if (wsUrl && (msgsUrl || blobsUrl)) {
76
    throw '--ws-url option conflicts with --msgs-url and --blobs-url'
77
  }
78
  if (viewerUrl && (msgsUrl || blobsUrl)) {
79
    throw '--viewer-url option conflicts with --msgs-url and --blobs-url'
80
  }
81
82
  if (wsUrl) {
83
    wsUrl = wsUrl.replace(/\/+$/, '')
84
    blobsUrl = wsUrl + '/blobs/get/%s'
85
    msgsUrl = wsUrl + '/msg/%s'
86
  }
87
88
  if (viewerUrl) {
89
    viewerUrl = viewerUrl.replace(/\/+$/, '')
90
    blobsUrl = viewerUrl + '/%s'
91
    msgsUrl = viewerUrl + '/%s.json'
92
  }
93
94
  var doExec = cmdArgs.length > 0 || cmd != null
95
  if (!doExec) port = 8044
96
97
  server = http.createServer()
98
  server.listen(port, host, function () {
99
    port = server.address().port
100
    server.on('request', SsbNpmRegistry.respond({
101
      whoami: whoami,
102
      get: ssbGet,
103
      blobs: {
104
        size: blobsSize,
105
        want: blobsWant,
106
        get: blobsGet,
107
      }
108
    }, {
109
      npm: ssbNpmConfig
110
    }))
111
112
    if (doExec) {
113
      var registryUrl = 'http://' + host + ':' + port
114
        + '/' + branches.map(encodeURIComponent).join(',')
115
      var env = {}
116
      for (var k in process.env) env[k] = process.env[k]
117
      env.npm_config_registry = registryUrl
118
      if (!cmd) cmd = 'npm'
119
      if (cmd === 'npm') {
120
        cmdArgs.unshift('--no-update-notifier')
121
        cmdArgs.unshift('--fetch-retries=0')
122
        cmdArgs.unshift('--download={registry}/-/prebuild/{name}-v{version}-{runtime}-v{abi}-{platform}{libc}-{arch}.tar.gz')
123
        cmdArgs.unshift('--' + registryUrl.replace(/^https?:/, '') + ':_authToken=1')
124
        cmdArgs.unshift('--registry={registry}')
125
      } else if (cmd === 'yarn') {
126
        cmdArgs.unshift('--registry={registry}')
127
      }
128
      cmdArgs.forEach(function (arg, i) {
129
        cmdArgs[i] = arg.replace(/\{registry\}/g, registryUrl)
130
      })
131
      var child = proc.spawn(cmd, cmdArgs, {
132
        env: env,
133
        stdio: 'inherit'
134
      })
135
      child.on('exit', process.exit)
136
      process.on('SIGINT', function () { child.kill('SIGINT') })
137
      process.on('SIGTERM', function () { child.kill('SIGTERM') })
138
      process.on('uncaughtException', function (e) {
139
        console.error(e)
140
        child.kill('SIGKILL')
141
        process.exit(1)
142
      })
143
    } else {
144
      printServerListening()
145
    }
146
  })
147
}
148
149
function printServerListening() {
150
  var addr = server.address()
151
  listenHostname = typeof addr === 'string' ? 'unix:' + addr
152
    : addr.family === 'IPv6' ? '[' + addr.address + ']:' + addr.port
153
    : addr.address + ':' + addr.port
154
  console.log('Listening on http://' + listenHostname)
155
}
156
157
function readNext(fn) {
158
  var next
159
  return function (end, cb) {
160
    if (next) return next(end, cb)
161
    try {
162
      fn(function (err, _next) {
163
        if (err) return cb(err)
164
        next = _next
165
        next(null, cb)
166
      })
167
    } catch(e) {
168
      cb(e)
169
    }
170
  }
171
}
172
173
function serveStatus(res, code, message) {
174
  res.writeHead(code, message)
175
  res.end(message)
176
}
177
178
var blobIdRegex = /^&([A-Za-z0-9\/+]{43}=)\.sha256$/
179
180
function idToBuf(id) {
181
  var m = blobIdRegex.exec(id)
182
  if (!m) return null
183
  return new Buffer(m[1], 'base64')
184
}
185
186
function blobFilename(buf) {
187
  if (!buf) return null
188
  var str = buf.toString('hex')
189
  return path.join(blobsPath, 'sha256', str.slice(0, 2), str.slice(2))
190
}
191
192
function getRemoteBlob(id, cb) {
193
  var url = blobsUrl.replace('%s', id)
194
  return /^https:\/\//.test(url) ? https.get(url, cb) : http.get(url, cb)
195
}
196
197
function getRemoteMsg(id, cb) {
198
  var url = msgsUrl.replace('%s', encodeURIComponent(id))
199
  return /^https:\/\//.test(url) ? https.get(url, cb) : http.get(url, cb)
200
}
201
202
function mkdirp(dir, cb) {
203
  fs.stat(dir, function (err, stats) {
204
    if (!err) return cb()
205
    fs.mkdir(dir, function (err) {
206
      if (!err) return cb()
207
      mkdirp(path.dirname(dir), function (err) {
208
        if (err) return cb(err)
209
        fs.mkdir(dir, cb)
210
      })
211
    })
212
  })
213
}
214
215
function rename(src, dest, cb) {
216
  mkdirp(path.dirname(dest), function (err) {
217
    if (err) return cb(err)
218
    fs.rename(src, dest, cb)
219
  })
220
}
221
222
function hash(arr) {
223
  return arr.reduce(function (hash, item) {
224
    return hash.update(String(item))
225
  }, crypto.createHash('sha256')).digest('base64')
226
}
227
228
function fetchAddBlob(id, hash, filename, opts, cb) {
229
  opts = opts || {}
230
  var readIt = opts.readIt !== false
231
  if (!blobsUrl) return cb(new Error('Missing blobs URL'))
232
  var req = getRemoteBlob(id, function (res) {
233
    req.removeListener('error', cb)
234
    if (res.statusCode !== 200) return cb(new Error(res.statusMessage))
235
    mkdirp(blobsTmpPath, function (err) {
236
      if (err) return res.destroy(), cb(err)
237
      var blobTmpPath = path.join(blobsTmpPath, Date.now() + '-' + numTmpBlobs++)
238
      fs.open(blobTmpPath, 'w+', function (err, fd) {
239
        if (err) return res.destroy(), cb(err)
240
        var writeStream = fs.createWriteStream(null, {
241
          fd: fd, flags: 'w+', autoClose: false})
242
        var hasher = crypto.createHash('sha256')
243
        var hashThrough = new Transform({
244
          transform: function (data, encoding, cb) {
245
            hasher.update(data)
246
            cb(null, data)
247
          }
248
        })
249
        res.pipe(hashThrough).pipe(writeStream, {end: false})
250
        res.on('error', function (err) {
251
          writeStream.end(function (err1) {
252
            fs.unlink(blobTmpPath, function (err2) {
253
              cb(err2 || err1 || err)
254
            })
255
          })
256
        })
257
        hashThrough.on('end', function () {
258
          var receivedHash = hasher.digest()
259
          if (receivedHash.compare(hash)) {
260
            writeStream.end(function (err) {
261
              fs.unlink(blobTmpPath, function (err1) {
262
                cb(err1 || err || new Error('mismatched hash'))
263
              })
264
            })
265
          } else {
266
            res.unpipe(hashThrough)
267
            rename(blobTmpPath, filename, function (err) {
268
              if (err) return console.error(err)
269
              if (readIt) cb(null, toPull(fs.createReadStream(null, {fd: fd, start: 0})))
270
              else fs.close(fd, function (err) {
271
                if (err) return cb(err)
272
                cb(null)
273
              })
274
            })
275
          }
276
        })
277
      })
278
    })
279
  })
280
  req.on('error', cb)
281
}
282
283
function getAddBlob(id, hash, filename, cb) {
284
  fs.access(filename, fs.constants.R_OK, function (err) {
285
    if (err && err.code === 'ENOENT') return fetchAddBlob(id, hash, filename, {}, cb)
286
    if (err) return cb(err)
287
    cb(null, pullFile(filename))
288
  })
289
}
290
291
function whoami(cb) {
292
  cb(null, {id: 'foo'})
293
}
294
295
var ssbGet = memo({
296
  cache: lru(1000)
297
}, function (id, cb) {
298
  if (!msgsUrl) return cb(new Error('Missing msgs URL'))
299
  var req = getRemoteMsg(id, function (res) {
300
    req.removeListener('error', cb)
301
    if (res.statusCode !== 200)
302
      return cb(new Error('unable to get msg ' + id + ': ' + res.statusMessage))
303
    var bufs = []
304
    res.on('data', function (buf) {
305
      bufs.push(buf)
306
    })
307
    res.on('error', onError)
308
    function onError(err) {
309
      cb(err)
310
      cb = null
311
    }
312
    res.on('end', function () {
313
      if (!cb) return
314
      res.removeListener('error', onError)
315
      var buf = Buffer.concat(bufs)
316
      try {
317
        var obj = JSON.parse(buf.toString('utf8'))
318
        if (Array.isArray(obj)) obj = obj[0]
319
        if (!obj) return cb(new Error('empty message'))
320
        if (obj.value) obj = obj.value
321
        gotMsg(obj, cb)
322
      } catch(e) {
323
        return cb(e)
324
      }
325
    })
326
  })
327
  req.removeListener('error', cb)
328
  function gotMsg(value, cb) {
329
    var encoded = new Buffer(JSON.stringify(value, null, 2), 'binary')
330
    var hash = crypto.createHash('sha256').update(encoded).digest('base64')
331
    var id1 = '%' + hash + '.sha256'
332
    if (id !== id1) return cb(new Error('mismatched hash ' + id + ' ' + id1))
333
    cb(null, value)
334
  }
335
})
336
337
function blobsSize(id, cb) {
338
  var filename = blobFilename(idToBuf(id))
339
  if (!filename) return cb(new Error('bad id'))
340
  fs.stat(filename, function (err, stats) {
341
    if (err) return cb(err)
342
    cb(null, stats.size)
343
  })
344
}
345
346
function blobsWant(id, cb) {
347
  var hash = idToBuf(id)
348
  var filename = blobFilename(hash)
349
  fetchAddBlob(id, hash, filename, {readIt: false}, function (err) {
350
    if (err) return cb(err)
351
    cb(null, true)
352
  })
353
}
354
355
function blobsGet(id) {
356
  return readNext(function (cb) {
357
    var hash = idToBuf(id)
358
    var filename = blobFilename(hash)
359
    getAddBlob(id, hash, filename, cb)
360
  })
361
}
362
363
var ssbNpmConfig
364
try {
365
  ssbNpmConfig = JSON.stringify(fs.readFileSync(configPath)).npm
366
} catch(e) {
367
}
368
369
main(process.argv.slice(2))
370