git ssb

3+

cel / ssb-npm-registry



Tree: e3dc59bd63e1aee4cb776449e47d41f18b72b205

Files: e3dc59bd63e1aee4cb776449e47d41f18b72b205 / bootstrap / bin.js

11229 bytesRaw
1#!/usr/bin/env node
2
3var http = require('http')
4var fs = require('fs')
5var proc = require('child_process')
6var path = require('path')
7var URL = require('url')
8var http = require('http')
9var https = require('https')
10var crypto = require('crypto')
11var Transform = require('stream').Transform
12var SsbNpmRegistry = require('../')
13var pullFile = require('pull-file')
14var lru = require('hashlru')
15var memo = require('asyncmemo')
16
17var ssbAppname = process.env.ssb_appname || 'ssb'
18var ssbPath = process.env.ssb_path || path.join(process.env.HOME, '.' + ssbAppname)
19var blobsPath = path.join(ssbPath, 'blobs')
20var configPath = path.join(ssbPath, 'config')
21var blobsTmpPath = path.join(blobsPath, 'tmp')
22var numTmpBlobs = 0
23
24var host = null
25var port = null
26var msgsUrl = null
27var blobsUrl = null
28var server
29var listenHostname
30var branches = []
31
32function shift(args) {
33  if (!args.length) return usage(1)
34  return args.shift()
35}
36
37function version() {
38  var pkg = require('../package')
39  console.log(pkg.name + ' ' + pkg.version)
40}
41
42function usage(code) {
43  (code ? process.stderr : process.stdout)
44    .write(fs.readFileSync(path.join(__dirname, 'usage.txt'), 'utf8'))
45  process.exit(code)
46}
47
48function main(args) {
49  var viewerUrl = null
50  var wsUrl = null
51
52  var cmdArgs = []
53  while (args.length) {
54    var arg = args.shift()
55    switch (arg) {
56      case '--help': return usage(0)
57      case '--version': return version()
58      case '--host': host = shift(args); break
59      case '--port': port = shift(args); break
60      case '--msgs-url': msgsUrl = shift(args); break
61      case '--blobs-url': blobsUrl = shift(args); break
62      case '--ws-url': wsUrl = shift(args); break
63      case '--viewer-url': viewerUrl = shift(args); break
64      case '--branch': branches.push(shift(args)); break
65      case '--': cmdArgs.push.apply(cmdArgs, args.splice(0)); break
66      default: cmdArgs.push(arg); break
67    }
68  }
69
70  if (wsUrl && viewerUrl) {
71    throw '--ws-url and --viewer-url options conflict'
72  }
73  if (wsUrl && (msgsUrl || blobsUrl)) {
74    throw '--ws-url option conflicts with --msgs-url and --blobs-url'
75  }
76  if (viewerUrl && (msgsUrl || blobsUrl)) {
77    throw '--viewer-url option conflicts with --msgs-url and --blobs-url'
78  }
79
80  if (wsUrl) {
81    wsUrl = wsUrl.replace(/\/+$/, '')
82    blobsUrl = wsUrl + '/blobs/get/%s'
83    msgsUrl = wsUrl + '/msg/%s'
84  }
85
86  if (viewerUrl) {
87    viewerUrl = viewerUrl.replace(/\/+$/, '')
88    blobsUrl = viewerUrl + '/%s'
89    msgsUrl = viewerUrl + '/%s.json'
90  }
91
92  if (!port && !cmdArgs.length) {
93    port = 8989
94  }
95
96  server = http.createServer(serve)
97  server.listen(port, host, function () {
98    port = server.address().port
99    serveNpm1 = SsbNpmRegistry.respond({
100      whoami: whoami,
101      get: ssbGet,
102      blobs: {
103        size: blobsSize,
104        want: blobsWant,
105        get: blobsGet,
106      }
107    }, {
108      ws: {
109        port: port,
110      },
111      host: host,
112      npm: ssbNpmConfig
113    })
114
115    if (cmdArgs.length) {
116      var cmd = cmdArgs.shift()
117
118      var registryUrl = 'http://' + (host || 'localhost') + ':' + port
119        + '/npm/' + branches.map(encodeURIComponent).join(',') + '/'
120      var env = {}
121      for (var k in process.env) env[k] = process.env[k]
122      env.npm_config_registry = registryUrl
123      if (cmd === 'npm') {
124        cmdArgs.unshift('--no-update-notifier')
125        cmdArgs.unshift('--fetch-retries=0')
126        cmdArgs.unshift('--download=' + registryUrl + '-/prebuild/{name}-v{version}-{runtime}-v{abi}-{platform}{libc}-{arch}.tar.gz')
127      }
128      cmdArgs.forEach(function (arg, i) {
129        cmdArgs[i] = arg.replace(/\{registry\}/g, registryUrl)
130      })
131      var child = proc.spawn(cmd, cmdArgs, {
132        env: env,
133        stdio: 'inherit'
134      })
135      child.on('exit', process.exit)
136      process.on('SIGINT', function () { child.kill('SIGINT') })
137      process.on('SIGTERM', function () { child.kill('SIGTERM') })
138      process.on('uncaughtException', function (e) {
139        console.error(e)
140        child.kill('SIGKILL')
141        process.exit(1)
142      })
143    } else {
144      printServerListening()
145    }
146  })
147}
148
149function printServerListening() {
150  var addr = server.address()
151  listenHostname = typeof addr === 'string' ? 'unix:' + addr
152    : addr.family === 'IPv6' ? '[' + addr.address + ']:' + addr.port
153    : addr.address + ':' + addr.port
154  console.log('Listening on http://' + listenHostname)
155}
156
157function serveStatus(res, code, message) {
158  res.writeHead(code, message)
159  res.end(message)
160}
161
162var blobIdRegex = /^&([A-Za-z0-9\/+]{43}=)\.sha256$/
163
164function idToBuf(id) {
165  var m = blobIdRegex.exec(id)
166  if (!m) return null
167  return new Buffer(m[1], 'base64')
168}
169
170function blobFilename(buf) {
171  if (!buf) return null
172  var str = buf.toString('hex')
173  return path.join(blobsPath, 'sha256', str.slice(0, 2), str.slice(2))
174}
175
176function getRemoteBlob(id, cb) {
177  var url = blobsUrl.replace('%s', id)
178  return /^https:\/\//.test(url) ? https.get(url, cb) : http.get(url, cb)
179}
180
181function getRemoteMsg(id, cb) {
182  var url = msgsUrl.replace('%s', encodeURIComponent(id))
183  return /^https:\/\//.test(url) ? https.get(url, cb) : http.get(url, cb)
184}
185
186function mkdirp(dir, cb) {
187  fs.stat(dir, function (err, stats) {
188    if (!err) return cb()
189    fs.mkdir(dir, function (err) {
190      if (!err) return cb()
191      mkdirp(path.dirname(dir), function (err) {
192        if (err) return cb(err)
193        fs.mkdir(dir, cb)
194      })
195    })
196  })
197}
198
199function rename(src, dest, cb) {
200  mkdirp(path.dirname(dest), function (err) {
201    if (err) return cb(err)
202    fs.rename(src, dest, cb)
203  })
204}
205
206function hash(arr) {
207  return arr.reduce(function (hash, item) {
208    return hash.update(String(item))
209  }, crypto.createHash('sha256')).digest('base64')
210}
211
212function fetchAddBlob(id, hash, filename, opts, cb) {
213  opts = opts || {}
214  var readIt = opts.readIt !== false
215  if (!blobsUrl) return cb(new Error('Missing blobs URL'))
216  var req = getRemoteBlob(id, function (res) {
217    req.removeListener('error', cb)
218    if (res.statusCode !== 200) return cb(new Error(res.statusMessage))
219    mkdirp(blobsTmpPath, function (err) {
220      if (err) return res.destroy(), cb(err)
221      var blobTmpPath = path.join(blobsTmpPath, Date.now() + '-' + numTmpBlobs++)
222      fs.open(blobTmpPath, 'w+', function (err, fd) {
223        if (err) return res.destroy(), cb(err)
224        var writeStream = fs.createWriteStream(null, {
225          fd: fd, flags: 'w+', autoClose: false})
226        var hasher = crypto.createHash('sha256')
227        var hashThrough = new Transform({
228          transform: function (data, encoding, cb) {
229            hasher.update(data)
230            cb(null, data)
231          }
232        })
233        res.pipe(hashThrough).pipe(writeStream, {end: false})
234        res.on('error', function (err) {
235          writeStream.end(function (err1) {
236            fs.unlink(blobTmpPath, function (err2) {
237              cb(err || err1 || err2)
238            })
239          })
240        })
241        hashThrough.on('end', function () {
242          var receivedHash = hasher.digest()
243          if (receivedHash.compare(hash)) {
244            writeStream.end(function (err) {
245              fs.unlink(blobTmpPath, function (err1) {
246                cb(err1 || err || new Error('mismatched hash'))
247              })
248            })
249          } else {
250            res.unpipe(hashThrough)
251            rename(blobTmpPath, filename, function (err) {
252              if (err) return console.error(err)
253              if (readIt) cb(null, fs.createReadStream(null, {fd: fd, start: 0}))
254              else fs.close(fd, function (err) {
255                if (err) return cb(err)
256                cb(null)
257              })
258            })
259          }
260        })
261      })
262    })
263  })
264  req.on('error', cb)
265}
266
267function getAddBlob(id, hash, filename, cb) {
268  fs.access(filename, fs.constants.R_OK, function (err) {
269    if (err && err.code === 'ENOENT') return fetchAddBlob(id, hash, filename, {}, cb)
270    if (err) return cb(err)
271    cb(null, fs.createReadStream(filename))
272  })
273}
274
275function serveBlobsGet(req, res, id) {
276  try { id = decodeURIComponent(id) }
277  catch (e) {}
278  var hash = idToBuf(id)
279  var filename = blobFilename(hash)
280  getAddBlob(id, hash, filename, function (err, stream) {
281    if (err) return serveStatus(res, 500, err.message)
282    if (!stream) return serveStatus(res, 404, 'Blob Not Found')
283    res.writeHead(200)
284    stream.pipe(res)
285  })
286}
287
288function serveMsg(req, res, id) {
289  try { id = decodeURIComponent(id) }
290  catch (e) {}
291  ssbGet(id, function (err, value) {
292    if (err) return serveStatus(res, 400, err.message)
293    if (!msg) return serveStatus(res, 404, 'Msg Not Found')
294    res.writeHead(200, {'Content-Type': 'application/json'})
295    res.end(JSON.stringify({key: id, value: msg}))
296  })
297}
298
299function whoami(cb) {
300  cb(null, {id: 'foo'})
301}
302
303var ssbGet = memo({
304  cache: lru(1000)
305}, function (id, cb) {
306  if (!msgsUrl) return cb(new Error('Missing msgs URL'))
307  var req = getRemoteMsg(id, function (res) {
308    req.removeListener('error', cb)
309    if (res.statusCode !== 200)
310      return cb(new Error('unable to get msg ' + id + ': ' + res.statusMessage))
311    var bufs = []
312    res.on('data', function (buf) {
313      bufs.push(buf)
314    })
315    res.on('error', onError)
316    function onError(err) {
317      cb(err)
318      cb = null
319    }
320    res.on('end', function () {
321      if (!cb) return
322      res.removeListener('error', onError)
323      var buf = Buffer.concat(bufs)
324      try {
325        var obj = JSON.parse(buf.toString('utf8'))
326        if (Array.isArray(obj)) obj = obj[0]
327        if (!obj) return cb(new Error('empty message'))
328        if (obj.value) obj = obj.value
329        gotMsg(obj, cb)
330      } catch(e) {
331        return cb(e)
332      }
333    })
334  })
335  req.removeListener('error', cb)
336  function gotMsg(value, cb) {
337    var encoded = new Buffer(JSON.stringify(value, null, 2), 'binary')
338    var hash = crypto.createHash('sha256').update(encoded).digest('base64')
339    var id1 = '%' + hash + '.sha256'
340    if (id !== id1) return cb(new Error('mismatched hash ' + id + ' ' + id1))
341    cb(null, value)
342  }
343})
344
345function blobsSize(id, cb) {
346  var filename = blobFilename(idToBuf(id))
347  if (!filename) return cb(new Error('bad id'))
348  fs.stat(filename, function (err, stats) {
349    if (err) return cb(err)
350    cb(null, stats.size)
351  })
352}
353
354function blobsWant(id, cb) {
355  var hash = idToBuf(id)
356  var filename = blobFilename(hash)
357  fetchAddBlob(id, hash, filename, {readIt: false}, function (err) {
358    if (err) return cb(err)
359    cb(null, true)
360  })
361}
362
363function blobsGet(id) {
364  var filename = blobFilename(idToBuf(id))
365  if (!filename) return cb(new Error('bad id'))
366  return pullFile(filename)
367}
368
369var ssbNpmConfig
370try {
371  ssbNpmConfig = JSON.stringify(fs.readFileSync(configPath)).npm
372} catch(e) {
373}
374
375var serveNpm1
376function serveNpm(req, res, url) {
377  req.url = url
378  serveNpm1(req, res)
379}
380
381function serve(req, res) {
382  res.setTimeout(0)
383  var p = URL.parse(req.url)
384  if (p.pathname.startsWith('/npm/')) return serveNpm(req, res, p.pathname.substr(4))
385  if (p.pathname.startsWith('/msg/')) return serveMsg(req, res, p.pathname.substr(5))
386  if (p.pathname.startsWith('/blobs/get/')) return serveBlobsGet(req, res, p.pathname.substr(11))
387  return serveStatus(res, 404, 'Not Found')
388}
389
390main(process.argv.slice(2))
391

Built with git-ssb-web