git ssb

3+

cel / ssb-npm-registry



Tree: 47eb321ffd75f51568db7204562dbaa274e07bb5

Files: 47eb321ffd75f51568db7204562dbaa274e07bb5 / index.js

17120 bytesRaw
1var http = require('http')
2var os = require('os')
3var path = require('path')
4var fs = require('fs')
5var crypto = require('crypto')
6var pkg = require('./package')
7
8function pullOnce(data) {
9  var ended
10  return function (abort, cb) {
11    if (ended || (ended = abort)) return cb(ended)
12    ended = true
13    cb(null, data)
14  }
15}
16
17function escapeHTML(str) {
18  return String(str)
19    .replace(/</g, '&lt;')
20    .replace(/>/g, '&gt;')
21}
22
23function onceify(fn, self) {
24  var cbs = [], err, data
25  return function (cb) {
26    if (fn) {
27      cbs.push(cb)
28      fn.call(self, function (_err, _data) {
29        err = _err, data = _data
30        var _cbs = cbs
31        cbs = null
32        while (_cbs.length) _cbs.shift()(err, data)
33      })
34      fn = null
35    } else if (cbs) {
36      cbs.push(cb)
37    } else {
38      cb(err, data)
39    }
40  }
41}
42
43function pkgLockToRegistryPkgs(pkgLock, wsPort) {
44  // convert a package-lock.json file into data for serving as an npm registry
45  var hasNonBlobUrl = false
46  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
47  var pkgs = {}
48  var queue = [pkgLock, pkgLock.name]
49  while (queue.length) {
50    var dep = queue.shift(), name = queue.shift()
51    if (name) {
52      var pkg = pkgs[name] || (pkgs[name] = {
53        _id: name,
54        name: name,
55        versions: {}
56      })
57      if (dep.version && dep.integrity && dep.resolved) {
58        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
59        pkg.versions[dep.version] = {
60          dist: {
61            integrity: dep.integrity,
62            tarball: dep.resolved
63          }
64        }
65      }
66    }
67    if (dep.dependencies) for (var depName in dep.dependencies) {
68      queue.push(dep.dependencies[depName], depName)
69    }
70  }
71  pkgs._hasNonBlobUrl = hasNonBlobUrl
72  return pkgs
73}
74
75function npmLogin(registryAddress, cb) {
76  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
77  var filename = path.join(os.homedir(), '.npmrc')
78  fs.readFile(filename, 'utf8', function (err, data) {
79    if (err && err.code === 'ENOENT') data = ''
80    else if (err) return cb(new Error(err.stack))
81    var lines = data ? data.split('\n') : []
82    if (lines.indexOf(tokenLine) > -1) return cb()
83    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
84    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
85    fs.appendFile(filename, line, cb)
86  })
87}
88
89function formatHost(host) {
90  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
91}
92
93exports.name = 'npm-registry'
94exports.version = '1.0.0'
95exports.manifest = {
96  getAddress: 'async'
97}
98exports.init = function (sbot, config) {
99  var port = config.npm ? config.npm.port : 8043
100  var host = config.npm && config.npm.host || null
101  var autoAuth = config.npm && config.npm.autoAuth !== false
102  var getAddressCbs = []
103
104  var server = http.createServer(exports.respond(sbot, config))
105  var getAddress = onceify(function (cb) {
106    server.on('error', cb)
107    server.listen(port, host, function () {
108      server.removeListener('error', cb)
109      var regHost = formatHost(host || 'localhost')
110      var regUrl = 'http://' + regHost + ':' + this.address().port + '/'
111      if (autoAuth) npmLogin(regUrl, next)
112      else next()
113      function next(err) {
114        cb(err, regUrl)
115      }
116    })
117    sbot.on('close', function () {
118      server.close()
119    })
120  })
121
122  getAddress(function (err, addr) {
123    if (err) return console.error(err)
124    console.log('[npm-registry] Listening on ' + addr)
125  })
126
127  return {
128    getAddress: getAddress
129  }
130}
131
132exports.respond = function (sbot, config) {
133  var reg = new SsbNpmRegistryServer(sbot, config)
134  return function (req, res) {
135    new Req(reg, req, res).serve()
136  }
137}
138
139function publishMsg(sbot, value, cb) {
140  var gotExpectedPrevious = false
141  sbot.publish(value, function next(err, msg) {
142    if (err && /^expected previous:/.test(err.message)) {
143      // retry once on this error
144      if (gotExpectedPrevious) return cb(err)
145      gotExpectedPrevious = true
146      return sbot.publish(value, next)
147    }
148    cb(err, msg)
149  })
150}
151
152function publishMentions(sbot, mentions, cb) {
153  // console.error("publishing %s mentions", mentions.length)
154  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
155  publishMsg(sbot, {
156    type: 'npm-packages',
157    mentions: mentions,
158  }, cb)
159}
160
161exports.publishPkgMentions = function (sbot, mentions, cb) {
162  // try to fit the mentions into as few messages as possible,
163  // while fitting under the message size limit.
164  var msgs = []
165  ;(function next(i, chunks) {
166    if (i >= mentions.length) return cb(null, msgs)
167    var chunkLen = Math.ceil(mentions.length / chunks)
168    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
169      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
170      if (err && msgs.length) return onPartialPublish(err)
171      if (err) return cb(err)
172      msgs.push(msg)
173      next(i + chunkLen, chunks)
174    })
175  })(0, 1)
176  function onPartialPublish(err) {
177    var remaining = mentions.length - i
178    return cb(new Error('Published messages ' +
179      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
180      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
181  }
182}
183
184function SsbNpmRegistryServer(sbot, config) {
185  this.sbot = sbot
186  this.config = config
187  this.links2 = sbot.links2
188  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
189  this.wsPort = config.ws && Number(config.ws.port) || '8989'
190  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
191    + this.wsPort + '/blobs/get/'
192  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
193}
194
195SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
196SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
197
198SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
199  var self = this
200  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
201  ;(function next(i) {
202    if (i >= ids.length) return cb()
203    self.sbot.blobs.push(ids[i], function (err) {
204      if (err) return cb(err)
205      next(i+1)
206    })
207  })(0)
208}
209
210SsbNpmRegistryServer.prototype.blobDist = function (id) {
211  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
212  if (!m) throw new Error('bad blob id: ' + id)
213  return {
214    integrity: m[2] + '-' + m[1],
215    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
216  }
217}
218
219SsbNpmRegistryServer.prototype.getMentions = function (name) {
220  return this.links2.read({
221    query: [
222      {$filter: {rel: ['mentions', name]}},
223      {$filter: {dest: {$prefix: '&'}}},
224      {$map: {
225        name: ['rel', 1],
226        size: ['rel', 2],
227        link: 'dest',
228        author: 'source',
229        ts: 'ts'
230      }}
231    ]
232  })
233}
234
235SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
236  var self = this
237  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
238
239  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
240    if (err) return cb(new Error(err.stack || err))
241    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
242    if (pkgs._hasNonBlobUrl) {
243      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
244    }
245
246    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
247    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
248
249    self.sbot.blobs.add(function (err, id) {
250      if (err) return cb(new Error(err.stack || err))
251      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
252      var versions = pkg.versions || (pkg.versions = {})
253      pkg.versions[sbotPkgLock.version] = {
254        dist: self.blobDist(id)
255      }
256      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
257      distTags.latest = sbotPkgLock.version
258      next()
259    })(self.sbot.bootstrap.pack())
260
261    function next() {
262      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
263        encoding: 'utf8'
264      }, function (err, bootstrapScript) {
265        if (err) return cb(err)
266        var script = bootstrapScript + '\n' +
267          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2)
268
269        self.sbot.blobs.add(function (err, id) {
270          if (err) return cb(new Error(err.stack || err))
271          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
272          if (!m) return cb(new Error('bad blob id: ' + id))
273          cb(null, {
274            name: sbotPkgLock.name,
275            blob: id,
276            hashType: m[2],
277            hashBuf: Buffer.from(m[1], 'base64'),
278          })
279        })(pullOnce(script))
280      })
281    }
282  })
283}
284
285SsbNpmRegistryServer.prototype.getBootstrapScriptHash = function (cb) {
286  var hasher = crypto.createHash('sha256')
287  hasher.update(data)
288  var hash = hasher.digest()
289  getBootstrapScriptHash = function (cb) {
290    return cb(null, hash)
291  }
292  getBootstrapScriptHash(cb)
293}
294
295function Req(server, req, res) {
296  this.server = server
297  this.req = req
298  this.res = res
299  this.blobsToPush = []
300}
301
302Req.prototype.serve = function () {
303  // console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
304  var pathname = this.req.url.replace(/\?.*/, '')
305  if (pathname === '/') return this.serveHome()
306  if (pathname === '/bootstrap') return this.serveBootstrap()
307  if (pathname === '/-/whoami') return this.serveWhoami()
308  if (pathname === '/-/ping') return this.respond(200, true)
309  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
310  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
311  return this.respond(404)
312}
313
314Req.prototype.respond = function (status, message) {
315  this.res.writeHead(status, {'content-type': 'application/json'})
316  this.res.end(message && JSON.stringify(message, 0, 2))
317}
318
319Req.prototype.respondError = function (status, message) {
320  this.respond(status, {error: message})
321}
322
323var bootstrapName = 'ssb-npm-bootstrap'
324
325Req.prototype.serveHome = function () {
326  var self = this
327  self.res.writeHead(200, {'content-type': 'text/html'})
328  var port = 8044
329  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
330    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
331    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
332    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
333    '</body></html>')
334}
335
336Req.prototype.serveBootstrap = function () {
337  var self = this
338  self.server.getBootstrapInfo(function (err, info) {
339    if (err) return this.respondError(err.stack || err)
340    var pkgNameText = info.name
341    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
342    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
343    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
344    var blobsHostname = httpHost + ':' + self.server.wsPort
345    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
346    var pkgHashText = info.hashBuf.toString('hex')
347    var hashCmd = info.hashType + 'sum'
348
349    var script =
350      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
351      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
352      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
353        'npm install -g ' + info.name + ' &&\n' +
354      'sbot server'
355
356    self.res.writeHead(200, {'content-type': 'text/plain'})
357    self.res.end(script)
358  })
359}
360
361Req.prototype.serveWhoami = function () {
362  var self = this
363  self.server.sbot.whoami(function (err, feed) {
364    if (err) return self.respondError(err.stack || err)
365    self.respond(200, {username: feed.id})
366  })
367}
368
369Req.prototype.serveUser1 = function () {
370  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
371}
372
373function decodeName(name) {
374  var parts = name.replace(/\.tgz$/, '').split(':')
375  return {
376    name: parts[1],
377    version: parts[2],
378    distTag: parts[3],
379  }
380}
381
382Req.prototype.servePkg = function (pathname) {
383  var self = this
384  var parts = pathname.split('/')
385  var pkgName = parts.shift()
386  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
387  if (parts.length > 0) return this.respondError(404)
388  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
389  var obj = {
390    _id: pkgName,
391    name: pkgName,
392    'dist-tags': {},
393    versions: {}
394  }
395  var oldest, newest
396  var getMention = self.server.getMentions({$prefix: 'npm:' + pkgName + ':'})
397  getMention(null, function next(err, mention) {
398    if (err === true) return self.respond(200, obj)
399    if (err) return self.respondError(500, err.stack || err)
400    var data = decodeName(mention.name)
401    if (!data.version) return
402    if (data.distTag) obj['dist-tags'][data.distTag] = data.version
403    obj.versions[data.version] = {
404      author: {
405        url: mention.author
406      },
407      dist: self.server.blobDist(mention.link)
408    }
409    getMention(null, next)
410  })
411}
412
413var localhosts = {
414  '::1': true,
415  '127.0.0.1': true,
416  '::ffff:127.0.0.1': true,
417}
418
419Req.prototype.publishPkg = function (pkgName) {
420  var self = this
421  var remoteAddress = self.req.socket.remoteAddress
422  if (!(remoteAddress in localhosts)) {
423    return self.respondError(403, 'You may not publish as this user.')
424  }
425
426  var chunks = []
427  self.req.on('data', function (data) {
428    chunks.push(data)
429  })
430  self.req.on('end', function () {
431    var data
432    try {
433      data = JSON.parse(Buffer.concat(chunks))
434    } catch(e) {
435      return self.respondError(400, e.stack)
436    }
437    return self.publishPkg2(pkgName, data || {})
438  })
439}
440
441Req.prototype.publishPkg2 = function (name, data) {
442  var self = this
443  if (data.users) console.trace('[npm-registry] users property is not supported')
444  var attachments = data._attachments || {}
445  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
446  var waiting = 0
447  Object.keys(attachments).forEach(function (filename) {
448    waiting++
449    var tarball = new Buffer(attachments[filename].data, 'base64')
450    var length = attachments[filename].length
451    if (length && length !== tarball.length) return self.respondError(400,
452      'Length mismatch for attachment \'' + filename + '\'')
453    self.server.sbot.blobs.add(function (err, id) {
454      if (err) return self.respondError(500,
455        'Adding attachment \'' + filename + '\' as blob failed')
456      self.blobsToPush.push(id)
457      links[filename] = {link: id, size: tarball.length}
458      if (!--waiting) next()
459    })(pullOnce(tarball))
460  })
461  function next() {
462    try {
463      self.publishPkg3(name, data, links)
464    } catch(e) {
465      self.respondError(500, e.stack || e)
466    }
467  }
468}
469
470Req.prototype.publishPkg3 = function (name, data, links) {
471  var self = this
472  var versions = data.versions || {}
473  var linksByVersion = {/* <version>: link */}
474
475  // associate tarball blobs with versions
476  for (var version in versions) {
477    var pkg = versions[version]
478    if (!pkg) return self.respondError(400, 'Bad package object')
479    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
480    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
481    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
482    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
483    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
484    var filename = m[1]
485    var link = links[filename]
486    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
487    // TODO?: try to find missing tarball mentioned in other messages
488    if (pkg.version && pkg.version !== version)
489      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
490    linksByVersion[version] = link
491    link.version = version
492  }
493
494  // associate blobs with dist-tags
495  var tags = data['dist-tags'] || {}
496  for (var tag in tags) {
497    var version = tags[tag]
498    var link = linksByVersion[version]
499    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
500    // TODO?: support setting dist-tag without version,
501    // by looking up a tarball blob for the version
502    link.tag = tag
503  }
504
505  // compute blob links to publish
506  var mentions = []
507  for (var filename in links) {
508    var link = links[filename] || {}
509    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
510    mentions.push({
511      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
512      link: link.link,
513      size: link.size,
514    })
515  }
516  return self.publishPkgs(mentions)
517}
518
519Req.prototype.publishPkgs = function (mentions) {
520  var self = this
521  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
522    if (err) self.respondError(500, err.stack || err)
523    self.server.pushBlobs(self.blobsToPush, function (err) {
524      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
525      self.respond(201)
526      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
527    })
528  })
529}
530

Built with git-ssb-web