git ssb

3+

cel / ssb-npm-registry



Tree: 90588c9cde81fa9487446808b33f14d73c821d63

Files: 90588c9cde81fa9487446808b33f14d73c821d63 / index.js

16849 bytesRaw
1var http = require('http')
2var os = require('os')
3var path = require('path')
4var fs = require('fs')
5var crypto = require('crypto')
6var pkg = require('./package')
7
8function pullOnce(data) {
9  var ended
10  return function (abort, cb) {
11    if (ended || (ended = abort)) return cb(ended)
12    ended = true
13    cb(null, data)
14  }
15}
16
17function escapeHTML(str) {
18  return String(str)
19    .replace(/</g, '&lt;')
20    .replace(/>/g, '&gt;')
21}
22
23function onceify(fn, self) {
24  var cbs = [], err, data
25  return function (cb) {
26    if (fn) {
27      cbs.push(cb)
28      fn.call(self, function (_err, _data) {
29        err = _err, data = _data
30        var _cbs = cbs
31        cbs = null
32        while (_cbs.length) _cbs.shift()(err, data)
33      })
34      fn = null
35    } else if (cbs) {
36      cbs.push(cb)
37    } else {
38      cb(err, data)
39    }
40  }
41}
42
43function pkgLockToRegistryPkgs(pkgLock, wsPort) {
44  // convert a package-lock.json file into data for serving as an npm registry
45  var hasNonBlobUrl = false
46  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
47  var pkgs = {}
48  var queue = [pkgLock, pkgLock.name]
49  while (queue.length) {
50    var dep = queue.shift(), name = queue.shift()
51    if (name) {
52      var pkg = pkgs[name] || (pkgs[name] = {
53        _id: name,
54        name: name,
55        versions: {}
56      })
57      if (dep.version && dep.integrity && dep.resolved) {
58        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
59        pkg.versions[dep.version] = {
60          dist: {
61            integrity: dep.integrity,
62            tarball: dep.resolved
63          }
64        }
65      }
66    }
67    if (dep.dependencies) for (var depName in dep.dependencies) {
68      queue.push(dep.dependencies[depName], depName)
69    }
70  }
71  pkgs._hasNonBlobUrl = hasNonBlobUrl
72  return pkgs
73}
74
75function npmLogin(registryAddress, cb) {
76  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
77  var filename = path.join(os.homedir(), '.npmrc')
78  fs.readFile(filename, 'utf8', function (err, data) {
79    if (err && err.code === 'ENOENT') data = ''
80    else if (err) return cb(new Error(err.stack))
81    var lines = data ? data.split('\n') : []
82    if (lines.indexOf(tokenLine) > -1) return cb()
83    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
84    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
85    fs.appendFile(filename, line, cb)
86  })
87}
88
89function formatHost(host) {
90  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
91}
92
93exports.name = 'npm-registry'
94exports.version = '1.0.0'
95exports.manifest = {
96  getAddress: 'async'
97}
98exports.init = function (sbot, config) {
99  var port = config.npm ? config.npm.port : 8043
100  var host = config.npm && config.npm.host || null
101  var autoAuth = config.npm && config.npm.autoAuth !== false
102  var getAddressCbs = []
103
104  var server = http.createServer(exports.respond(sbot, config))
105  var getAddress = onceify(function (cb) {
106    server.on('error', cb)
107    server.listen(port, host, function () {
108      server.removeListener('error', cb)
109      var regHost = formatHost(host || 'localhost')
110      var regUrl = 'http://' + regHost + ':' + this.address().port + '/'
111      if (autoAuth) npmLogin(regUrl, next)
112      else next()
113      function next(err) {
114        cb(err, regUrl)
115      }
116    })
117    sbot.on('close', function () {
118      server.close()
119    })
120  })
121
122  getAddress(function (err, addr) {
123    if (err) return console.error(err)
124    console.log('[npm-registry] Listening on ' + addr)
125  })
126
127  return {
128    getAddress: getAddress
129  }
130}
131
132exports.respond = function (sbot, config) {
133  var reg = new SsbNpmRegistryServer(sbot, config)
134  return function (req, res) {
135    new Req(reg, req, res).serve()
136  }
137}
138
139function publishMsg(sbot, value, cb) {
140  var gotExpectedPrevious = false
141  sbot.publish(value, function next(err, msg) {
142    if (err && /^expected previous:/.test(err.message)) {
143      // retry once on this error
144      if (gotExpectedPrevious) return cb(err)
145      gotExpectedPrevious = true
146      return sbot.publish(value, next)
147    }
148    cb(err, msg)
149  })
150}
151
152function publishMentions(sbot, mentions, cb) {
153  // console.error("publishing %s mentions", mentions.length)
154  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
155  publishMsg(sbot, {
156    type: 'npm-packages',
157    mentions: mentions,
158  }, cb)
159}
160
161exports.publishPkgMentions = function (sbot, mentions, cb) {
162  // try to fit the mentions into as few messages as possible,
163  // while fitting under the message size limit.
164  var msgs = []
165  ;(function next(i, chunks) {
166    if (i >= mentions.length) return cb(null, msgs)
167    var chunkLen = Math.ceil(mentions.length / chunks)
168    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
169      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
170      if (err && msgs.length) return onPartialPublish(err)
171      if (err) return cb(err)
172      msgs.push(msg)
173      next(i + chunkLen, chunks)
174    })
175  })(0, 1)
176  function onPartialPublish(err) {
177    var remaining = mentions.length - i
178    return cb(new Error('Published messages ' +
179      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
180      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
181  }
182}
183
184function SsbNpmRegistryServer(sbot, config) {
185  this.sbot = sbot
186  this.config = config
187  this.links2 = sbot.links2
188  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
189  this.wsPort = config.ws && Number(config.ws.port) || '8989'
190  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
191    + this.wsPort + '/blobs/get/'
192  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
193}
194
195SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
196SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
197
198SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
199  var self = this
200  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
201  ;(function next(i) {
202    if (i >= ids.length) return cb()
203    self.sbot.blobs.push(ids[i], function (err) {
204      if (err) return cb(err)
205      next(i+1)
206    })
207  })(0)
208}
209
210SsbNpmRegistryServer.prototype.blobDist = function (id) {
211  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
212  if (!m) throw new Error('bad blob id: ' + id)
213  return {
214    integrity: m[2] + '-' + m[1],
215    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
216  }
217}
218
219SsbNpmRegistryServer.prototype.getMentions = function (name) {
220  return this.links2.read({
221    query: [
222      {$filter: {rel: ['mentions', name]}},
223      {$filter: {dest: {$prefix: '&'}}},
224      {$map: {
225        name: ['rel', 1],
226        size: ['rel', 2],
227        link: 'dest',
228        author: 'source',
229        ts: 'ts'
230      }}
231    ]
232  })
233}
234
235SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
236  var self = this
237  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
238
239  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
240    if (err) return cb(new Error(err.stack || err))
241    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
242    if (pkgs._hasNonBlobUrl) {
243      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
244    }
245
246    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
247    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
248
249    self.sbot.blobs.add(function (err, id) {
250      if (err) return cb(new Error(err.stack || err))
251      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
252      var versions = pkg.versions || (pkg.versions = {})
253      pkg.versions[sbotPkgLock.version] = {
254        dist: self.blobDist(id)
255      }
256      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
257      distTags.latest = sbotPkgLock.version
258      next()
259    })(self.sbot.bootstrap.pack())
260
261    function next() {
262      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
263        encoding: 'utf8'
264      }, function (err, bootstrapScript) {
265        if (err) return cb(err)
266        var script = bootstrapScript + '\n' +
267          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2)
268
269        self.sbot.blobs.add(function (err, id) {
270          if (err) return cb(new Error(err.stack || err))
271          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
272          if (!m) return cb(new Error('bad blob id: ' + id))
273          cb(null, {
274            name: sbotPkgLock.name,
275            blob: id,
276            hashType: m[2],
277            hashBuf: Buffer.from(m[1], 'base64'),
278          })
279        })(pullOnce(script))
280      })
281    }
282  })
283}
284
285function Req(server, req, res) {
286  this.server = server
287  this.req = req
288  this.res = res
289  this.blobsToPush = []
290}
291
292Req.prototype.serve = function () {
293  // console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
294  var pathname = this.req.url.replace(/\?.*/, '')
295  if (pathname === '/') return this.serveHome()
296  if (pathname === '/bootstrap') return this.serveBootstrap()
297  if (pathname === '/-/whoami') return this.serveWhoami()
298  if (pathname === '/-/ping') return this.respond(200, true)
299  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
300  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
301  return this.respond(404)
302}
303
304Req.prototype.respond = function (status, message) {
305  this.res.writeHead(status, {'content-type': 'application/json'})
306  this.res.end(message && JSON.stringify(message, 0, 2))
307}
308
309Req.prototype.respondError = function (status, message) {
310  this.respond(status, {error: message})
311}
312
313var bootstrapName = 'ssb-npm-bootstrap'
314
315Req.prototype.serveHome = function () {
316  var self = this
317  self.res.writeHead(200, {'content-type': 'text/html'})
318  var port = 8044
319  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
320    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
321    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
322    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
323    '</body></html>')
324}
325
326Req.prototype.serveBootstrap = function () {
327  var self = this
328  self.server.getBootstrapInfo(function (err, info) {
329    if (err) return this.respondError(err.stack || err)
330    var pkgNameText = info.name
331    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
332    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
333    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
334    var blobsHostname = httpHost + ':' + self.server.wsPort
335    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
336    var pkgHashText = info.hashBuf.toString('hex')
337    var hashCmd = info.hashType + 'sum'
338
339    var script =
340      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
341      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
342      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
343        'npm install -g ' + info.name + ' &&\n' +
344      'sbot server'
345
346    self.res.writeHead(200, {'content-type': 'text/plain'})
347    self.res.end(script)
348  })
349}
350
351Req.prototype.serveWhoami = function () {
352  var self = this
353  self.server.sbot.whoami(function (err, feed) {
354    if (err) return self.respondError(err.stack || err)
355    self.respond(200, {username: feed.id})
356  })
357}
358
359Req.prototype.serveUser1 = function () {
360  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
361}
362
363function decodeName(name) {
364  var parts = name.replace(/\.tgz$/, '').split(':')
365  return {
366    name: parts[1],
367    version: parts[2],
368    distTag: parts[3],
369  }
370}
371
372Req.prototype.servePkg = function (pathname) {
373  var self = this
374  var parts = pathname.split('/')
375  var pkgName = parts.shift()
376  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
377  if (parts.length > 0) return this.respondError(404)
378  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
379  var obj = {
380    _id: pkgName,
381    name: pkgName,
382    'dist-tags': {},
383    versions: {}
384  }
385  var oldest, newest
386  var getMention = self.server.getMentions({$prefix: 'npm:' + pkgName + ':'})
387  getMention(null, function next(err, mention) {
388    if (err === true) return self.respond(200, obj)
389    if (err) return self.respondError(500, err.stack || err)
390    var data = decodeName(mention.name)
391    if (!data.version) return
392    if (data.distTag) obj['dist-tags'][data.distTag] = data.version
393    obj.versions[data.version] = {
394      author: {
395        url: mention.author
396      },
397      dist: self.server.blobDist(mention.link)
398    }
399    getMention(null, next)
400  })
401}
402
403var localhosts = {
404  '::1': true,
405  '127.0.0.1': true,
406  '::ffff:127.0.0.1': true,
407}
408
409Req.prototype.publishPkg = function (pkgName) {
410  var self = this
411  var remoteAddress = self.req.socket.remoteAddress
412  if (!(remoteAddress in localhosts)) {
413    return self.respondError(403, 'You may not publish as this user.')
414  }
415
416  var chunks = []
417  self.req.on('data', function (data) {
418    chunks.push(data)
419  })
420  self.req.on('end', function () {
421    var data
422    try {
423      data = JSON.parse(Buffer.concat(chunks))
424    } catch(e) {
425      return self.respondError(400, e.stack)
426    }
427    return self.publishPkg2(pkgName, data || {})
428  })
429}
430
431Req.prototype.publishPkg2 = function (name, data) {
432  var self = this
433  if (data.users) console.trace('[npm-registry] users property is not supported')
434  var attachments = data._attachments || {}
435  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
436  var waiting = 0
437  Object.keys(attachments).forEach(function (filename) {
438    waiting++
439    var tarball = new Buffer(attachments[filename].data, 'base64')
440    var length = attachments[filename].length
441    if (length && length !== tarball.length) return self.respondError(400,
442      'Length mismatch for attachment \'' + filename + '\'')
443    self.server.sbot.blobs.add(function (err, id) {
444      if (err) return self.respondError(500,
445        'Adding attachment \'' + filename + '\' as blob failed')
446      self.blobsToPush.push(id)
447      links[filename] = {link: id, size: tarball.length}
448      if (!--waiting) next()
449    })(pullOnce(tarball))
450  })
451  function next() {
452    try {
453      self.publishPkg3(name, data, links)
454    } catch(e) {
455      self.respondError(500, e.stack || e)
456    }
457  }
458}
459
460Req.prototype.publishPkg3 = function (name, data, links) {
461  var self = this
462  var versions = data.versions || {}
463  var linksByVersion = {/* <version>: link */}
464
465  // associate tarball blobs with versions
466  for (var version in versions) {
467    var pkg = versions[version]
468    if (!pkg) return self.respondError(400, 'Bad package object')
469    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
470    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
471    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
472    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
473    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
474    var filename = m[1]
475    var link = links[filename]
476    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
477    // TODO?: try to find missing tarball mentioned in other messages
478    if (pkg.version && pkg.version !== version)
479      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
480    linksByVersion[version] = link
481    link.version = version
482  }
483
484  // associate blobs with dist-tags
485  var tags = data['dist-tags'] || {}
486  for (var tag in tags) {
487    var version = tags[tag]
488    var link = linksByVersion[version]
489    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
490    // TODO?: support setting dist-tag without version,
491    // by looking up a tarball blob for the version
492    link.tag = tag
493  }
494
495  // compute blob links to publish
496  var mentions = []
497  for (var filename in links) {
498    var link = links[filename] || {}
499    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
500    mentions.push({
501      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
502      link: link.link,
503      size: link.size,
504    })
505  }
506  return self.publishPkgs(mentions)
507}
508
509Req.prototype.publishPkgs = function (mentions) {
510  var self = this
511  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
512    if (err) self.respondError(500, err.stack || err)
513    self.server.pushBlobs(self.blobsToPush, function (err) {
514      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
515      self.respond(201)
516      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
517    })
518  })
519}
520

Built with git-ssb-web