git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
8
function pullOnce(data) {
9
  var ended
10
  return function (abort, cb) {
11
    if (ended || (ended = abort)) return cb(ended)
12
    ended = true
13
    cb(null, data)
14
  }
15
}
16
17
function escapeHTML(str) {
18
  return String(str)
19
    .replace(/</g, '&lt;')
20
    .replace(/>/g, '&gt;')
21
}
22
23
function onceify(fn, self) {
24
  var cbs = [], err, data
25
  return function (cb) {
26
    if (fn) {
27
      cbs.push(cb)
28
      fn.call(self, function (_err, _data) {
29
        err = _err, data = _data
30
        var _cbs = cbs
31
        cbs = null
32
        while (_cbs.length) _cbs.shift()(err, data)
33
      })
34
      fn = null
35
    } else if (cbs) {
36
      cbs.push(cb)
37
    } else {
38
      cb(err, data)
39
    }
40
  }
41
}
42
43
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
44
  // convert a package-lock.json file into data for serving as an npm registry
45
  var hasNonBlobUrl = false
46
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
47
  var pkgs = {}
48
  var queue = [pkgLock, pkgLock.name]
49
  while (queue.length) {
50
    var dep = queue.shift(), name = queue.shift()
51
    if (name) {
52
      var pkg = pkgs[name] || (pkgs[name] = {
53
        _id: name,
54
        name: name,
55
        versions: {}
56
      })
57
      if (dep.version && dep.integrity && dep.resolved) {
58
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
59
        pkg.versions[dep.version] = {
60
          dist: {
61
            integrity: dep.integrity,
62
            tarball: dep.resolved
63
          }
64
        }
65
      }
66
    }
67
    if (dep.dependencies) for (var depName in dep.dependencies) {
68
      queue.push(dep.dependencies[depName], depName)
69
    }
70
  }
71
  pkgs._hasNonBlobUrl = hasNonBlobUrl
72
  return pkgs
73
}
74
75
function npmLogin(registryAddress, cb) {
76
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
77
  var filename = path.join(os.homedir(), '.npmrc')
78
  fs.readFile(filename, 'utf8', function (err, data) {
79
    if (err && err.code === 'ENOENT') data = ''
80
    else if (err) return cb(new Error(err.stack))
81
    var lines = data ? data.split('\n') : []
82
    if (lines.indexOf(tokenLine) > -1) return cb()
83
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
84
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
85
    fs.appendFile(filename, line, cb)
86
  })
87
}
88
89
function formatHost(host) {
90
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
91
}
92
93
exports.name = 'npm-registry'
94
exports.version = '1.0.0'
95
exports.manifest = {
96
  getAddress: 'async'
97
}
98
exports.init = function (sbot, config) {
99
  var conf = config.npm || {}
100
  var port = conf.port || 8043
101
  var host = conf.host || null
102
  var autoAuth = conf.autoAuth !== false
103
104
  var server = http.createServer(exports.respond(sbot, config))
105
  var getAddress = onceify(function (cb) {
106
    server.on('error', cb)
107
    server.listen(port, host, function () {
108
      server.removeListener('error', cb)
109
      var regHost = formatHost(host || 'localhost')
110
      var regPort = this.address().port
111
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
112
      if (autoAuth) npmLogin(regUrl, next)
113
      else next()
114
      function next(err) {
115
        cb(err, regUrl)
116
      }
117
    })
118
    sbot.on('close', function () {
119
      server.close()
120
    })
121
  })
122
123
  getAddress(function (err, addr) {
124
    if (err) return console.error(err)
125
    console.log('[npm-registry] Listening on ' + addr)
126
  })
127
128
  return {
129
    getAddress: getAddress
130
  }
131
}
132
133
exports.respond = function (sbot, config) {
134
  var reg = new SsbNpmRegistryServer(sbot, config)
135
  return function (req, res) {
136
    new Req(reg, req, res).serve()
137
  }
138
}
139
140
function publishMsg(sbot, value, cb) {
141
  var gotExpectedPrevious = false
142
  sbot.publish(value, function next(err, msg) {
143
    if (err && /^expected previous:/.test(err.message)) {
144
      // retry once on this error
145
      if (gotExpectedPrevious) return cb(err)
146
      gotExpectedPrevious = true
147
      return sbot.publish(value, next)
148
    }
149
    cb(err, msg)
150
  })
151
}
152
153
function publishMentions(sbot, mentions, cb) {
154
  // console.error("publishing %s mentions", mentions.length)
155
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
156
  publishMsg(sbot, {
157
    type: 'npm-packages',
158
    mentions: mentions,
159
  }, cb)
160
}
161
162
exports.publishPkgMentions = function (sbot, mentions, cb) {
163
  // try to fit the mentions into as few messages as possible,
164
  // while fitting under the message size limit.
165
  var msgs = []
166
  ;(function next(i, chunks) {
167
    if (i >= mentions.length) return cb(null, msgs)
168
    var chunkLen = Math.ceil(mentions.length / chunks)
169
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
170
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
171
      if (err && msgs.length) return onPartialPublish(err)
172
      if (err) return cb(err)
173
      msgs.push(msg)
174
      next(i + chunkLen, chunks)
175
    })
176
  })(0, 1)
177
  function onPartialPublish(err) {
178
    var remaining = mentions.length - i
179
    return cb(new Error('Published messages ' +
180
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
181
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
182
  }
183
}
184
185
function SsbNpmRegistryServer(sbot, config) {
186
  this.sbot = sbot
187
  this.config = config
188
  this.npmConfig = config.npm || {}
189
  this.host = this.npmConfig.host || 'localhost'
190
  this.links2 = sbot.links2
191
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
192
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
193
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
194
    + this.wsPort + '/blobs/get/'
195
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
196
}
197
198
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
199
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
200
201
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
202
  var self = this
203
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
204
  ;(function next(i) {
205
    if (i >= ids.length) return cb()
206
    self.sbot.blobs.push(ids[i], function (err) {
207
      if (err) return cb(err)
208
      next(i+1)
209
    })
210
  })(0)
211
}
212
213
SsbNpmRegistryServer.prototype.blobDist = function (id) {
214
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
215
  if (!m) throw new Error('bad blob id: ' + id)
216
  return {
217
    integrity: m[2] + '-' + m[1],
218
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
219
  }
220
}
221
222
SsbNpmRegistryServer.prototype.getMentions = function (name) {
223
  return this.links2.read({
224
    query: [
225
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
226
      {$filter: {dest: {$prefix: '&'}}},
227
      {$map: {
228
        name: ['rel', 1],
229
        size: ['rel', 2],
230
        link: 'dest',
231
        author: 'source',
232
        ts: 'ts'
233
      }}
234
    ]
235
  })
236
}
237
238
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
239
  var self = this
240
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
241
  var ids = {}
242
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
243
  fs.readdir(prebuildsDir, function (err, filenames) {
244
    if (err) return cb(new Error(err.stack || err))
245
    ;(function next(i) {
246
      if (i >= filenames.length) return cb(null, ids)
247
      var m = nameRegex.exec(filenames[i])
248
      if (!m) return next(i+1)
249
      var name = m[1]
250
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
251
        if (err) return cb(new Error(err.stack || err))
252
        self.sbot.blobs.add(function (err, id) {
253
          if (err) return cb(new Error(err.stack || err))
254
          ids[name] = id
255
          next(i+1)
256
        })(pullOnce(data))
257
      })
258
    })(0)
259
  })
260
}
261
262
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
263
  var self = this
264
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
265
266
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
267
    if (err) return cb(new Error(err.stack || err))
268
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
269
    if (pkgs._hasNonBlobUrl) {
270
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
271
    }
272
273
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
274
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
275
276
    var waiting = 2
277
278
    self.sbot.blobs.add(function (err, id) {
279
      if (err) return next(new Error(err.stack || err))
280
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
281
      var versions = pkg.versions || (pkg.versions = {})
282
      pkg.versions[sbotPkgLock.version] = {
283
        dist: self.blobDist(id)
284
      }
285
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
286
      distTags.latest = sbotPkgLock.version
287
      next()
288
    })(self.sbot.bootstrap.pack())
289
290
    var prebuilds
291
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
292
      if (err) return next(err)
293
      prebuilds = _prebuilds
294
      next()
295
    })
296
297
    function next(err) {
298
      if (err) return waiting = 0, cb(err)
299
      if (--waiting) return
300
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
301
        encoding: 'utf8'
302
      }, function (err, bootstrapScript) {
303
        if (err) return cb(err)
304
        var script = bootstrapScript + '\n' +
305
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
306
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
307
308
        self.sbot.blobs.add(function (err, id) {
309
          if (err) return cb(new Error(err.stack || err))
310
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
311
          if (!m) return cb(new Error('bad blob id: ' + id))
312
          cb(null, {
313
            name: sbotPkgLock.name,
314
            blob: id,
315
            hashType: m[2],
316
            hashBuf: Buffer.from(m[1], 'base64'),
317
          })
318
        })(pullOnce(script))
319
      })
320
    }
321
  })
322
}
323
324
function Req(server, req, res) {
325
  this.server = server
326
  this.req = req
327
  this.res = res
328
  this.blobsToPush = []
329
}
330
331
Req.prototype.serve = function () {
332
  // console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
333
  var pathname = this.req.url.replace(/\?.*/, '')
334
  var m
335
  if (pathname === '/') return this.serveHome()
336
  if (pathname === '/bootstrap') return this.serveBootstrap()
337
  if (pathname === '/-/whoami') return this.serveWhoami()
338
  if (pathname === '/-/ping') return this.respond(200, true)
339
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
340
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
341
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
342
  return this.respond(404)
343
}
344
345
Req.prototype.respond = function (status, message) {
346
  this.res.writeHead(status, {'content-type': 'application/json'})
347
  this.res.end(message && JSON.stringify(message, 0, 2))
348
}
349
350
Req.prototype.respondError = function (status, message) {
351
  this.respond(status, {error: message})
352
}
353
354
var bootstrapName = 'ssb-npm-bootstrap'
355
356
Req.prototype.serveHome = function () {
357
  var self = this
358
  self.res.writeHead(200, {'content-type': 'text/html'})
359
  var port = 8044
360
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
361
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
362
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
363
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
364
    '</body></html>')
365
}
366
367
Req.prototype.serveBootstrap = function () {
368
  var self = this
369
  self.server.getBootstrapInfo(function (err, info) {
370
    if (err) return this.respondError(err.stack || err)
371
    var pkgNameText = info.name
372
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
373
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
374
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
375
    var blobsHostname = httpHost + ':' + self.server.wsPort
376
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
377
    var pkgHashText = info.hashBuf.toString('hex')
378
    var hashCmd = info.hashType + 'sum'
379
380
    var script =
381
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
382
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
383
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
384
        'npm install -g ' + info.name + ' &&\n' +
385
      'sbot server'
386
387
    self.res.writeHead(200, {'content-type': 'text/plain'})
388
    self.res.end(script)
389
  })
390
}
391
392
Req.prototype.serveWhoami = function () {
393
  var self = this
394
  self.server.sbot.whoami(function (err, feed) {
395
    if (err) return self.respondError(err.stack || err)
396
    self.respond(200, {username: feed.id})
397
  })
398
}
399
400
Req.prototype.serveUser1 = function () {
401
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
402
}
403
404
function decodeName(name) {
405
  var parts = name.replace(/\.tgz$/, '').split(':')
406
  return {
407
    name: parts[1],
408
    version: parts[2],
409
    distTag: parts[3],
410
  }
411
}
412
413
Req.prototype.servePkg = function (pathname) {
414
  var self = this
415
  var parts = pathname.split('/')
416
  var pkgName = parts.shift()
417
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
418
  if (parts.length > 0) return this.respondError(404)
419
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
420
  var obj = {
421
    _id: pkgName,
422
    name: pkgName,
423
    'dist-tags': {},
424
    versions: {}
425
  }
426
  var oldest, newest
427
  var getMention = self.server.getMentions({$prefix: 'npm:' + pkgName + ':'})
428
  getMention(null, function next(err, mention) {
429
    if (err === true) return self.respond(200, obj)
430
    if (err) return self.respondError(500, err.stack || err)
431
    var data = decodeName(mention.name)
432
    if (!data.version) return
433
    if (data.distTag) obj['dist-tags'][data.distTag] = data.version
434
    obj.versions[data.version] = {
435
      author: {
436
        url: mention.author
437
      },
438
      dist: self.server.blobDist(mention.link)
439
    }
440
    getMention(null, next)
441
  })
442
}
443
444
Req.prototype.servePrebuild = function (name) {
445
  var self = this
446
  var getMention = self.server.getMentions('prebuild:' + name)
447
  var blobsByAuthor = {/* <author>: BlobId */}
448
  getMention(null, function next(err, link) {
449
    if (err === true) return done()
450
    if (err) return self.respondError(500, err.stack || err)
451
    blobsByAuthor[link.author] = link.link
452
    getMention(null, next)
453
  })
454
  function done() {
455
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
456
    var blobId
457
    for (var feed in blobsByAuthor) {
458
      var blob = blobId = blobsByAuthor[feed]
459
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
460
      feeds.push(feed)
461
    }
462
    switch (Object.keys(authorsByLink).length) {
463
      case 0:
464
        return self.respondError(404, 'Not Found')
465
      case 1:
466
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
467
        return self.res.end()
468
      default:
469
        return self.respond(300, {choices: authorsByLink})
470
    }
471
  }
472
}
473
474
var localhosts = {
475
  '::1': true,
476
  '127.0.0.1': true,
477
  '::ffff:127.0.0.1': true,
478
}
479
480
Req.prototype.publishPkg = function (pkgName) {
481
  var self = this
482
  var remoteAddress = self.req.socket.remoteAddress
483
  if (!(remoteAddress in localhosts)) {
484
    return self.respondError(403, 'You may not publish as this user.')
485
  }
486
487
  var chunks = []
488
  self.req.on('data', function (data) {
489
    chunks.push(data)
490
  })
491
  self.req.on('end', function () {
492
    var data
493
    try {
494
      data = JSON.parse(Buffer.concat(chunks))
495
    } catch(e) {
496
      return self.respondError(400, e.stack)
497
    }
498
    return self.publishPkg2(pkgName, data || {})
499
  })
500
}
501
502
Req.prototype.publishPkg2 = function (name, data) {
503
  var self = this
504
  if (data.users) console.trace('[npm-registry] users property is not supported')
505
  var attachments = data._attachments || {}
506
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
507
  var waiting = 0
508
  Object.keys(attachments).forEach(function (filename) {
509
    waiting++
510
    var tarball = new Buffer(attachments[filename].data, 'base64')
511
    var length = attachments[filename].length
512
    if (length && length !== tarball.length) return self.respondError(400,
513
      'Length mismatch for attachment \'' + filename + '\'')
514
    self.server.sbot.blobs.add(function (err, id) {
515
      if (err) return self.respondError(500,
516
        'Adding attachment \'' + filename + '\' as blob failed')
517
      self.blobsToPush.push(id)
518
      links[filename] = {link: id, size: tarball.length}
519
      if (!--waiting) next()
520
    })(pullOnce(tarball))
521
  })
522
  function next() {
523
    try {
524
      self.publishPkg3(name, data, links)
525
    } catch(e) {
526
      self.respondError(500, e.stack || e)
527
    }
528
  }
529
}
530
531
Req.prototype.publishPkg3 = function (name, data, links) {
532
  var self = this
533
  var versions = data.versions || {}
534
  var linksByVersion = {/* <version>: link */}
535
536
  // associate tarball blobs with versions
537
  for (var version in versions) {
538
    var pkg = versions[version]
539
    if (!pkg) return self.respondError(400, 'Bad package object')
540
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
541
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
542
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
543
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
544
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
545
    var filename = m[1]
546
    var link = links[filename]
547
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
548
    // TODO?: try to find missing tarball mentioned in other messages
549
    if (pkg.version && pkg.version !== version)
550
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
551
    linksByVersion[version] = link
552
    link.version = version
553
  }
554
555
  // associate blobs with dist-tags
556
  var tags = data['dist-tags'] || {}
557
  for (var tag in tags) {
558
    var version = tags[tag]
559
    var link = linksByVersion[version]
560
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
561
    // TODO?: support setting dist-tag without version,
562
    // by looking up a tarball blob for the version
563
    link.tag = tag
564
  }
565
566
  // compute blob links to publish
567
  var mentions = []
568
  for (var filename in links) {
569
    var link = links[filename] || {}
570
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
571
    mentions.push({
572
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
573
      link: link.link,
574
      size: link.size,
575
    })
576
  }
577
  return self.publishPkgs(mentions)
578
}
579
580
Req.prototype.publishPkgs = function (mentions) {
581
  var self = this
582
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
583
    if (err) self.respondError(500, err.stack || err)
584
    self.server.pushBlobs(self.blobsToPush, function (err) {
585
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
586
      self.respond(201)
587
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
588
    })
589
  })
590
}
591