git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function onceify(fn, self) {
25
  var cbs = [], err, data
26
  return function (cb) {
27
    if (fn) {
28
      cbs.push(cb)
29
      fn.call(self, function (_err, _data) {
30
        err = _err, data = _data
31
        var _cbs = cbs
32
        cbs = null
33
        while (_cbs.length) _cbs.shift()(err, data)
34
      })
35
      fn = null
36
    } else if (cbs) {
37
      cbs.push(cb)
38
    } else {
39
      cb(err, data)
40
    }
41
  }
42
}
43
44
function once(cb) {
45
  var done
46
  return function (err, result) {
47
    if (done) {
48
      if (err) console.trace(err)
49
    } else {
50
      done = true
51
      cb(err, result)
52
    }
53
  }
54
}
55
56
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57
  // convert a package-lock.json file into data for serving as an npm registry
58
  var hasNonBlobUrl = false
59
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60
  var pkgs = {}
61
  var queue = [pkgLock, pkgLock.name]
62
  while (queue.length) {
63
    var dep = queue.shift(), name = queue.shift()
64
    if (name) {
65
      var pkg = pkgs[name] || (pkgs[name] = {
66
        _id: name,
67
        name: name,
68
        versions: {}
69
      })
70
      if (dep.version && dep.integrity && dep.resolved) {
71
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72
        pkg.versions[dep.version] = {
73
          name: name,
74
          version: dep.version,
75
          dist: {
76
            integrity: dep.integrity,
77
            tarball: dep.resolved
78
          }
79
        }
80
      }
81
    }
82
    if (dep.dependencies) for (var depName in dep.dependencies) {
83
      queue.push(dep.dependencies[depName], depName)
84
    }
85
  }
86
  pkgs._hasNonBlobUrl = hasNonBlobUrl
87
  return pkgs
88
}
89
90
function npmLogin(registryAddress, cb) {
91
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
92
  var filename = path.join(os.homedir(), '.npmrc')
93
  fs.readFile(filename, 'utf8', function (err, data) {
94
    if (err && err.code === 'ENOENT') data = ''
95
    else if (err) return cb(new Error(err.stack))
96
    var lines = data ? data.split('\n') : []
97
    if (lines.indexOf(tokenLine) > -1) return cb()
98
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
99
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
100
    fs.appendFile(filename, line, cb)
101
  })
102
}
103
104
function formatHost(host) {
105
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
106
}
107
108
exports.name = 'npm-registry'
109
exports.version = '1.0.0'
110
exports.manifest = {
111
  getAddress: 'async'
112
}
113
exports.init = function (sbot, config) {
114
  var conf = config.npm || {}
115
  var port = conf.port || 8043
116
  var host = conf.host || null
117
  var autoAuth = conf.autoAuth !== false
118
119
  var server = http.createServer(exports.respond(sbot, config))
120
  var getAddress = onceify(function (cb) {
121
    server.on('error', cb)
122
    server.listen(port, host, function () {
123
      server.removeListener('error', cb)
124
      var regHost = formatHost(host || 'localhost')
125
      var regPort = this.address().port
126
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
127
      if (autoAuth) npmLogin(regUrl, next)
128
      else next()
129
      function next(err) {
130
        cb(err, regUrl)
131
      }
132
    })
133
    sbot.on('close', function () {
134
      server.close()
135
    })
136
  })
137
138
  getAddress(function (err, addr) {
139
    if (err) return console.error(err)
140
    console.log('[npm-registry] Listening on ' + addr)
141
  })
142
143
  return {
144
    getAddress: getAddress
145
  }
146
}
147
148
exports.respond = function (sbot, config) {
149
  var reg = new SsbNpmRegistryServer(sbot, config)
150
  return function (req, res) {
151
    new Req(reg, req, res).serve()
152
  }
153
}
154
155
function publishMsg(sbot, value, cb) {
156
  var gotExpectedPrevious = false
157
  sbot.publish(value, function next(err, msg) {
158
    if (err && /^expected previous:/.test(err.message)) {
159
      // retry once on this error
160
      if (gotExpectedPrevious) return cb(err)
161
      gotExpectedPrevious = true
162
      return sbot.publish(value, next)
163
    }
164
    cb(err, msg)
165
  })
166
}
167
168
function getDependencyBranches(sbot, id, cb) {
169
  // get ids of heads of tree of dependencyBranch message links to include all
170
  // the dependencies of the given tarball id.
171
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
172
  var msgs = {}
173
  var branched = {}
174
  var blobs = {}
175
176
  function addPkgById(id, cb) {
177
    if (blobs[id]) return cb()
178
    blobs[id] = true
179
    getPackageJsonCached(id, function (err, pkg) {
180
      if (err) return cb(err)
181
      var done = multicb()
182
      for (var name in pkg.dependencies || {}) {
183
        addPkgBySpec(name, pkg.dependencies[name], done())
184
      }
185
      done(cb)
186
    })
187
  }
188
189
  function addPkgBySpec(name, spec, cb) {
190
    var done = multicb()
191
    pull(
192
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
193
      pull.map(function (mention) {
194
        addPkgById(mention.link, done())
195
        return packageLinks(sbot, mention.author, mention.link, name, spec)
196
      }),
197
      pull.flatten(),
198
      pull.drain(function (msg) {
199
        var c = msg && msg.value && msg.value.content
200
        if (!c) return
201
        msgs[msg.key] = msg.value
202
        if (Array.isArray(c.dependencyBranch)) {
203
          for (var k = 0; k < c.dependencyBranch.length; k++) {
204
            branched[c.dependencyBranch[k]] = true
205
          }
206
        }
207
      }, function (err) {
208
        if (err) return cb(err)
209
        done(cb)
210
      })
211
    )
212
  }
213
214
  addPkgById(id, function (err) {
215
    if (err) return cb(err)
216
    var ids = []
217
    for (var key in msgs) {
218
      if (!branched[key]) ids.push(key)
219
    }
220
    cb(null, ids)
221
  })
222
}
223
224
function packageLinks(sbot, feed, id, name, spec) {
225
  function matches(mention) {
226
    var data = mention
227
      && mention.link === id
228
      && mention.name
229
      && decodeName(mention.name)
230
    return data
231
      && data.name === name
232
      && (spec ? semver.satisfies(data.version, spec) : true)
233
  }
234
  return pull(
235
    sbot.links({
236
      source: feed,
237
      dest: id,
238
      rel: 'mentions',
239
      values: true,
240
    }),
241
    pull.filter(function (msg) {
242
      var c = msg && msg.value && msg.value.content
243
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
244
    })
245
  )
246
}
247
248
function getVersionBranches(sbot, link, cb) {
249
  var data = decodeName(link.name)
250
  var msgs = {}, branched = {}
251
  pull(
252
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
253
    pull.map(function (mention) {
254
      return packageLinks(sbot, mention.author, mention.link, data.name)
255
    }),
256
    pull.flatten(),
257
    pull.drain(function (msg) {
258
      var c = msg && msg.value && msg.value.content
259
      if (!c) return
260
      msgs[msg.key] = msg.value
261
      if (Array.isArray(c.versionBranch)) {
262
        for (var k = 0; k < c.versionBranch.length; k++) {
263
          branched[c.versionBranch[k]] = true
264
        }
265
      }
266
    }, function (err) {
267
      if (err) return cb(err)
268
      var ids = []
269
      for (var key in msgs) {
270
        if (!branched[key]) ids.push(key)
271
      }
272
      cb(null, ids)
273
    })
274
  )
275
}
276
277
// For each dependency that is not a bundledDependency, get message ids for
278
// that dependency name + version.
279
function publishSingleMention(sbot, mention, cb) {
280
  // Calculate dependencyBranch and versionBranch message ids.
281
  var value = {
282
    type: 'npm-packages',
283
    mentions: [mention]
284
  }
285
  var done = multicb({pluck: 1, spread: true})
286
  getDependencyBranches(sbot, mention.link, done())
287
  getVersionBranches(sbot, mention, done())
288
  done(function (err, dependencyBranches, versionBranches) {
289
    if (err) return cb(err)
290
    value.dependencyBranch = dependencyBranches || undefined
291
    value.versionBranch = versionBranches || undefined
292
    publishMsg(sbot, value, cb)
293
  })
294
}
295
296
function publishMentions(sbot, mentions, cb) {
297
  // console.error("publishing %s mentions", mentions.length)
298
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
299
  // if it is just one mention, fetch and add useful metadata
300
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
301
  publishMsg(sbot, {
302
    type: 'npm-packages',
303
    mentions: mentions,
304
  }, cb)
305
}
306
307
exports.publishPkgMentions = function (sbot, mentions, cb) {
308
  // try to fit the mentions into as few messages as possible,
309
  // while fitting under the message size limit.
310
  var msgs = []
311
  ;(function next(i, chunks) {
312
    if (i >= mentions.length) return cb(null, msgs)
313
    var chunkLen = Math.ceil(mentions.length / chunks)
314
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
315
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
316
      if (err && msgs.length) return onPartialPublish(err)
317
      if (err) return cb(err)
318
      msgs.push(msg)
319
      next(i + chunkLen, chunks)
320
    })
321
  })(0, 1)
322
  function onPartialPublish(err) {
323
    var remaining = mentions.length - i
324
    return cb(new Error('Published messages ' +
325
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
326
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
327
  }
328
}
329
330
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
331
  cb = once(cb)
332
  var waiting = 0
333
  var expandedMentions = mentions.map(function (link) {
334
    var id = link && link.link
335
    if (!id) return link
336
    waiting++
337
    var newLink = {}
338
    for (var k in link) newLink[k] = link[k]
339
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
340
      if (err) return cb(err)
341
      for (var k in props) newLink[k] = pkg[k]
342
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
343
      if (!--waiting) next()
344
    })
345
    return newLink
346
  })
347
  if (!waiting) next()
348
  function next() {
349
    cb(null, expandedMentions)
350
  }
351
}
352
353
function SsbNpmRegistryServer(sbot, config) {
354
  this.sbot = sbot
355
  this.config = config
356
  this.npmConfig = config.npm || {}
357
  this.host = this.npmConfig.host || 'localhost'
358
  this.fetchAll = this.npmConfig.fetchAll
359
  this.needShasum = this.npmConfig.needShasum
360
  this.links2 = sbot.links2
361
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
362
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
363
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
364
    + this.wsPort + '/blobs/get/'
365
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
366
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
367
}
368
369
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
370
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
371
372
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
373
  var self = this
374
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
375
  ;(function next(i) {
376
    if (i >= ids.length) return cb()
377
    self.sbot.blobs.push(ids[i], function (err) {
378
      if (err) return cb(err)
379
      next(i+1)
380
    })
381
  })(0)
382
}
383
384
function getBlob(sbot, id, cb) {
385
  var blobs = sbot.blobs
386
  blobs.size(id, function (err, size) {
387
    if (typeof size === 'number') cb(null, blobs.get(id))
388
    else blobs.want(id, function (err, got) {
389
      if (err) cb(err)
390
      else if (!got) cb('missing blob ' + id)
391
      else cb(null, blobs.get(id))
392
    })
393
  })
394
}
395
396
SsbNpmRegistryServer.prototype.blobDist = function (id) {
397
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
398
  if (!m) throw new Error('bad blob id: ' + id)
399
  return {
400
    integrity: m[2] + '-' + m[1],
401
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
402
  }
403
}
404
405
function getMentions(links2, name) {
406
  return links2.read({
407
    query: [
408
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
409
      {$filter: {dest: {$prefix: '&'}}},
410
      {$map: {
411
        name: ['rel', 1],
412
        size: ['rel', 2],
413
        link: 'dest',
414
        author: 'source',
415
        ts: 'ts'
416
      }}
417
    ]
418
  })
419
}
420
421
SsbNpmRegistryServer.prototype.getMentions = function (name) {
422
  return getMentions(this.links2, name)
423
}
424
425
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
426
  var self = this
427
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
428
  var ids = {}
429
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
430
  fs.readdir(prebuildsDir, function (err, filenames) {
431
    if (err) return cb(new Error(err.stack || err))
432
    ;(function next(i) {
433
      if (i >= filenames.length) return cb(null, ids)
434
      var m = nameRegex.exec(filenames[i])
435
      if (!m) return next(i+1)
436
      var name = m[1]
437
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
438
        if (err) return cb(new Error(err.stack || err))
439
        self.sbot.blobs.add(function (err, id) {
440
          if (err) return cb(new Error(err.stack || err))
441
          ids[name] = id
442
          next(i+1)
443
        })(pull.once(data))
444
      })
445
    })(0)
446
  })
447
}
448
449
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
450
  var self = this
451
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
452
453
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
454
    if (err) return cb(new Error(err.stack || err))
455
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
456
    if (pkgs._hasNonBlobUrl) {
457
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
458
    }
459
460
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
461
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
462
463
    var waiting = 2
464
465
    self.sbot.blobs.add(function (err, id) {
466
      if (err) return next(new Error(err.stack || err))
467
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
468
      var versions = pkg.versions || (pkg.versions = {})
469
      pkg.versions[sbotPkgLock.version] = {
470
        name: sbotPkgLock.name,
471
        version: sbotPkgLock.version,
472
        dist: self.blobDist(id)
473
      }
474
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
475
      distTags.latest = sbotPkgLock.version
476
      next()
477
    })(self.sbot.bootstrap.pack())
478
479
    var prebuilds
480
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
481
      if (err) return next(err)
482
      prebuilds = _prebuilds
483
      next()
484
    })
485
486
    function next(err) {
487
      if (err) return waiting = 0, cb(err)
488
      if (--waiting) return
489
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
490
        encoding: 'utf8'
491
      }, function (err, bootstrapScript) {
492
        if (err) return cb(err)
493
        var script = bootstrapScript + '\n' +
494
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
495
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
496
497
        self.sbot.blobs.add(function (err, id) {
498
          if (err) return cb(new Error(err.stack || err))
499
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
500
          if (!m) return cb(new Error('bad blob id: ' + id))
501
          cb(null, {
502
            name: sbotPkgLock.name,
503
            blob: id,
504
            hashType: m[2],
505
            hashBuf: Buffer.from(m[1], 'base64'),
506
          })
507
        })(pull.once(script))
508
      })
509
    }
510
  })
511
}
512
513
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
514
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
515
  else this.sbot.get(id, function (err, value) {
516
    if (err) return cb(err)
517
    cb(null, {key: id, value: value})
518
  })
519
}
520
521
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
522
  var self = this
523
  var stack = heads.slice()
524
  var seen = {}
525
  return function (abort, cb) {
526
    if (abort) return cb(abort)
527
    for (var id; stack.length && seen[id = stack.pop()];);
528
    if (!id) return cb(true)
529
    seen[id] = true
530
    // TODO: use DFS
531
    self.getMsg(id, function (err, msg) {
532
      if (err) return cb(new Error(err.stack || err))
533
      var c = msg && msg.value && msg.value.content
534
      var links = c && c[prop]
535
      if (Array.isArray(links)) {
536
        stack.push.apply(stack, links)
537
      } else if (links) {
538
        stack.push(links)
539
      }
540
      cb(null, msg)
541
    })
542
  }
543
}
544
545
function Req(server, req, res) {
546
  this.server = server
547
  this.req = req
548
  this.res = res
549
  this.blobsToPush = []
550
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
551
  var ua = this.req.headers['user-agent']
552
  var m = /\bnpm\/([0-9]*)/.exec(ua)
553
  var npmVersion = m && m[1]
554
  this.needShasum = server.needShasum != null ? server.needShasum :
555
    (npmVersion && npmVersion < 5)
556
}
557
558
Req.prototype.serve = function () {
559
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
560
  this.res.setTimeout(0)
561
  var pathname = this.req.url.replace(/\?.*/, '')
562
  var m
563
  if ((m = /^\/(%25.*sha256)(\/.*)$/.exec(pathname))) {
564
    try {
565
      this.headMsgId = decodeURIComponent(m[1])
566
    } catch(e) {
567
      return this.respondError(400, e.stack || e)
568
    }
569
    pathname = m[2]
570
  }
571
  if (pathname === '/') return this.serveHome()
572
  if (pathname === '/bootstrap') return this.serveBootstrap()
573
  if (pathname === '/-/whoami') return this.serveWhoami()
574
  if (pathname === '/-/ping') return this.respond(200, true)
575
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
576
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
577
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
578
  return this.respond(404)
579
}
580
581
Req.prototype.respond = function (status, message) {
582
  this.res.writeHead(status, {'content-type': 'application/json'})
583
  this.res.end(message && JSON.stringify(message, 0, 2))
584
}
585
586
Req.prototype.respondError = function (status, message) {
587
  this.respond(status, {error: message})
588
}
589
590
var bootstrapName = 'ssb-npm-bootstrap'
591
592
Req.prototype.serveHome = function () {
593
  var self = this
594
  self.res.writeHead(200, {'content-type': 'text/html'})
595
  var port = 8044
596
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
597
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
598
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
599
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
600
    '</body></html>')
601
}
602
603
Req.prototype.serveBootstrap = function () {
604
  var self = this
605
  self.server.getBootstrapInfo(function (err, info) {
606
    if (err) return self.respondError(err.stack || err)
607
    var pkgNameText = info.name
608
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
609
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
610
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
611
    var blobsHostname = httpHost + ':' + self.server.wsPort
612
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
613
    var pkgHashText = info.hashBuf.toString('hex')
614
    var hashCmd = info.hashType + 'sum'
615
616
    var script =
617
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
618
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
619
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
620
        'npm install -g ' + info.name + ' &&\n' +
621
      'sbot server'
622
623
    self.res.writeHead(200, {'content-type': 'text/plain'})
624
    self.res.end(script)
625
  })
626
}
627
628
Req.prototype.serveWhoami = function () {
629
  var self = this
630
  self.server.sbot.whoami(function (err, feed) {
631
    if (err) return self.respondError(err.stack || err)
632
    self.respond(200, {username: feed.id})
633
  })
634
}
635
636
Req.prototype.serveUser1 = function () {
637
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
638
}
639
640
function decodeName(name) {
641
  var parts = String(name).replace(/\.tgz$/, '').split(':')
642
  return {
643
    name: parts[1],
644
    version: parts[2],
645
    distTag: parts[3],
646
  }
647
}
648
649
Req.prototype.getMentions = function (name) {
650
  var serverMentions = this.server.getMentions(name)
651
  var msgMentions = this.headMsgId ? pull(
652
    this.server.streamTree([this.headMsgId], 'dependencyBranch'),
653
    // decryption could be done here
654
    pull.map(function (msg) {
655
      var c = msg.value && msg.value.content
656
      if (!c.mentions || !Array.isArray(c.mentions)) return []
657
      return c.mentions.map(function (mention) {
658
        return {
659
          name: mention.name,
660
          size: mention.size,
661
          link: mention.link,
662
          author: msg.value.author,
663
          ts: msg.value.timestamp,
664
        }
665
      })
666
    }),
667
    pull.flatten(),
668
    pull.filter(typeof name === 'string' ? function (link) {
669
      return link.name === name
670
    } : name && name.$prefix ? function (link) {
671
      return link.name.substr(0, name.$prefix.length) === name.$prefix
672
    } : function () {
673
      throw new TypeError('unsupported name filter')
674
    }),
675
  ) : pull.empty()
676
  return cat([msgMentions, serverMentions])
677
}
678
679
Req.prototype.getMentionLinks = function (blobId) {
680
  return pull(
681
    this.headMsgId
682
      ? this.server.streamTree([this.headMsgId], 'dependencyBranch')
683
      : this.server.sbot.links({
684
        dest: blobId,
685
        rel: 'mentions',
686
        values: true,
687
      }),
688
    // decryption could be done here
689
    pull.map(function (msg) {
690
      var c = msg.value && msg.value.content
691
      return c && Array.isArray(c.mentions) || []
692
    }),
693
    pull.flatten(),
694
    pull.filter(function (link) {
695
      return link && link.link === blobId
696
    })
697
  )
698
}
699
700
Req.prototype.servePkg = function (pathname) {
701
  var self = this
702
  var parts = pathname.split('/')
703
  var pkgName = parts.shift().replace(/%2f/i, '/')
704
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
705
  var spec = parts.shift()
706
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
707
  if (parts.length > 0) return this.respondError(404)
708
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
709
  var obj = {
710
    _id: pkgName,
711
    name: pkgName,
712
    'dist-tags': {},
713
    versions: {}
714
  }
715
  var distTags = {/* <tag>: {version, ts}*/}
716
  pull(
717
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
718
    pull.drain(function (mention) {
719
      var data = decodeName(mention.name)
720
      if (!data.version) return
721
      if (data.distTag) {
722
        var tag = distTags[data.distTag]
723
        if (!tag || mention.ts > tag.ts) {
724
          /* TODO: sort by causal order (versionBranch links) instead of just
725
          * by timestamps */
726
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
727
        }
728
      }
729
      obj.versions[data.version] = {
730
        author: {
731
          url: mention.author
732
        },
733
        name: pkgName,
734
        version: data.version,
735
        dist: self.server.blobDist(mention.link)
736
      }
737
    }, function (err) {
738
      if (err) return self.respondError(500, err.stack || err)
739
      for (var tag in distTags) {
740
        obj['dist-tags'][tag] = distTags[tag].version
741
      }
742
      if (spec) resolveSpec()
743
      else if (self.fetchAll) resolveAll()
744
      else done()
745
    })
746
  )
747
  function resolveSpec() {
748
    var version = obj['dist-tags'][spec]
749
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
750
    obj = obj.versions[version]
751
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
752
    self.populatePackageJson(obj, function (err, pkg) {
753
      if (err) return self.respondError(500, err.stack || err)
754
      obj = pkg || obj
755
      done()
756
    })
757
  }
758
  function resolveVersion(version, cb) {
759
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
760
      if (err) return cb(err)
761
      if (pkg) obj.versions[version] = pkg
762
      cb()
763
    })
764
  }
765
  function resolveAll() {
766
    var done = multicb()
767
    for (var version in obj.versions) {
768
      resolveVersion(version, done())
769
    }
770
    done(resolved)
771
  }
772
  function resolved(err) {
773
    if (err) return self.respondError(500, err.stack || err)
774
    self.respond(200, obj)
775
  }
776
}
777
778
Req.prototype.servePrebuild = function (name) {
779
  var self = this
780
  var getMention = self.getMentions('prebuild:' + name)
781
  var blobsByAuthor = {/* <author>: BlobId */}
782
  getMention(null, function next(err, link) {
783
    if (err === true) return done()
784
    if (err) return self.respondError(500, err.stack || err)
785
    blobsByAuthor[link.author] = link.link
786
    getMention(null, next)
787
  })
788
  function done() {
789
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
790
    var blobId
791
    for (var feed in blobsByAuthor) {
792
      var blob = blobId = blobsByAuthor[feed]
793
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
794
      feeds.push(feed)
795
    }
796
    switch (Object.keys(authorsByLink).length) {
797
      case 0:
798
        return self.respondError(404, 'Not Found')
799
      case 1:
800
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
801
        return self.res.end()
802
      default:
803
        return self.respond(300, {choices: authorsByLink})
804
    }
805
  }
806
}
807
808
var localhosts = {
809
  '::1': true,
810
  '127.0.0.1': true,
811
  '::ffff:127.0.0.1': true,
812
}
813
814
Req.prototype.publishPkg = function (pkgName) {
815
  var self = this
816
  var remoteAddress = self.req.socket.remoteAddress
817
  if (!(remoteAddress in localhosts)) {
818
    return self.respondError(403, 'You may not publish as this user.')
819
  }
820
821
  var chunks = []
822
  self.req.on('data', function (data) {
823
    chunks.push(data)
824
  })
825
  self.req.on('end', function () {
826
    var data
827
    try {
828
      data = JSON.parse(Buffer.concat(chunks))
829
    } catch(e) {
830
      return self.respondError(400, e.stack)
831
    }
832
    return self.publishPkg2(pkgName, data || {})
833
  })
834
}
835
836
Req.prototype.publishPkg2 = function (name, data) {
837
  var self = this
838
  if (data.users) console.trace('[npm-registry] users property is not supported')
839
  var attachments = data._attachments || {}
840
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
841
  var done = multicb()
842
  function addAttachmentAsBlob(filename, cb) {
843
    var data = attachments[filename].data
844
    var tarball = new Buffer(data, 'base64')
845
    var length = attachments[filename].length
846
    if (length && length !== tarball.length) return self.respondError(400,
847
      'Length mismatch for attachment \'' + filename + '\'')
848
    self.server.sbot.blobs.add(function (err, id) {
849
      if (err) return cb(err)
850
      self.blobsToPush.push(id)
851
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
852
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
853
      cb()
854
    })(pull.once(tarball))
855
  }
856
  for (var filename in attachments) {
857
    addAttachmentAsBlob(filename, done())
858
  }
859
  done(function (err) {
860
    if (err) return self.respondError(500, err.stack || err)
861
    try {
862
      self.publishPkg3(name, data, links)
863
    } catch(e) {
864
      self.respondError(500, e.stack || e)
865
    }
866
  })
867
}
868
869
Req.prototype.publishPkg3 = function (name, data, links) {
870
  var self = this
871
  var versions = data.versions || {}
872
  var linksByVersion = {/* <version>: link */}
873
874
  // associate tarball blobs with versions
875
  for (var version in versions) {
876
    var pkg = versions[version]
877
    if (!pkg) return self.respondError(400, 'Bad package object')
878
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
879
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
880
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
881
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
882
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
883
    var filename = m[1]
884
    var link = links[filename]
885
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
886
    // TODO?: try to find missing tarball mentioned in other messages
887
    if (pkg.version && pkg.version !== version)
888
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
889
    linksByVersion[version] = link
890
    link.version = version
891
    link.dependencies = pkg.dependencies || {}
892
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
893
  }
894
895
  // associate blobs with dist-tags
896
  var tags = data['dist-tags'] || {}
897
  for (var tag in tags) {
898
    var version = tags[tag]
899
    var link = linksByVersion[version]
900
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
901
    // TODO?: support setting dist-tag without version,
902
    // by looking up a tarball blob for the version
903
    link.tag = tag
904
  }
905
906
  // compute blob links to publish
907
  var mentions = []
908
  for (var filename in links) {
909
    var link = links[filename] || {}
910
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
911
    mentions.push({
912
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
913
      link: link.link,
914
      size: link.size,
915
      shasum: link.shasum,
916
      dependencies: link.dependencies,
917
      bundledDependencies: link.bundledDependencies,
918
    })
919
  }
920
  return self.publishPkgs(mentions)
921
}
922
923
Req.prototype.publishPkgs = function (mentions) {
924
  var self = this
925
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
926
    if (err) return self.respondError(500, err.stack || err)
927
    self.server.pushBlobs(self.blobsToPush, function (err) {
928
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
929
      self.respond(201)
930
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
931
    })
932
  })
933
}
934
935
Req.prototype.populatePackageJson = function (obj, cb) {
936
  var self = this
937
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
938
  var deps, bundledDeps, shasum
939
940
  // look for dependencies in links.
941
  // then fallback to getting it from the tarball blob
942
943
  pull(
944
    self.getMentionLinks(blobId),
945
    pull.drain(function (link) {
946
      if (link.dependencies) deps = link.dependencies
947
      if (link.shasum) shasum = link.shasum
948
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
949
      // how to handle multiple assignments of dependencies to a package?
950
    }, function (err) {
951
      if (err) return cb(new Error(err.stack || err))
952
      if (deps && (shasum || !self.needShasum)) {
953
        // assume that the dependencies in the links to the blob are
954
        // correct.
955
        obj.dependencies = deps
956
        obj.bundledDependencies = bundledDeps
957
        if (shasum) obj.dist.shasum = obj._shasum = shasum
958
        cb(null, obj)
959
      } else {
960
        // get dependencies from the tarball
961
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
962
          if (err) return cb(err)
963
          pkg.dist = obj.dist
964
          pkg.dist.shasum = pkg._shasum
965
          pkg.author = pkg.author || obj.author
966
          pkg.version = pkg.version || obj.version
967
          pkg.name = pkg.name || obj.name
968
          cb(null, pkg)
969
        })
970
      }
971
    })
972
  )
973
}
974
975
function getPackageJsonFromTarballBlob(sbot, id, cb) {
976
  var self = this
977
  getBlob(sbot, id, function (err, readBlob) {
978
    if (err) return cb(err)
979
    cb = once(cb)
980
    var extract = tar.extract()
981
    var pkg, shasum
982
    extract.on('entry', function (header, stream, next) {
983
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
984
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
985
          if (err) return cb(err)
986
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
987
          catch(e) { return cb(e) }
988
          next()
989
        }))
990
      } else {
991
        stream.on('end', next)
992
        stream.resume()
993
      }
994
    })
995
    extract.on('finish', function () {
996
      pkg._shasum = shasum
997
      cb(null, pkg)
998
    })
999
    pull(
1000
      readBlob,
1001
      hash('sha1', 'hex', function (err, sum) {
1002
        if (err) return cb(err)
1003
        shasum = sum
1004
      }),
1005
      toPull(zlib.createGunzip()),
1006
      toPull(extract)
1007
    )
1008
  })
1009
}
1010