git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function onceify(fn, self) {
25
  var cbs = [], err, data
26
  return function (cb) {
27
    if (fn) {
28
      cbs.push(cb)
29
      fn.call(self, function (_err, _data) {
30
        err = _err, data = _data
31
        var _cbs = cbs
32
        cbs = null
33
        while (_cbs.length) _cbs.shift()(err, data)
34
      })
35
      fn = null
36
    } else if (cbs) {
37
      cbs.push(cb)
38
    } else {
39
      cb(err, data)
40
    }
41
  }
42
}
43
44
function once(cb) {
45
  var done
46
  return function (err, result) {
47
    if (done) {
48
      if (err) console.trace(err)
49
    } else {
50
      done = true
51
      cb(err, result)
52
    }
53
  }
54
}
55
56
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57
  // convert a package-lock.json file into data for serving as an npm registry
58
  var hasNonBlobUrl = false
59
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60
  var pkgs = {}
61
  var queue = [pkgLock, pkgLock.name]
62
  while (queue.length) {
63
    var dep = queue.shift(), name = queue.shift()
64
    if (name) {
65
      var pkg = pkgs[name] || (pkgs[name] = {
66
        _id: name,
67
        name: name,
68
        versions: {}
69
      })
70
      if (dep.version && dep.integrity && dep.resolved) {
71
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72
        pkg.versions[dep.version] = {
73
          name: name,
74
          version: dep.version,
75
          dist: {
76
            integrity: dep.integrity,
77
            tarball: dep.resolved
78
          }
79
        }
80
      }
81
    }
82
    if (dep.dependencies) for (var depName in dep.dependencies) {
83
      queue.push(dep.dependencies[depName], depName)
84
    }
85
  }
86
  pkgs._hasNonBlobUrl = hasNonBlobUrl
87
  return pkgs
88
}
89
90
function npmLogin(registryAddress, cb) {
91
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
92
  var filename = path.join(os.homedir(), '.npmrc')
93
  fs.readFile(filename, 'utf8', function (err, data) {
94
    if (err && err.code === 'ENOENT') data = ''
95
    else if (err) return cb(new Error(err.stack))
96
    var lines = data ? data.split('\n') : []
97
    if (lines.indexOf(tokenLine) > -1) return cb()
98
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
99
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
100
    fs.appendFile(filename, line, cb)
101
  })
102
}
103
104
function formatHost(host) {
105
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
106
}
107
108
exports.name = 'npm-registry'
109
exports.version = '1.0.0'
110
exports.manifest = {
111
  getAddress: 'async'
112
}
113
exports.init = function (sbot, config) {
114
  var conf = config.npm || {}
115
  var port = conf.port || 8043
116
  var host = conf.host || null
117
  var autoAuth = conf.autoAuth !== false
118
119
  var server = http.createServer(exports.respond(sbot, config))
120
  var getAddress = onceify(function (cb) {
121
    server.on('error', cb)
122
    server.listen(port, host, function () {
123
      server.removeListener('error', cb)
124
      var regHost = formatHost(host || 'localhost')
125
      var regPort = this.address().port
126
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
127
      if (autoAuth) npmLogin(regUrl, next)
128
      else next()
129
      function next(err) {
130
        cb(err, regUrl)
131
      }
132
    })
133
    sbot.on('close', function () {
134
      server.close()
135
    })
136
  })
137
138
  getAddress(function (err, addr) {
139
    if (err) return console.error(err)
140
    console.log('[npm-registry] Listening on ' + addr)
141
  })
142
143
  return {
144
    getAddress: getAddress
145
  }
146
}
147
148
exports.respond = function (sbot, config) {
149
  var reg = new SsbNpmRegistryServer(sbot, config)
150
  return function (req, res) {
151
    new Req(reg, req, res).serve()
152
  }
153
}
154
155
function publishMsg(sbot, value, cb) {
156
  var gotExpectedPrevious = false
157
  sbot.publish(value, function next(err, msg) {
158
    if (err && /^expected previous:/.test(err.message)) {
159
      // retry once on this error
160
      if (gotExpectedPrevious) return cb(err)
161
      gotExpectedPrevious = true
162
      return sbot.publish(value, next)
163
    }
164
    cb(err, msg)
165
  })
166
}
167
168
function getDependencyBranches(sbot, id, cb) {
169
  // get ids of heads of tree of dependencyBranch message links to include all
170
  // the dependencies of the given tarball id.
171
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
172
  var msgs = {}
173
  var branched = {}
174
  var blobs = {}
175
176
  function addPkgById(id, cb) {
177
    if (blobs[id]) return cb()
178
    blobs[id] = true
179
    getPackageJsonCached(id, function (err, pkg) {
180
      if (err) return cb(err)
181
      var done = multicb()
182
      for (var name in pkg.dependencies || {}) {
183
        addPkgBySpec(name, pkg.dependencies[name], done())
184
      }
185
      done(cb)
186
    })
187
  }
188
189
  function addPkgBySpec(name, spec, cb) {
190
    var done = multicb()
191
    pull(
192
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
193
      pull.map(function (mention) {
194
        addPkgById(mention.link, done())
195
        return packageLinks(sbot, mention.author, mention.link, name, spec)
196
      }),
197
      pull.flatten(),
198
      pull.drain(function (msg) {
199
        var c = msg && msg.value && msg.value.content
200
        if (!c) return
201
        msgs[msg.key] = msg.value
202
        if (Array.isArray(c.dependencyBranch)) {
203
          for (var k = 0; k < c.dependencyBranch.length; k++) {
204
            branched[c.dependencyBranch[k]] = true
205
          }
206
        }
207
      }, function (err) {
208
        if (err) return cb(err)
209
        done(cb)
210
      })
211
    )
212
  }
213
214
  addPkgById(id, function (err) {
215
    if (err) return cb(err)
216
    var ids = []
217
    for (var key in msgs) {
218
      if (!branched[key]) ids.push(key)
219
    }
220
    cb(null, ids)
221
  })
222
}
223
224
function packageLinks(sbot, feed, id, name, spec) {
225
  function matches(mention) {
226
    var data = mention
227
      && mention.link === id
228
      && decodeName(mention.name)
229
    return data
230
      && data.name === name
231
      && (spec ? semver.satisfies(data.version, spec) : true)
232
  }
233
  return pull(
234
    sbot.links({
235
      source: feed,
236
      dest: id,
237
      rel: 'mentions',
238
      values: true,
239
    }),
240
    pull.filter(function (msg) {
241
      var c = msg && msg.value && msg.value.content
242
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
243
    })
244
  )
245
}
246
247
function getVersionBranches(sbot, link, cb) {
248
  var data = decodeName(link.name)
249
  var msgs = {}, branched = {}
250
  pull(
251
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
252
    pull.map(function (mention) {
253
      return packageLinks(sbot, mention.author, mention.link, data.name)
254
    }),
255
    pull.flatten(),
256
    pull.drain(function (msg) {
257
      var c = msg && msg.value && msg.value.content
258
      if (!c) return
259
      msgs[msg.key] = msg.value
260
      if (Array.isArray(c.versionBranch)) {
261
        for (var k = 0; k < c.versionBranch.length; k++) {
262
          branched[c.versionBranch[k]] = true
263
        }
264
      }
265
    }, function (err) {
266
      if (err) return cb(err)
267
      var ids = []
268
      for (var key in msgs) {
269
        if (!branched[key]) ids.push(key)
270
      }
271
      cb(null, ids)
272
    })
273
  )
274
}
275
276
// For each dependency that is not a bundledDependency, get message ids for
277
// that dependency name + version.
278
function publishSingleMention(sbot, mention, cb) {
279
  // Calculate dependencyBranch and versionBranch message ids.
280
  var value = {
281
    type: 'npm-packages',
282
    mentions: [mention]
283
  }
284
  var done = multicb({pluck: 1, spread: true})
285
  getDependencyBranches(sbot, mention.link, done())
286
  getVersionBranches(sbot, mention, done())
287
  done(function (err, dependencyBranches, versionBranches) {
288
    if (err) return cb(err)
289
    value.dependencyBranch = dependencyBranches || undefined
290
    value.versionBranch = versionBranches || undefined
291
    publishMsg(sbot, value, cb)
292
  })
293
}
294
295
function publishMentions(sbot, mentions, cb) {
296
  // console.error("publishing %s mentions", mentions.length)
297
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
298
  // if it is just one mention, fetch and add useful metadata
299
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
300
  publishMsg(sbot, {
301
    type: 'npm-packages',
302
    mentions: mentions,
303
  }, cb)
304
}
305
306
exports.publishPkgMentions = function (sbot, mentions, cb) {
307
  // try to fit the mentions into as few messages as possible,
308
  // while fitting under the message size limit.
309
  var msgs = []
310
  ;(function next(i, chunks) {
311
    if (i >= mentions.length) return cb(null, msgs)
312
    var chunkLen = Math.ceil(mentions.length / chunks)
313
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
314
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
315
      if (err && msgs.length) return onPartialPublish(err)
316
      if (err) return cb(err)
317
      msgs.push(msg)
318
      next(i + chunkLen, chunks)
319
    })
320
  })(0, 1)
321
  function onPartialPublish(err) {
322
    var remaining = mentions.length - i
323
    return cb(new Error('Published messages ' +
324
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
325
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
326
  }
327
}
328
329
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
330
  cb = once(cb)
331
  var waiting = 0
332
  var expandedMentions = mentions.map(function (link) {
333
    var id = link && link.link
334
    if (!id) return link
335
    waiting++
336
    var newLink = {}
337
    for (var k in link) newLink[k] = link[k]
338
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
339
      if (err) return cb(err)
340
      for (var k in props) newLink[k] = pkg[k]
341
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
342
      if (!--waiting) next()
343
    })
344
    return newLink
345
  })
346
  if (!waiting) next()
347
  function next() {
348
    cb(null, expandedMentions)
349
  }
350
}
351
352
function SsbNpmRegistryServer(sbot, config) {
353
  this.sbot = sbot
354
  this.config = config
355
  this.npmConfig = config.npm || {}
356
  this.host = this.npmConfig.host || 'localhost'
357
  this.fetchAll = this.npmConfig.fetchAll
358
  this.needShasum = this.npmConfig.needShasum
359
  this.links2 = sbot.links2
360
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
361
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
362
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
363
    + this.wsPort + '/blobs/get/'
364
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
365
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
366
}
367
368
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
369
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
370
371
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
372
  var self = this
373
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
374
  ;(function next(i) {
375
    if (i >= ids.length) return cb()
376
    self.sbot.blobs.push(ids[i], function (err) {
377
      if (err) return cb(err)
378
      next(i+1)
379
    })
380
  })(0)
381
}
382
383
function getBlob(sbot, id, cb) {
384
  var blobs = sbot.blobs
385
  blobs.size(id, function (err, size) {
386
    if (typeof size === 'number') cb(null, blobs.get(id))
387
    else blobs.want(id, function (err, got) {
388
      if (err) cb(err)
389
      else if (!got) cb('missing blob ' + id)
390
      else cb(null, blobs.get(id))
391
    })
392
  })
393
}
394
395
SsbNpmRegistryServer.prototype.blobDist = function (id) {
396
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
397
  if (!m) throw new Error('bad blob id: ' + id)
398
  return {
399
    integrity: m[2] + '-' + m[1],
400
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
401
  }
402
}
403
404
function getMentions(links2, name) {
405
  return links2.read({
406
    query: [
407
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
408
      {$filter: {dest: {$prefix: '&'}}},
409
      {$map: {
410
        name: ['rel', 1],
411
        size: ['rel', 2],
412
        link: 'dest',
413
        author: 'source',
414
        ts: 'ts'
415
      }}
416
    ]
417
  })
418
}
419
420
SsbNpmRegistryServer.prototype.getMentions = function (name) {
421
  return getMentions(this.links2, name)
422
}
423
424
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
425
  var self = this
426
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
427
  var ids = {}
428
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
429
  fs.readdir(prebuildsDir, function (err, filenames) {
430
    if (err) return cb(new Error(err.stack || err))
431
    ;(function next(i) {
432
      if (i >= filenames.length) return cb(null, ids)
433
      var m = nameRegex.exec(filenames[i])
434
      if (!m) return next(i+1)
435
      var name = m[1]
436
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
437
        if (err) return cb(new Error(err.stack || err))
438
        self.sbot.blobs.add(function (err, id) {
439
          if (err) return cb(new Error(err.stack || err))
440
          ids[name] = id
441
          next(i+1)
442
        })(pull.once(data))
443
      })
444
    })(0)
445
  })
446
}
447
448
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
449
  var self = this
450
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
451
452
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
453
    if (err) return cb(new Error(err.stack || err))
454
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
455
    if (pkgs._hasNonBlobUrl) {
456
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
457
    }
458
459
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
460
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
461
462
    var waiting = 2
463
464
    self.sbot.blobs.add(function (err, id) {
465
      if (err) return next(new Error(err.stack || err))
466
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
467
      var versions = pkg.versions || (pkg.versions = {})
468
      pkg.versions[sbotPkgLock.version] = {
469
        name: sbotPkgLock.name,
470
        version: sbotPkgLock.version,
471
        dist: self.blobDist(id)
472
      }
473
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
474
      distTags.latest = sbotPkgLock.version
475
      next()
476
    })(self.sbot.bootstrap.pack())
477
478
    var prebuilds
479
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
480
      if (err) return next(err)
481
      prebuilds = _prebuilds
482
      next()
483
    })
484
485
    function next(err) {
486
      if (err) return waiting = 0, cb(err)
487
      if (--waiting) return
488
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
489
        encoding: 'utf8'
490
      }, function (err, bootstrapScript) {
491
        if (err) return cb(err)
492
        var script = bootstrapScript + '\n' +
493
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
494
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
495
496
        self.sbot.blobs.add(function (err, id) {
497
          if (err) return cb(new Error(err.stack || err))
498
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
499
          if (!m) return cb(new Error('bad blob id: ' + id))
500
          cb(null, {
501
            name: sbotPkgLock.name,
502
            blob: id,
503
            hashType: m[2],
504
            hashBuf: Buffer.from(m[1], 'base64'),
505
          })
506
        })(pull.once(script))
507
      })
508
    }
509
  })
510
}
511
512
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
513
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
514
  else this.sbot.get(id, function (err, value) {
515
    if (err) return cb(err)
516
    cb(null, {key: id, value: value})
517
  })
518
}
519
520
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
521
  var self = this
522
  var stack = heads.slice()
523
  var seen = {}
524
  return function (abort, cb) {
525
    if (abort) return cb(abort)
526
    for (var id; stack.length && seen[id = stack.pop()];);
527
    if (!id) return cb(true)
528
    seen[id] = true
529
    // TODO: use DFS
530
    self.getMsg(id, function (err, msg) {
531
      if (err) return cb(new Error(err.stack || err))
532
      var c = msg && msg.value && msg.value.content
533
      var links = c && c[prop]
534
      if (Array.isArray(links)) {
535
        stack.push.apply(stack, links)
536
      } else if (links) {
537
        stack.push(links)
538
      }
539
      cb(null, msg)
540
    })
541
  }
542
}
543
544
function Req(server, req, res) {
545
  this.server = server
546
  this.req = req
547
  this.res = res
548
  this.blobsToPush = []
549
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
550
  var ua = this.req.headers['user-agent']
551
  var m = /\bnpm\/([0-9]*)/.exec(ua)
552
  var npmVersion = m && m[1]
553
  this.needShasum = server.needShasum != null ? server.needShasum :
554
    (npmVersion && npmVersion < 5)
555
}
556
557
Req.prototype.serve = function () {
558
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
559
  this.res.setTimeout(0)
560
  var pathname = this.req.url.replace(/\?.*/, '')
561
  var m
562
  if ((m = /^\/(%25.*sha256)(\/.*)$/.exec(pathname))) {
563
    try {
564
      this.headMsgId = decodeURIComponent(m[1])
565
    } catch(e) {
566
      return this.respondError(400, e.stack || e)
567
    }
568
    pathname = m[2]
569
  }
570
  if (pathname === '/') return this.serveHome()
571
  if (pathname === '/bootstrap') return this.serveBootstrap()
572
  if (pathname === '/-/whoami') return this.serveWhoami()
573
  if (pathname === '/-/ping') return this.respond(200, true)
574
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
575
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
576
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
577
  return this.respond(404)
578
}
579
580
Req.prototype.respond = function (status, message) {
581
  this.res.writeHead(status, {'content-type': 'application/json'})
582
  this.res.end(message && JSON.stringify(message, 0, 2))
583
}
584
585
Req.prototype.respondError = function (status, message) {
586
  this.respond(status, {error: message})
587
}
588
589
var bootstrapName = 'ssb-npm-bootstrap'
590
591
Req.prototype.serveHome = function () {
592
  var self = this
593
  self.res.writeHead(200, {'content-type': 'text/html'})
594
  var port = 8044
595
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
596
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
597
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
598
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
599
    '</body></html>')
600
}
601
602
Req.prototype.serveBootstrap = function () {
603
  var self = this
604
  self.server.getBootstrapInfo(function (err, info) {
605
    if (err) return self.respondError(err.stack || err)
606
    var pkgNameText = info.name
607
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
608
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
609
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
610
    var blobsHostname = httpHost + ':' + self.server.wsPort
611
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
612
    var pkgHashText = info.hashBuf.toString('hex')
613
    var hashCmd = info.hashType + 'sum'
614
615
    var script =
616
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
617
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
618
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
619
        'npm install -g ' + info.name + ' &&\n' +
620
      'sbot server'
621
622
    self.res.writeHead(200, {'content-type': 'text/plain'})
623
    self.res.end(script)
624
  })
625
}
626
627
Req.prototype.serveWhoami = function () {
628
  var self = this
629
  self.server.sbot.whoami(function (err, feed) {
630
    if (err) return self.respondError(err.stack || err)
631
    self.respond(200, {username: feed.id})
632
  })
633
}
634
635
Req.prototype.serveUser1 = function () {
636
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
637
}
638
639
function decodeName(name) {
640
  var parts = name.replace(/\.tgz$/, '').split(':')
641
  return {
642
    name: parts[1],
643
    version: parts[2],
644
    distTag: parts[3],
645
  }
646
}
647
648
Req.prototype.getMentions = function (name) {
649
  var serverMentions = this.server.getMentions(name)
650
  var msgMentions = this.headMsgId ? pull(
651
    this.server.streamTree([this.headMsgId], 'dependencyBranch'),
652
    // decryption could be done here
653
    pull.map(function (msg) {
654
      var c = msg.value && msg.value.content
655
      if (!c.mentions || !Array.isArray(c.mentions)) return []
656
      return c.mentions.map(function (mention) {
657
        return {
658
          name: mention.name,
659
          size: mention.size,
660
          link: mention.link,
661
          author: msg.value.author,
662
          ts: msg.value.timestamp,
663
        }
664
      })
665
    }),
666
    pull.flatten(),
667
    pull.filter(typeof name === 'string' ? function (link) {
668
      return link.name === name
669
    } : name && name.$prefix ? function (link) {
670
      return link.name.substr(0, name.$prefix.length) === name.$prefix
671
    } : function () {
672
      throw new TypeError('unsupported name filter')
673
    }),
674
  ) : pull.empty()
675
  return cat([msgMentions, serverMentions])
676
}
677
678
Req.prototype.getMentionLinks = function (blobId) {
679
  return pull(
680
    this.headMsgId
681
      ? this.server.streamTree([this.headMsgId], 'dependencyBranch')
682
      : this.server.sbot.links({
683
        dest: blobId,
684
        rel: 'mentions',
685
        values: true,
686
      }),
687
    // decryption could be done here
688
    pull.map(function (msg) {
689
      var c = msg.value && msg.value.content
690
      return c && Array.isArray(c.mentions) || []
691
    }),
692
    pull.flatten(),
693
    pull.filter(function (link) {
694
      return link && link.link === blobId
695
    })
696
  )
697
}
698
699
Req.prototype.servePkg = function (pathname) {
700
  var self = this
701
  var parts = pathname.split('/')
702
  var pkgName = parts.shift().replace(/%2f/i, '/')
703
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
704
  var spec = parts.shift()
705
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
706
  if (parts.length > 0) return this.respondError(404)
707
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
708
  var obj = {
709
    _id: pkgName,
710
    name: pkgName,
711
    'dist-tags': {},
712
    versions: {}
713
  }
714
  var distTags = {/* <tag>: {version, ts}*/}
715
  pull(
716
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
717
    pull.drain(function (mention) {
718
      var data = decodeName(mention.name)
719
      if (!data.version) return
720
      if (data.distTag) {
721
        var tag = distTags[data.distTag]
722
        if (!tag || mention.ts > tag.ts) {
723
          /* TODO: sort by causal order (versionBranch links) instead of just
724
          * by timestamps */
725
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
726
        }
727
      }
728
      obj.versions[data.version] = {
729
        author: {
730
          url: mention.author
731
        },
732
        name: pkgName,
733
        version: data.version,
734
        dist: self.server.blobDist(mention.link)
735
      }
736
    }, function (err) {
737
      if (err) return self.respondError(500, err.stack || err)
738
      for (var tag in distTags) {
739
        obj['dist-tags'][tag] = distTags[tag].version
740
      }
741
      if (spec) resolveSpec()
742
      else if (self.fetchAll) resolveAll()
743
      else done()
744
    })
745
  )
746
  function resolveSpec() {
747
    var version = obj['dist-tags'][spec]
748
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
749
    obj = obj.versions[version]
750
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
751
    self.populatePackageJson(obj, function (err, pkg) {
752
      if (err) return self.respondError(500, err.stack || err)
753
      obj = pkg || obj
754
      done()
755
    })
756
  }
757
  function resolveVersion(version, cb) {
758
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
759
      if (err) return cb(err)
760
      if (pkg) obj.versions[version] = pkg
761
      cb()
762
    })
763
  }
764
  function resolveAll() {
765
    var done = multicb()
766
    for (var version in obj.versions) {
767
      resolveVersion(version, done())
768
    }
769
    done(resolved)
770
  }
771
  function resolved(err) {
772
    if (err) return self.respondError(500, err.stack || err)
773
    self.respond(200, obj)
774
  }
775
}
776
777
Req.prototype.servePrebuild = function (name) {
778
  var self = this
779
  var getMention = self.getMentions('prebuild:' + name)
780
  var blobsByAuthor = {/* <author>: BlobId */}
781
  getMention(null, function next(err, link) {
782
    if (err === true) return done()
783
    if (err) return self.respondError(500, err.stack || err)
784
    blobsByAuthor[link.author] = link.link
785
    getMention(null, next)
786
  })
787
  function done() {
788
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
789
    var blobId
790
    for (var feed in blobsByAuthor) {
791
      var blob = blobId = blobsByAuthor[feed]
792
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
793
      feeds.push(feed)
794
    }
795
    switch (Object.keys(authorsByLink).length) {
796
      case 0:
797
        return self.respondError(404, 'Not Found')
798
      case 1:
799
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
800
        return self.res.end()
801
      default:
802
        return self.respond(300, {choices: authorsByLink})
803
    }
804
  }
805
}
806
807
var localhosts = {
808
  '::1': true,
809
  '127.0.0.1': true,
810
  '::ffff:127.0.0.1': true,
811
}
812
813
Req.prototype.publishPkg = function (pkgName) {
814
  var self = this
815
  var remoteAddress = self.req.socket.remoteAddress
816
  if (!(remoteAddress in localhosts)) {
817
    return self.respondError(403, 'You may not publish as this user.')
818
  }
819
820
  var chunks = []
821
  self.req.on('data', function (data) {
822
    chunks.push(data)
823
  })
824
  self.req.on('end', function () {
825
    var data
826
    try {
827
      data = JSON.parse(Buffer.concat(chunks))
828
    } catch(e) {
829
      return self.respondError(400, e.stack)
830
    }
831
    return self.publishPkg2(pkgName, data || {})
832
  })
833
}
834
835
Req.prototype.publishPkg2 = function (name, data) {
836
  var self = this
837
  if (data.users) console.trace('[npm-registry] users property is not supported')
838
  var attachments = data._attachments || {}
839
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
840
  var done = multicb()
841
  function addAttachmentAsBlob(filename, cb) {
842
    var data = attachments[filename].data
843
    var tarball = new Buffer(data, 'base64')
844
    var length = attachments[filename].length
845
    if (length && length !== tarball.length) return self.respondError(400,
846
      'Length mismatch for attachment \'' + filename + '\'')
847
    self.server.sbot.blobs.add(function (err, id) {
848
      if (err) return cb(err)
849
      self.blobsToPush.push(id)
850
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
851
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
852
      cb()
853
    })(pull.once(tarball))
854
  }
855
  for (var filename in attachments) {
856
    addAttachmentAsBlob(filename, done())
857
  }
858
  done(function (err) {
859
    if (err) return self.respondError(500, err.stack || err)
860
    try {
861
      self.publishPkg3(name, data, links)
862
    } catch(e) {
863
      self.respondError(500, e.stack || e)
864
    }
865
  })
866
}
867
868
Req.prototype.publishPkg3 = function (name, data, links) {
869
  var self = this
870
  var versions = data.versions || {}
871
  var linksByVersion = {/* <version>: link */}
872
873
  // associate tarball blobs with versions
874
  for (var version in versions) {
875
    var pkg = versions[version]
876
    if (!pkg) return self.respondError(400, 'Bad package object')
877
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
878
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
879
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
880
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
881
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
882
    var filename = m[1]
883
    var link = links[filename]
884
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
885
    // TODO?: try to find missing tarball mentioned in other messages
886
    if (pkg.version && pkg.version !== version)
887
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
888
    linksByVersion[version] = link
889
    link.version = version
890
    link.dependencies = pkg.dependencies || {}
891
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
892
  }
893
894
  // associate blobs with dist-tags
895
  var tags = data['dist-tags'] || {}
896
  for (var tag in tags) {
897
    var version = tags[tag]
898
    var link = linksByVersion[version]
899
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
900
    // TODO?: support setting dist-tag without version,
901
    // by looking up a tarball blob for the version
902
    link.tag = tag
903
  }
904
905
  // compute blob links to publish
906
  var mentions = []
907
  for (var filename in links) {
908
    var link = links[filename] || {}
909
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
910
    mentions.push({
911
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
912
      link: link.link,
913
      size: link.size,
914
      shasum: link.shasum,
915
      dependencies: link.dependencies,
916
      bundledDependencies: link.bundledDependencies,
917
    })
918
  }
919
  return self.publishPkgs(mentions)
920
}
921
922
Req.prototype.publishPkgs = function (mentions) {
923
  var self = this
924
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
925
    if (err) return self.respondError(500, err.stack || err)
926
    self.server.pushBlobs(self.blobsToPush, function (err) {
927
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
928
      self.respond(201)
929
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
930
    })
931
  })
932
}
933
934
Req.prototype.populatePackageJson = function (obj, cb) {
935
  var self = this
936
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
937
  var deps, bundledDeps, shasum
938
939
  // look for dependencies in links.
940
  // then fallback to getting it from the tarball blob
941
942
  pull(
943
    self.getMentionLinks(blobId),
944
    pull.drain(function (link) {
945
      if (link.dependencies) deps = link.dependencies
946
      if (link.shasum) shasum = link.shasum
947
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
948
      // how to handle multiple assignments of dependencies to a package?
949
    }, function (err) {
950
      if (err) return cb(new Error(err.stack || err))
951
      if (deps && (shasum || !self.needShasum)) {
952
        // assume that the dependencies in the links to the blob are
953
        // correct.
954
        obj.dependencies = deps
955
        obj.bundledDependencies = bundledDeps
956
        if (shasum) obj.dist.shasum = obj._shasum = shasum
957
        cb(null, obj)
958
      } else {
959
        // get dependencies from the tarball
960
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
961
          if (err) return cb(err)
962
          pkg.dist = obj.dist
963
          pkg.dist.shasum = pkg._shasum
964
          pkg.author = pkg.author || obj.author
965
          pkg.version = pkg.version || obj.version
966
          pkg.name = pkg.name || obj.name
967
          cb(null, pkg)
968
        })
969
      }
970
    })
971
  )
972
}
973
974
function getPackageJsonFromTarballBlob(sbot, id, cb) {
975
  var self = this
976
  getBlob(sbot, id, function (err, readBlob) {
977
    if (err) return cb(err)
978
    cb = once(cb)
979
    var extract = tar.extract()
980
    var pkg, shasum
981
    extract.on('entry', function (header, stream, next) {
982
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
983
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
984
          if (err) return cb(err)
985
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
986
          catch(e) { return cb(e) }
987
          next()
988
        }))
989
      } else {
990
        stream.on('end', next)
991
        stream.resume()
992
      }
993
    })
994
    extract.on('finish', function () {
995
      pkg._shasum = shasum
996
      cb(null, pkg)
997
    })
998
    pull(
999
      readBlob,
1000
      hash('sha1', 'hex', function (err, sum) {
1001
        if (err) return cb(err)
1002
        shasum = sum
1003
      }),
1004
      toPull(zlib.createGunzip()),
1005
      toPull(extract)
1006
    )
1007
  })
1008
}
1009