git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function onceify(fn, self) {
25
  var cbs = [], err, data
26
  return function (cb) {
27
    if (fn) {
28
      cbs.push(cb)
29
      fn.call(self, function (_err, _data) {
30
        err = _err, data = _data
31
        var _cbs = cbs
32
        cbs = null
33
        while (_cbs.length) _cbs.shift()(err, data)
34
      })
35
      fn = null
36
    } else if (cbs) {
37
      cbs.push(cb)
38
    } else {
39
      cb(err, data)
40
    }
41
  }
42
}
43
44
function once(cb) {
45
  var done
46
  return function (err, result) {
47
    if (done) {
48
      if (err) console.trace(err)
49
    } else {
50
      done = true
51
      cb(err, result)
52
    }
53
  }
54
}
55
56
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57
  // convert a package-lock.json file into data for serving as an npm registry
58
  var hasNonBlobUrl = false
59
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60
  var pkgs = {}
61
  var queue = [pkgLock, pkgLock.name]
62
  while (queue.length) {
63
    var dep = queue.shift(), name = queue.shift()
64
    if (name) {
65
      var pkg = pkgs[name] || (pkgs[name] = {
66
        _id: name,
67
        name: name,
68
        versions: {}
69
      })
70
      if (dep.version && dep.integrity && dep.resolved) {
71
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72
        pkg.versions[dep.version] = {
73
          name: name,
74
          version: dep.version,
75
          dist: {
76
            integrity: dep.integrity,
77
            tarball: dep.resolved
78
          }
79
        }
80
      }
81
    }
82
    if (dep.dependencies) for (var depName in dep.dependencies) {
83
      queue.push(dep.dependencies[depName], depName)
84
    }
85
  }
86
  pkgs._hasNonBlobUrl = hasNonBlobUrl
87
  return pkgs
88
}
89
90
function npmLogin(registryAddress, cb) {
91
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
92
  var filename = path.join(os.homedir(), '.npmrc')
93
  fs.readFile(filename, 'utf8', function (err, data) {
94
    if (err && err.code === 'ENOENT') data = ''
95
    else if (err) return cb(new Error(err.stack))
96
    var lines = data ? data.split('\n') : []
97
    if (lines.indexOf(tokenLine) > -1) return cb()
98
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
99
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
100
    fs.appendFile(filename, line, cb)
101
  })
102
}
103
104
function formatHost(host) {
105
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
106
}
107
108
exports.name = 'npm-registry'
109
exports.version = '1.0.0'
110
exports.manifest = {
111
  getAddress: 'async'
112
}
113
exports.init = function (sbot, config) {
114
  var conf = config.npm || {}
115
  var port = conf.port || 8043
116
  var host = conf.host || null
117
  var autoAuth = conf.autoAuth !== false
118
119
  var server = http.createServer(exports.respond(sbot, config))
120
  var getAddress = onceify(function (cb) {
121
    server.on('error', cb)
122
    server.listen(port, host, function () {
123
      server.removeListener('error', cb)
124
      var regHost = formatHost(host || 'localhost')
125
      var regPort = this.address().port
126
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
127
      if (autoAuth) npmLogin(regUrl, next)
128
      else next()
129
      function next(err) {
130
        cb(err, regUrl)
131
      }
132
    })
133
    sbot.on('close', function () {
134
      server.close()
135
    })
136
  })
137
138
  getAddress(function (err, addr) {
139
    if (err) return console.error(err)
140
    console.log('[npm-registry] Listening on ' + addr)
141
  })
142
143
  return {
144
    getAddress: getAddress
145
  }
146
}
147
148
exports.respond = function (sbot, config) {
149
  var reg = new SsbNpmRegistryServer(sbot, config)
150
  return function (req, res) {
151
    new Req(reg, req, res).serve()
152
  }
153
}
154
155
function publishMsg(sbot, value, cb) {
156
  var gotExpectedPrevious = false
157
  sbot.publish(value, function next(err, msg) {
158
    if (err && /^expected previous:/.test(err.message)) {
159
      // retry once on this error
160
      if (gotExpectedPrevious) return cb(err)
161
      gotExpectedPrevious = true
162
      return sbot.publish(value, next)
163
    }
164
    cb(err, msg)
165
  })
166
}
167
168
function getDependencyBranches(sbot, id, cb) {
169
  // get ids of heads of tree of dependencyBranch message links to include all
170
  // the dependencies of the given tarball id.
171
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
172
  var msgs = {}
173
  var branched = {}
174
  var blobs = {}
175
176
  function addPkgById(id, cb) {
177
    if (blobs[id]) return cb()
178
    blobs[id] = true
179
    getPackageJsonCached(id, function (err, pkg) {
180
      if (err) return cb(err)
181
      var done = multicb()
182
      for (var name in pkg.dependencies || {}) {
183
        addPkgBySpec(name, pkg.dependencies[name], done())
184
      }
185
      done(cb)
186
    })
187
  }
188
189
  function addPkgBySpec(name, spec, cb) {
190
    var done = multicb()
191
    pull(
192
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
193
      pull.map(function (mention) {
194
        addPkgById(mention.link, done())
195
        return packageLinks(sbot, mention.author, mention.link, name, spec)
196
      }),
197
      pull.flatten(),
198
      pull.drain(function (msg) {
199
        var c = msg && msg.value && msg.value.content
200
        if (!c) return
201
        msgs[msg.key] = msg.value
202
        if (Array.isArray(c.dependencyBranch)) {
203
          for (var k = 0; k < c.dependencyBranch.length; k++) {
204
            branched[c.dependencyBranch[k]] = true
205
          }
206
        }
207
      }, function (err) {
208
        if (err) return cb(err)
209
        done(cb)
210
      })
211
    )
212
  }
213
214
  addPkgById(id, function (err) {
215
    if (err) return cb(err)
216
    var ids = []
217
    for (var key in msgs) {
218
      if (!branched[key]) ids.push(key)
219
    }
220
    cb(null, ids)
221
  })
222
}
223
224
function packageLinks(sbot, feed, id, name, spec) {
225
  function matches(mention) {
226
    var data = mention
227
      && mention.link === id
228
      && mention.name
229
      && decodeName(mention.name)
230
    return data
231
      && data.name === name
232
      && (spec ? semver.satisfies(data.version, spec) : true)
233
  }
234
  return pull(
235
    sbot.links({
236
      source: feed,
237
      dest: id,
238
      rel: 'mentions',
239
      values: true,
240
    }),
241
    pull.filter(function (msg) {
242
      var c = msg && msg.value && msg.value.content
243
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
244
    })
245
  )
246
}
247
248
function getVersionBranches(sbot, link, cb) {
249
  var data = decodeName(link.name)
250
  var msgs = {}, branched = {}
251
  pull(
252
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
253
    pull.map(function (mention) {
254
      return packageLinks(sbot, mention.author, mention.link, data.name)
255
    }),
256
    pull.flatten(),
257
    pull.drain(function (msg) {
258
      var c = msg && msg.value && msg.value.content
259
      if (!c) return
260
      msgs[msg.key] = msg.value
261
      if (Array.isArray(c.versionBranch)) {
262
        for (var k = 0; k < c.versionBranch.length; k++) {
263
          branched[c.versionBranch[k]] = true
264
        }
265
      }
266
    }, function (err) {
267
      if (err) return cb(err)
268
      var ids = []
269
      for (var key in msgs) {
270
        if (!branched[key]) ids.push(key)
271
      }
272
      cb(null, ids)
273
    })
274
  )
275
}
276
277
// For each dependency that is not a bundledDependency, get message ids for
278
// that dependency name + version.
279
function publishSingleMention(sbot, mention, cb) {
280
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
281
  // Calculate dependencyBranch and versionBranch message ids.
282
  var value = {
283
    type: 'npm-packages',
284
    mentions: [mention]
285
  }
286
  var done = multicb({pluck: 1, spread: true})
287
  getDependencyBranches(sbot, mention.link, done())
288
  getVersionBranches(sbot, mention, done())
289
  done(function (err, dependencyBranches, versionBranches) {
290
    if (err) return cb(err)
291
    value.dependencyBranch = dependencyBranches || undefined
292
    value.versionBranch = versionBranches || undefined
293
    publishMsg(sbot, value, cb)
294
  })
295
}
296
297
function publishMentions(sbot, mentions, cb) {
298
  // console.error("publishing %s mentions", mentions.length)
299
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
300
  // if it is just one mention, fetch and add useful metadata
301
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
302
  publishMsg(sbot, {
303
    type: 'npm-packages',
304
    mentions: mentions,
305
  }, cb)
306
}
307
308
exports.publishPkgMentions = function (sbot, mentions, cb) {
309
  // try to fit the mentions into as few messages as possible,
310
  // while fitting under the message size limit.
311
  var msgs = []
312
  ;(function next(i, chunks) {
313
    if (i >= mentions.length) return cb(null, msgs)
314
    var chunkLen = Math.ceil(mentions.length / chunks)
315
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
316
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
317
      if (err && msgs.length) return onPartialPublish(err)
318
      if (err) return cb(err)
319
      msgs.push(msg)
320
      next(i + chunkLen, chunks)
321
    })
322
  })(0, 1)
323
  function onPartialPublish(err) {
324
    var remaining = mentions.length - i
325
    return cb(new Error('Published messages ' +
326
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
327
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
328
  }
329
}
330
331
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
332
  cb = once(cb)
333
  var waiting = 0
334
  var expandedMentions = mentions.map(function (link) {
335
    var id = link && link.link
336
    if (!id) return link
337
    waiting++
338
    var newLink = {}
339
    for (var k in link) newLink[k] = link[k]
340
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
341
      if (err) return cb(err)
342
      for (var k in props) newLink[k] = pkg[k]
343
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
344
      if (!--waiting) next()
345
    })
346
    return newLink
347
  })
348
  if (!waiting) next()
349
  function next() {
350
    cb(null, expandedMentions)
351
  }
352
}
353
354
function SsbNpmRegistryServer(sbot, config) {
355
  this.sbot = sbot
356
  this.config = config
357
  this.npmConfig = config.npm || {}
358
  this.host = this.npmConfig.host || 'localhost'
359
  this.fetchAll = this.npmConfig.fetchAll
360
  this.needShasum = this.npmConfig.needShasum
361
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
362
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
363
    + this.wsPort + '/blobs/get/'
364
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
365
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
366
}
367
368
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
369
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
370
371
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
372
  var self = this
373
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
374
  ;(function next(i) {
375
    if (i >= ids.length) return cb()
376
    self.sbot.blobs.push(ids[i], function (err) {
377
      if (err) return cb(err)
378
      next(i+1)
379
    })
380
  })(0)
381
}
382
383
function getBlob(sbot, id, cb) {
384
  var blobs = sbot.blobs
385
  blobs.size(id, function (err, size) {
386
    if (typeof size === 'number') cb(null, blobs.get(id))
387
    else blobs.want(id, function (err, got) {
388
      if (err) cb(err)
389
      else if (!got) cb('missing blob ' + id)
390
      else cb(null, blobs.get(id))
391
    })
392
  })
393
}
394
395
SsbNpmRegistryServer.prototype.blobDist = function (id) {
396
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
397
  if (!m) throw new Error('bad blob id: ' + id)
398
  return {
399
    integrity: m[2] + '-' + m[1],
400
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
401
  }
402
}
403
404
function getMentions(links2, name) {
405
  return links2.read({
406
    query: [
407
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
408
      {$filter: {dest: {$prefix: '&'}}},
409
      {$map: {
410
        name: ['rel', 1],
411
        size: ['rel', 2],
412
        link: 'dest',
413
        author: 'source',
414
        ts: 'ts'
415
      }}
416
    ]
417
  })
418
}
419
420
SsbNpmRegistryServer.prototype.getMentions = function (name) {
421
  if (!this.sbot.links2) return pull.empty()
422
  return getMentions(this.sbot.links2, name)
423
}
424
425
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
426
  var self = this
427
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
428
  var ids = {}
429
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
430
  fs.readdir(prebuildsDir, function (err, filenames) {
431
    if (err) return cb(new Error(err.stack || err))
432
    ;(function next(i) {
433
      if (i >= filenames.length) return cb(null, ids)
434
      var m = nameRegex.exec(filenames[i])
435
      if (!m) return next(i+1)
436
      var name = m[1]
437
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
438
        if (err) return cb(new Error(err.stack || err))
439
        self.sbot.blobs.add(function (err, id) {
440
          if (err) return cb(new Error(err.stack || err))
441
          ids[name] = id
442
          next(i+1)
443
        })(pull.once(data))
444
      })
445
    })(0)
446
  })
447
}
448
449
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
450
  var self = this
451
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
452
453
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
454
    if (err) return cb(new Error(err.stack || err))
455
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
456
    if (pkgs._hasNonBlobUrl) {
457
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
458
    }
459
460
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
461
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
462
463
    var waiting = 2
464
465
    self.sbot.blobs.add(function (err, id) {
466
      if (err) return next(new Error(err.stack || err))
467
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
468
      var versions = pkg.versions || (pkg.versions = {})
469
      pkg.versions[sbotPkgLock.version] = {
470
        name: sbotPkgLock.name,
471
        version: sbotPkgLock.version,
472
        dist: self.blobDist(id)
473
      }
474
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
475
      distTags.latest = sbotPkgLock.version
476
      next()
477
    })(self.sbot.bootstrap.pack())
478
479
    var prebuilds
480
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
481
      if (err) return next(err)
482
      prebuilds = _prebuilds
483
      next()
484
    })
485
486
    function next(err) {
487
      if (err) return waiting = 0, cb(err)
488
      if (--waiting) return
489
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
490
        encoding: 'utf8'
491
      }, function (err, bootstrapScript) {
492
        if (err) return cb(err)
493
        var script = bootstrapScript + '\n' +
494
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
495
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
496
497
        self.sbot.blobs.add(function (err, id) {
498
          if (err) return cb(new Error(err.stack || err))
499
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
500
          if (!m) return cb(new Error('bad blob id: ' + id))
501
          cb(null, {
502
            name: sbotPkgLock.name,
503
            blob: id,
504
            hashType: m[2],
505
            hashBuf: Buffer.from(m[1], 'base64'),
506
          })
507
        })(pull.once(script))
508
      })
509
    }
510
  })
511
}
512
513
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
514
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
515
  else this.sbot.get(id, function (err, value) {
516
    if (err) return cb(err)
517
    cb(null, {key: id, value: value})
518
  })
519
}
520
521
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
522
  var self = this
523
  var stack = heads.slice()
524
  var seen = {}
525
  return function (abort, cb) {
526
    if (abort) return cb(abort)
527
    for (var id; stack.length && seen[id = stack.pop()];);
528
    if (!id) return cb(true)
529
    seen[id] = true
530
    // TODO: use DFS
531
    self.getMsg(id, function (err, msg) {
532
      if (err) return cb(new Error(err.stack || err))
533
      var c = msg && msg.value && msg.value.content
534
      var links = c && c[prop]
535
      if (Array.isArray(links)) {
536
        stack.push.apply(stack, links)
537
      } else if (links) {
538
        stack.push(links)
539
      }
540
      cb(null, msg)
541
    })
542
  }
543
}
544
545
function Req(server, req, res) {
546
  this.server = server
547
  this.req = req
548
  this.res = res
549
  this.blobsToPush = []
550
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
551
  var ua = this.req.headers['user-agent']
552
  var m = /\bnpm\/([0-9]*)/.exec(ua)
553
  var npmVersion = m && m[1]
554
  this.needShasum = server.needShasum != null ? server.needShasum :
555
    (npmVersion && npmVersion < 5)
556
}
557
558
Req.prototype.serve = function () {
559
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
560
  this.res.setTimeout(0)
561
  var pathname = this.req.url.replace(/\?.*/, '')
562
  var m
563
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)(\/.*)$/.exec(pathname))) {
564
    try {
565
      this.headMsgId = decodeURIComponent(m[2])
566
      this.headMsgPlus = !!m[1]
567
      // ^ means also include packages published after the head message id
568
    } catch(e) {
569
      return this.respondError(400, e.stack || e)
570
    }
571
    pathname = m[3]
572
  }
573
  if (pathname === '/') return this.serveHome()
574
  if (pathname === '/bootstrap') return this.serveBootstrap()
575
  if (pathname === '/-/whoami') return this.serveWhoami()
576
  if (pathname === '/-/ping') return this.respond(200, true)
577
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
578
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
579
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
580
  return this.respond(404)
581
}
582
583
Req.prototype.respond = function (status, message) {
584
  this.res.writeHead(status, {'content-type': 'application/json'})
585
  this.res.end(message && JSON.stringify(message, 0, 2))
586
}
587
588
Req.prototype.respondError = function (status, message) {
589
  this.respond(status, {error: message})
590
}
591
592
var bootstrapName = 'ssb-npm-bootstrap'
593
594
Req.prototype.serveHome = function () {
595
  var self = this
596
  self.res.writeHead(200, {'content-type': 'text/html'})
597
  var port = 8044
598
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
599
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
600
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
601
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
602
    '</body></html>')
603
}
604
605
Req.prototype.serveBootstrap = function () {
606
  var self = this
607
  self.server.getBootstrapInfo(function (err, info) {
608
    if (err) return self.respondError(err.stack || err)
609
    var pkgNameText = info.name
610
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
611
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
612
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
613
    var blobsHostname = httpHost + ':' + self.server.wsPort
614
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
615
    var pkgHashText = info.hashBuf.toString('hex')
616
    var hashCmd = info.hashType + 'sum'
617
618
    var script =
619
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
620
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
621
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
622
        'npm install -g ' + info.name + ' &&\n' +
623
      'sbot server'
624
625
    self.res.writeHead(200, {'content-type': 'text/plain'})
626
    self.res.end(script)
627
  })
628
}
629
630
Req.prototype.serveWhoami = function () {
631
  var self = this
632
  self.server.sbot.whoami(function (err, feed) {
633
    if (err) return self.respondError(err.stack || err)
634
    self.respond(200, {username: feed.id})
635
  })
636
}
637
638
Req.prototype.serveUser1 = function () {
639
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
640
}
641
642
function decodeName(name) {
643
  var parts = String(name).replace(/\.tgz$/, '').split(':')
644
  return {
645
    name: parts[1],
646
    version: parts[2],
647
    distTag: parts[3],
648
  }
649
}
650
651
Req.prototype.getMsgMentions = function (name) {
652
  return pull(
653
    this.server.streamTree([this.headMsgId], 'dependencyBranch'),
654
    // decryption could be done here
655
    pull.map(function (msg) {
656
      var c = msg.value && msg.value.content
657
      if (!c.mentions || !Array.isArray(c.mentions)) return []
658
      return c.mentions.map(function (mention) {
659
        return {
660
          name: mention.name,
661
          size: mention.size,
662
          link: mention.link,
663
          author: msg.value.author,
664
          ts: msg.value.timestamp,
665
        }
666
      })
667
    }),
668
    pull.flatten(),
669
    pull.filter(typeof name === 'string' ? function (link) {
670
      return link.name === name
671
    } : name && name.$prefix ? function (link) {
672
      return link.name.substr(0, name.$prefix.length) === name.$prefix
673
    } : function () {
674
      throw new TypeError('unsupported name filter')
675
    })
676
  )
677
}
678
679
Req.prototype.getMentions = function (name) {
680
  var useMsgMentions = this.headMsgId
681
  var useServerMentions = !this.headMsgId || this.headMsgPlus
682
  if (useServerMentions && !this.server.sbot.links2) {
683
    return this.headMsgPlus
684
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
685
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
686
  }
687
  return cat([
688
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
689
    useServerMentions ? this.server.getMentions(name) : pull.empty()
690
  ])
691
}
692
693
Req.prototype.getMentionLinks = function (blobId) {
694
  return pull(
695
    this.headMsgId
696
      ? this.server.streamTree([this.headMsgId], 'dependencyBranch')
697
      : this.server.sbot.links({
698
        dest: blobId,
699
        rel: 'mentions',
700
        values: true,
701
      }),
702
    // decryption could be done here
703
    pull.map(function (msg) {
704
      var c = msg.value && msg.value.content
705
      return c && Array.isArray(c.mentions) || []
706
    }),
707
    pull.flatten(),
708
    pull.filter(function (link) {
709
      return link && link.link === blobId
710
    })
711
  )
712
}
713
714
Req.prototype.servePkg = function (pathname) {
715
  var self = this
716
  var parts = pathname.split('/')
717
  var pkgName = parts.shift().replace(/%2f/i, '/')
718
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
719
  var spec = parts.shift()
720
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
721
  if (parts.length > 0) return this.respondError(404)
722
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
723
  var obj = {
724
    _id: pkgName,
725
    name: pkgName,
726
    'dist-tags': {},
727
    versions: {}
728
  }
729
  var distTags = {/* <tag>: {version, ts}*/}
730
  pull(
731
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
732
    pull.drain(function (mention) {
733
      var data = decodeName(mention.name)
734
      if (!data.version) return
735
      if (data.distTag) {
736
        var tag = distTags[data.distTag]
737
        if (!tag || mention.ts > tag.ts) {
738
          /* TODO: sort by causal order (versionBranch links) instead of just
739
          * by timestamps */
740
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
741
        }
742
      }
743
      obj.versions[data.version] = {
744
        author: {
745
          url: mention.author
746
        },
747
        name: pkgName,
748
        version: data.version,
749
        dist: self.server.blobDist(mention.link)
750
      }
751
    }, function (err) {
752
      if (err) return self.respondError(500, err.stack || err)
753
      for (var tag in distTags) {
754
        obj['dist-tags'][tag] = distTags[tag].version
755
      }
756
      if (spec) resolveSpec()
757
      else if (self.fetchAll) resolveAll()
758
      else resolved()
759
    })
760
  )
761
  function resolveSpec() {
762
    var version = obj['dist-tags'][spec]
763
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
764
    obj = obj.versions[version]
765
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
766
    self.populatePackageJson(obj, function (err, pkg) {
767
      if (err) return resolved(err)
768
      obj = pkg || obj
769
      resolved()
770
    })
771
  }
772
  function resolveVersion(version, cb) {
773
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
774
      if (err) return cb(err)
775
      if (pkg) obj.versions[version] = pkg
776
      cb()
777
    })
778
  }
779
  function resolveAll() {
780
    var done = multicb()
781
    for (var version in obj.versions) {
782
      resolveVersion(version, done())
783
    }
784
    done(resolved)
785
  }
786
  function resolved(err) {
787
    if (err) return self.respondError(500, err.stack || err)
788
    self.respond(200, obj)
789
  }
790
}
791
792
Req.prototype.servePrebuild = function (name) {
793
  var self = this
794
  var getMention = self.getMentions('prebuild:' + name)
795
  var blobsByAuthor = {/* <author>: BlobId */}
796
  getMention(null, function next(err, link) {
797
    if (err === true) return done()
798
    if (err) return self.respondError(500, err.stack || err)
799
    blobsByAuthor[link.author] = link.link
800
    getMention(null, next)
801
  })
802
  function done() {
803
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
804
    var blobId
805
    for (var feed in blobsByAuthor) {
806
      var blob = blobId = blobsByAuthor[feed]
807
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
808
      feeds.push(feed)
809
    }
810
    switch (Object.keys(authorsByLink).length) {
811
      case 0:
812
        return self.respondError(404, 'Not Found')
813
      case 1:
814
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
815
        return self.res.end()
816
      default:
817
        return self.respond(300, {choices: authorsByLink})
818
    }
819
  }
820
}
821
822
var localhosts = {
823
  '::1': true,
824
  '127.0.0.1': true,
825
  '::ffff:127.0.0.1': true,
826
}
827
828
Req.prototype.publishPkg = function (pkgName) {
829
  var self = this
830
  var remoteAddress = self.req.socket.remoteAddress
831
  if (!(remoteAddress in localhosts)) {
832
    return self.respondError(403, 'You may not publish as this user.')
833
  }
834
835
  var chunks = []
836
  self.req.on('data', function (data) {
837
    chunks.push(data)
838
  })
839
  self.req.on('end', function () {
840
    var data
841
    try {
842
      data = JSON.parse(Buffer.concat(chunks))
843
    } catch(e) {
844
      return self.respondError(400, e.stack)
845
    }
846
    return self.publishPkg2(pkgName, data || {})
847
  })
848
}
849
850
Req.prototype.publishPkg2 = function (name, data) {
851
  var self = this
852
  if (data.users) console.trace('[npm-registry] users property is not supported')
853
  var attachments = data._attachments || {}
854
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
855
  var done = multicb()
856
  function addAttachmentAsBlob(filename, cb) {
857
    var data = attachments[filename].data
858
    var tarball = new Buffer(data, 'base64')
859
    var length = attachments[filename].length
860
    if (length && length !== tarball.length) return self.respondError(400,
861
      'Length mismatch for attachment \'' + filename + '\'')
862
    self.server.sbot.blobs.add(function (err, id) {
863
      if (err) return cb(err)
864
      self.blobsToPush.push(id)
865
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
866
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
867
      cb()
868
    })(pull.once(tarball))
869
  }
870
  for (var filename in attachments) {
871
    addAttachmentAsBlob(filename, done())
872
  }
873
  done(function (err) {
874
    if (err) return self.respondError(500, err.stack || err)
875
    try {
876
      self.publishPkg3(name, data, links)
877
    } catch(e) {
878
      self.respondError(500, e.stack || e)
879
    }
880
  })
881
}
882
883
Req.prototype.publishPkg3 = function (name, data, links) {
884
  var self = this
885
  var versions = data.versions || {}
886
  var linksByVersion = {/* <version>: link */}
887
888
  // associate tarball blobs with versions
889
  for (var version in versions) {
890
    var pkg = versions[version]
891
    if (!pkg) return self.respondError(400, 'Bad package object')
892
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
893
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
894
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
895
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
896
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
897
    var filename = m[1]
898
    var link = links[filename]
899
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
900
    // TODO?: try to find missing tarball mentioned in other messages
901
    if (pkg.version && pkg.version !== version)
902
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
903
    linksByVersion[version] = link
904
    link.version = version
905
    link.dependencies = pkg.dependencies || {}
906
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
907
  }
908
909
  // associate blobs with dist-tags
910
  var tags = data['dist-tags'] || {}
911
  for (var tag in tags) {
912
    var version = tags[tag]
913
    var link = linksByVersion[version]
914
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
915
    // TODO?: support setting dist-tag without version,
916
    // by looking up a tarball blob for the version
917
    link.tag = tag
918
  }
919
920
  // compute blob links to publish
921
  var mentions = []
922
  for (var filename in links) {
923
    var link = links[filename] || {}
924
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
925
    mentions.push({
926
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
927
      link: link.link,
928
      size: link.size,
929
      shasum: link.shasum,
930
      dependencies: link.dependencies,
931
      bundledDependencies: link.bundledDependencies,
932
    })
933
  }
934
  return self.publishPkgs(mentions)
935
}
936
937
Req.prototype.publishPkgs = function (mentions) {
938
  var self = this
939
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
940
    if (err) return self.respondError(500, err.stack || err)
941
    self.server.pushBlobs(self.blobsToPush, function (err) {
942
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
943
      self.respond(201)
944
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
945
    })
946
  })
947
}
948
949
Req.prototype.populatePackageJson = function (obj, cb) {
950
  var self = this
951
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
952
  var deps, bundledDeps, shasum
953
954
  // look for dependencies in links.
955
  // then fallback to getting it from the tarball blob
956
957
  pull(
958
    self.getMentionLinks(blobId),
959
    pull.drain(function (link) {
960
      if (link.dependencies) deps = link.dependencies
961
      if (link.shasum) shasum = link.shasum
962
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
963
      // how to handle multiple assignments of dependencies to a package?
964
    }, function (err) {
965
      if (err) return cb(new Error(err.stack || err))
966
      if (deps && (shasum || !self.needShasum)) {
967
        // assume that the dependencies in the links to the blob are
968
        // correct.
969
        obj.dependencies = deps
970
        obj.bundledDependencies = bundledDeps
971
        if (shasum) obj.dist.shasum = obj._shasum = shasum
972
        cb(null, obj)
973
      } else {
974
        // get dependencies from the tarball
975
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
976
          if (err) return cb(err)
977
          pkg.dist = obj.dist
978
          pkg.dist.shasum = pkg._shasum
979
          pkg.author = pkg.author || obj.author
980
          pkg.version = pkg.version || obj.version
981
          pkg.name = pkg.name || obj.name
982
          cb(null, pkg)
983
        })
984
      }
985
    })
986
  )
987
}
988
989
function getPackageJsonFromTarballBlob(sbot, id, cb) {
990
  var self = this
991
  getBlob(sbot, id, function (err, readBlob) {
992
    if (err) return cb(err)
993
    cb = once(cb)
994
    var extract = tar.extract()
995
    var pkg, shasum
996
    extract.on('entry', function (header, stream, next) {
997
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
998
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
999
          if (err) return cb(err)
1000
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1001
          catch(e) { return cb(e) }
1002
          next()
1003
        }))
1004
      } else {
1005
        stream.on('end', next)
1006
        stream.resume()
1007
      }
1008
    })
1009
    extract.on('finish', function () {
1010
      pkg._shasum = shasum
1011
      cb(null, pkg)
1012
    })
1013
    pull(
1014
      readBlob,
1015
      hash('sha1', 'hex', function (err, sum) {
1016
        if (err) return cb(err)
1017
        shasum = sum
1018
      }),
1019
      toPull(zlib.createGunzip()),
1020
      toPull(extract)
1021
    )
1022
  })
1023
}
1024