git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function onceify(fn, self) {
25
  var cbs = [], err, data
26
  return function (cb) {
27
    if (fn) {
28
      cbs.push(cb)
29
      fn.call(self, function (_err, _data) {
30
        err = _err, data = _data
31
        var _cbs = cbs
32
        cbs = null
33
        while (_cbs.length) _cbs.shift()(err, data)
34
      })
35
      fn = null
36
    } else if (cbs) {
37
      cbs.push(cb)
38
    } else {
39
      cb(err, data)
40
    }
41
  }
42
}
43
44
function once(cb) {
45
  var done
46
  return function (err, result) {
47
    if (done) {
48
      if (err) console.trace(err)
49
    } else {
50
      done = true
51
      cb(err, result)
52
    }
53
  }
54
}
55
56
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57
  // convert a package-lock.json file into data for serving as an npm registry
58
  var hasNonBlobUrl = false
59
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60
  var pkgs = {}
61
  var queue = [pkgLock, pkgLock.name]
62
  while (queue.length) {
63
    var dep = queue.shift(), name = queue.shift()
64
    if (name) {
65
      var pkg = pkgs[name] || (pkgs[name] = {
66
        _id: name,
67
        name: name,
68
        versions: {}
69
      })
70
      if (dep.version && dep.integrity && dep.resolved) {
71
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72
        pkg.versions[dep.version] = {
73
          name: name,
74
          version: dep.version,
75
          dist: {
76
            integrity: dep.integrity,
77
            tarball: dep.resolved
78
          }
79
        }
80
      }
81
    }
82
    if (dep.dependencies) for (var depName in dep.dependencies) {
83
      queue.push(dep.dependencies[depName], depName)
84
    }
85
  }
86
  pkgs._hasNonBlobUrl = hasNonBlobUrl
87
  return pkgs
88
}
89
90
function npmLogin(registryAddress, cb) {
91
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
92
  var filename = path.join(os.homedir(), '.npmrc')
93
  fs.readFile(filename, 'utf8', function (err, data) {
94
    if (err && err.code === 'ENOENT') data = ''
95
    else if (err) return cb(new Error(err.stack))
96
    var lines = data ? data.split('\n') : []
97
    if (lines.indexOf(tokenLine) > -1) return cb()
98
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
99
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
100
    fs.appendFile(filename, line, cb)
101
  })
102
}
103
104
function formatHost(host) {
105
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
106
}
107
108
exports.name = 'npm-registry'
109
exports.version = '1.0.0'
110
exports.manifest = {
111
  getAddress: 'async'
112
}
113
exports.init = function (sbot, config) {
114
  var conf = config.npm || {}
115
  var port = conf.port || 8043
116
  var host = conf.host || null
117
  var autoAuth = conf.autoAuth !== false
118
119
  var server = http.createServer(exports.respond(sbot, config))
120
  var getAddress = onceify(function (cb) {
121
    server.on('error', cb)
122
    server.listen(port, host, function () {
123
      server.removeListener('error', cb)
124
      var regHost = formatHost(host || 'localhost')
125
      var regPort = this.address().port
126
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
127
      if (autoAuth) npmLogin(regUrl, next)
128
      else next()
129
      function next(err) {
130
        cb(err, regUrl)
131
      }
132
    })
133
    sbot.on('close', function () {
134
      server.close()
135
    })
136
  })
137
138
  getAddress(function (err, addr) {
139
    if (err) return console.error(err)
140
    console.log('[npm-registry] Listening on ' + addr)
141
  })
142
143
  return {
144
    getAddress: getAddress
145
  }
146
}
147
148
exports.respond = function (sbot, config) {
149
  var reg = new SsbNpmRegistryServer(sbot, config)
150
  return function (req, res) {
151
    new Req(reg, req, res).serve()
152
  }
153
}
154
155
function publishMsg(sbot, value, cb) {
156
  var gotExpectedPrevious = false
157
  sbot.publish(value, function next(err, msg) {
158
    if (err && /^expected previous:/.test(err.message)) {
159
      // retry once on this error
160
      if (gotExpectedPrevious) return cb(err)
161
      gotExpectedPrevious = true
162
      return sbot.publish(value, next)
163
    }
164
    cb(err, msg)
165
  })
166
}
167
168
function getDependencyBranches(sbot, id, cb) {
169
  // get ids of heads of tree of dependencyBranch message links to include all
170
  // the dependencies of the given tarball id.
171
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
172
  var msgs = {}
173
  var branched = {}
174
  var blobs = {}
175
176
  function addPkgById(id, cb) {
177
    if (blobs[id]) return cb()
178
    blobs[id] = true
179
    getPackageJsonCached(id, function (err, pkg) {
180
      if (err) return cb(err)
181
      var done = multicb()
182
      for (var name in pkg.dependencies || {}) {
183
        addPkgBySpec(name, pkg.dependencies[name], done())
184
      }
185
      done(cb)
186
    })
187
  }
188
189
  function addPkgBySpec(name, spec, cb) {
190
    var done = multicb()
191
    pull(
192
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
193
      pull.map(function (mention) {
194
        addPkgById(mention.link, done())
195
        return packageLinks(sbot, mention.author, mention.link, name, spec)
196
      }),
197
      pull.flatten(),
198
      pull.drain(function (msg) {
199
        var c = msg && msg.value && msg.value.content
200
        if (!c) return
201
        msgs[msg.key] = msg.value
202
        if (Array.isArray(c.dependencyBranch)) {
203
          for (var k = 0; k < c.dependencyBranch.length; k++) {
204
            branched[c.dependencyBranch[k]] = true
205
          }
206
        }
207
      }, function (err) {
208
        if (err) return cb(err)
209
        done(cb)
210
      })
211
    )
212
  }
213
214
  addPkgById(id, function (err) {
215
    if (err) return cb(err)
216
    var ids = []
217
    for (var key in msgs) {
218
      if (!branched[key]) ids.push(key)
219
    }
220
    cb(null, ids)
221
  })
222
}
223
224
function packageLinks(sbot, feed, id, name, spec) {
225
  function matches(mention) {
226
    var data = mention
227
      && mention.link === id
228
      && decodeName(mention.name)
229
    return data
230
      && data.name === name
231
      && (spec ? semver.satisfies(data.version, spec) : true)
232
  }
233
  return pull(
234
    sbot.links({
235
      source: feed,
236
      dest: id,
237
      rel: 'mentions',
238
      values: true,
239
    }),
240
    pull.filter(function (msg) {
241
      var c = msg && msg.value && msg.value.content
242
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
243
    })
244
  )
245
}
246
247
function getVersionBranches(sbot, link, cb) {
248
  var data = decodeName(link.name)
249
  var msgs = {}, branched = {}
250
  pull(
251
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
252
    pull.map(function (mention) {
253
      return packageLinks(sbot, mention.author, mention.link, data.name)
254
    }),
255
    pull.flatten(),
256
    pull.drain(function (msg) {
257
      var c = msg && msg.value && msg.value.content
258
      if (!c) return
259
      msgs[msg.key] = msg.value
260
      if (Array.isArray(c.versionBranch)) {
261
        for (var k = 0; k < c.versionBranch.length; k++) {
262
          branched[c.versionBranch[k]] = true
263
        }
264
      }
265
    }, function (err) {
266
      if (err) return cb(err)
267
      var ids = []
268
      for (var key in msgs) {
269
        if (!branched[key]) ids.push(key)
270
      }
271
      cb(null, ids)
272
    })
273
  )
274
}
275
276
// For each dependency that is not a bundledDependency, get message ids for
277
// that dependency name + version.
278
function publishSingleMention(sbot, mention, cb) {
279
  // Calculate dependencyBranch and versionBranch message ids.
280
  var value = {
281
    type: 'npm-packages',
282
    mentions: [mention]
283
  }
284
  var done = multicb({pluck: 1, spread: true})
285
  getDependencyBranches(sbot, mention.link, done())
286
  getVersionBranches(sbot, mention, done())
287
  done(function (err, dependencyBranches, versionBranches) {
288
    if (err) return cb(err)
289
    value.dependencyBranch = dependencyBranches || undefined
290
    value.versionBranch = versionBranches || undefined
291
    publishMsg(sbot, value, cb)
292
  })
293
}
294
295
function publishMentions(sbot, mentions, cb) {
296
  // console.error("publishing %s mentions", mentions.length)
297
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
298
  // if it is just one mention, fetch and add useful metadata
299
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
300
  publishMsg(sbot, {
301
    type: 'npm-packages',
302
    mentions: mentions,
303
  }, cb)
304
}
305
306
exports.publishPkgMentions = function (sbot, mentions, cb) {
307
  // try to fit the mentions into as few messages as possible,
308
  // while fitting under the message size limit.
309
  var msgs = []
310
  ;(function next(i, chunks) {
311
    if (i >= mentions.length) return cb(null, msgs)
312
    var chunkLen = Math.ceil(mentions.length / chunks)
313
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
314
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
315
      if (err && msgs.length) return onPartialPublish(err)
316
      if (err) return cb(err)
317
      msgs.push(msg)
318
      next(i + chunkLen, chunks)
319
    })
320
  })(0, 1)
321
  function onPartialPublish(err) {
322
    var remaining = mentions.length - i
323
    return cb(new Error('Published messages ' +
324
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
325
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
326
  }
327
}
328
329
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
330
  cb = once(cb)
331
  var waiting = 0
332
  var expandedMentions = mentions.map(function (link) {
333
    var id = link && link.link
334
    if (!id) return link
335
    waiting++
336
    var newLink = {}
337
    for (var k in link) newLink[k] = link[k]
338
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
339
      if (err) return cb(err)
340
      for (var k in props) newLink[k] = pkg[k]
341
      if (!--waiting) next()
342
    })
343
    return newLink
344
  })
345
  if (!waiting) next()
346
  function next() {
347
    cb(null, expandedMentions)
348
  }
349
}
350
351
function SsbNpmRegistryServer(sbot, config) {
352
  this.sbot = sbot
353
  this.config = config
354
  this.npmConfig = config.npm || {}
355
  this.host = this.npmConfig.host || 'localhost'
356
  this.fetchAll = this.npmConfig.fetchAll
357
  this.links2 = sbot.links2
358
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
359
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
360
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
361
    + this.wsPort + '/blobs/get/'
362
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
363
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
364
}
365
366
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
367
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
368
369
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
370
  var self = this
371
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
372
  ;(function next(i) {
373
    if (i >= ids.length) return cb()
374
    self.sbot.blobs.push(ids[i], function (err) {
375
      if (err) return cb(err)
376
      next(i+1)
377
    })
378
  })(0)
379
}
380
381
function getBlob(sbot, id, cb) {
382
  var blobs = sbot.blobs
383
  blobs.size(id, function (err, size) {
384
    if (typeof size === 'number') cb(null, blobs.get(id))
385
    else blobs.want(id, function (err, got) {
386
      if (err) cb(err)
387
      else if (!got) cb('missing blob ' + id)
388
      else cb(null, blobs.get(id))
389
    })
390
  })
391
}
392
393
SsbNpmRegistryServer.prototype.blobDist = function (id) {
394
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
395
  if (!m) throw new Error('bad blob id: ' + id)
396
  return {
397
    integrity: m[2] + '-' + m[1],
398
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
399
  }
400
}
401
402
function getMentions(links2, name) {
403
  return links2.read({
404
    query: [
405
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
406
      {$filter: {dest: {$prefix: '&'}}},
407
      {$map: {
408
        name: ['rel', 1],
409
        size: ['rel', 2],
410
        link: 'dest',
411
        author: 'source',
412
        ts: 'ts'
413
      }}
414
    ]
415
  })
416
}
417
418
SsbNpmRegistryServer.prototype.getMentions = function (name) {
419
  return getMentions(this.links2, name)
420
}
421
422
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
423
  var self = this
424
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
425
  var ids = {}
426
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
427
  fs.readdir(prebuildsDir, function (err, filenames) {
428
    if (err) return cb(new Error(err.stack || err))
429
    ;(function next(i) {
430
      if (i >= filenames.length) return cb(null, ids)
431
      var m = nameRegex.exec(filenames[i])
432
      if (!m) return next(i+1)
433
      var name = m[1]
434
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
435
        if (err) return cb(new Error(err.stack || err))
436
        self.sbot.blobs.add(function (err, id) {
437
          if (err) return cb(new Error(err.stack || err))
438
          ids[name] = id
439
          next(i+1)
440
        })(pull.once(data))
441
      })
442
    })(0)
443
  })
444
}
445
446
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
447
  var self = this
448
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
449
450
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
451
    if (err) return cb(new Error(err.stack || err))
452
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
453
    if (pkgs._hasNonBlobUrl) {
454
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
455
    }
456
457
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
458
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
459
460
    var waiting = 2
461
462
    self.sbot.blobs.add(function (err, id) {
463
      if (err) return next(new Error(err.stack || err))
464
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
465
      var versions = pkg.versions || (pkg.versions = {})
466
      pkg.versions[sbotPkgLock.version] = {
467
        name: sbotPkgLock.name,
468
        version: sbotPkgLock.version,
469
        dist: self.blobDist(id)
470
      }
471
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
472
      distTags.latest = sbotPkgLock.version
473
      next()
474
    })(self.sbot.bootstrap.pack())
475
476
    var prebuilds
477
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
478
      if (err) return next(err)
479
      prebuilds = _prebuilds
480
      next()
481
    })
482
483
    function next(err) {
484
      if (err) return waiting = 0, cb(err)
485
      if (--waiting) return
486
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
487
        encoding: 'utf8'
488
      }, function (err, bootstrapScript) {
489
        if (err) return cb(err)
490
        var script = bootstrapScript + '\n' +
491
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
492
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
493
494
        self.sbot.blobs.add(function (err, id) {
495
          if (err) return cb(new Error(err.stack || err))
496
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
497
          if (!m) return cb(new Error('bad blob id: ' + id))
498
          cb(null, {
499
            name: sbotPkgLock.name,
500
            blob: id,
501
            hashType: m[2],
502
            hashBuf: Buffer.from(m[1], 'base64'),
503
          })
504
        })(pull.once(script))
505
      })
506
    }
507
  })
508
}
509
510
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
511
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
512
  else this.sbot.get(id, function (err, value) {
513
    if (err) return cb(err)
514
    cb(null, {key: id, value: value})
515
  })
516
}
517
518
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
519
  var self = this
520
  var stack = heads.slice()
521
  var seen = {}
522
  return function (abort, cb) {
523
    if (abort) return cb(abort)
524
    for (var id; stack.length && seen[id = stack.pop()];);
525
    if (!id) return cb(true)
526
    seen[id] = true
527
    // TODO: use DFS
528
    self.getMsg(id, function (err, msg) {
529
      if (err) return cb(new Error(err.stack || err))
530
      var c = msg && msg.value && msg.value.content
531
      var links = c && c[prop]
532
      if (Array.isArray(links)) {
533
        stack.push.apply(stack, links)
534
      } else if (links) {
535
        stack.push(links)
536
      }
537
      cb(null, msg)
538
    })
539
  }
540
}
541
542
function Req(server, req, res) {
543
  this.server = server
544
  this.req = req
545
  this.res = res
546
  this.blobsToPush = []
547
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
548
}
549
550
Req.prototype.serve = function () {
551
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
552
  var pathname = this.req.url.replace(/\?.*/, '')
553
  var m
554
  if ((m = /^\/(%25.*sha256)(\/.*)$/.exec(pathname))) {
555
    try {
556
      this.headMsgId = decodeURIComponent(m[1])
557
    } catch(e) {
558
      return this.respondError(400, e.stack || e)
559
    }
560
    pathname = m[2]
561
  }
562
  if (pathname === '/') return this.serveHome()
563
  if (pathname === '/bootstrap') return this.serveBootstrap()
564
  if (pathname === '/-/whoami') return this.serveWhoami()
565
  if (pathname === '/-/ping') return this.respond(200, true)
566
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
567
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
568
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
569
  return this.respond(404)
570
}
571
572
Req.prototype.respond = function (status, message) {
573
  this.res.writeHead(status, {'content-type': 'application/json'})
574
  this.res.end(message && JSON.stringify(message, 0, 2))
575
}
576
577
Req.prototype.respondError = function (status, message) {
578
  this.respond(status, {error: message})
579
}
580
581
var bootstrapName = 'ssb-npm-bootstrap'
582
583
Req.prototype.serveHome = function () {
584
  var self = this
585
  self.res.writeHead(200, {'content-type': 'text/html'})
586
  var port = 8044
587
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
588
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
589
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
590
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
591
    '</body></html>')
592
}
593
594
Req.prototype.serveBootstrap = function () {
595
  var self = this
596
  self.server.getBootstrapInfo(function (err, info) {
597
    if (err) return self.respondError(err.stack || err)
598
    var pkgNameText = info.name
599
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
600
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
601
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
602
    var blobsHostname = httpHost + ':' + self.server.wsPort
603
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
604
    var pkgHashText = info.hashBuf.toString('hex')
605
    var hashCmd = info.hashType + 'sum'
606
607
    var script =
608
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
609
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
610
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
611
        'npm install -g ' + info.name + ' &&\n' +
612
      'sbot server'
613
614
    self.res.writeHead(200, {'content-type': 'text/plain'})
615
    self.res.end(script)
616
  })
617
}
618
619
Req.prototype.serveWhoami = function () {
620
  var self = this
621
  self.server.sbot.whoami(function (err, feed) {
622
    if (err) return self.respondError(err.stack || err)
623
    self.respond(200, {username: feed.id})
624
  })
625
}
626
627
Req.prototype.serveUser1 = function () {
628
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
629
}
630
631
function decodeName(name) {
632
  var parts = name.replace(/\.tgz$/, '').split(':')
633
  return {
634
    name: parts[1],
635
    version: parts[2],
636
    distTag: parts[3],
637
  }
638
}
639
640
Req.prototype.getMentions = function (name) {
641
  var serverMentions = this.server.getMentions(name)
642
  var msgMentions = this.headMsgId ? pull(
643
    this.server.streamTree([this.headMsgId], 'dependencyBranch'),
644
    // decryption could be done here
645
    pull.map(function (msg) {
646
      var c = msg.value && msg.value.content
647
      if (!c.mentions || !Array.isArray(c.mentions)) return []
648
      return c.mentions.map(function (mention) {
649
        return {
650
          name: mention.name,
651
          size: mention.size,
652
          link: mention.link,
653
          author: msg.value.author,
654
          ts: msg.value.timestamp,
655
        }
656
      })
657
    }),
658
    pull.flatten(),
659
    pull.filter(typeof name === 'string' ? function (link) {
660
      return link.name === name
661
    } : name && name.$prefix ? function (link) {
662
      return link.name.substr(0, name.$prefix.length) === name.$prefix
663
    } : function () {
664
      throw new TypeError('unsupported name filter')
665
    }),
666
  ) : pull.empty()
667
  return cat([msgMentions, serverMentions])
668
}
669
670
Req.prototype.getMentionLinks = function (blobId) {
671
  return pull(
672
    this.headMsgId
673
      ? this.server.streamTree([this.headMsgId], 'dependencyBranch')
674
      : this.server.sbot.links({
675
        dest: blobId,
676
        rel: 'mentions',
677
        values: true,
678
      }),
679
    // decryption could be done here
680
    pull.map(function (msg) {
681
      var c = msg.value && msg.value.content
682
      return c && Array.isArray(c.mentions) || []
683
    }),
684
    pull.flatten(),
685
    pull.filter(function (link) {
686
      return link && link.link === blobId
687
    })
688
  )
689
}
690
691
Req.prototype.servePkg = function (pathname) {
692
  var self = this
693
  var parts = pathname.split('/')
694
  var pkgName = parts.shift().replace(/%2f/i, '/')
695
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
696
  var spec = parts.shift()
697
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
698
  if (parts.length > 0) return this.respondError(404)
699
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
700
  var obj = {
701
    _id: pkgName,
702
    name: pkgName,
703
    'dist-tags': {},
704
    versions: {}
705
  }
706
  var distTags = {/* <tag>: {version, ts}*/}
707
  pull(
708
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
709
    pull.drain(function (mention) {
710
      var data = decodeName(mention.name)
711
      if (!data.version) return
712
      if (data.distTag) {
713
        var tag = distTags[data.distTag]
714
        if (!tag || mention.ts > tag.ts) {
715
          /* TODO: sort by causal order (versionBranch links) instead of just
716
          * by timestamps */
717
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
718
        }
719
      }
720
      obj.versions[data.version] = {
721
        author: {
722
          url: mention.author
723
        },
724
        name: pkgName,
725
        version: data.version,
726
        dist: self.server.blobDist(mention.link)
727
      }
728
    }, function (err) {
729
      if (err) return self.respondError(500, err.stack || err)
730
      for (var tag in distTags) {
731
        obj['dist-tags'][tag] = distTags[tag].version
732
      }
733
      if (spec) resolveSpec()
734
      else if (self.fetchAll) resolveAll()
735
      else done()
736
    })
737
  )
738
  function resolveSpec() {
739
    var version = obj['dist-tags'][spec]
740
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
741
    obj = obj.versions[version]
742
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
743
    self.populatePackageJson(obj, function (err, pkg) {
744
      if (err) return self.respondError(500, err.stack || err)
745
      obj = pkg || obj
746
      done()
747
    })
748
  }
749
  function resolveAll() {
750
    var waiting = 0
751
    for (var version in obj.versions) (function (version) {
752
      waiting++
753
      self.populatePackageJson(obj.versions[version], function (err, pkg) {
754
        if (err && waiting <= 0) return console.trace(err)
755
        if (err) return waiting = 0, self.respondError(500, err.stack || err)
756
        if (pkg) obj.versions[version] = pkg
757
        if (!--waiting) done()
758
      })
759
    }(version))
760
    if (!waiting) done()
761
  }
762
  function done() {
763
    self.respond(200, obj)
764
  }
765
}
766
767
Req.prototype.servePrebuild = function (name) {
768
  var self = this
769
  var getMention = self.getMentions('prebuild:' + name)
770
  var blobsByAuthor = {/* <author>: BlobId */}
771
  getMention(null, function next(err, link) {
772
    if (err === true) return done()
773
    if (err) return self.respondError(500, err.stack || err)
774
    blobsByAuthor[link.author] = link.link
775
    getMention(null, next)
776
  })
777
  function done() {
778
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
779
    var blobId
780
    for (var feed in blobsByAuthor) {
781
      var blob = blobId = blobsByAuthor[feed]
782
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
783
      feeds.push(feed)
784
    }
785
    switch (Object.keys(authorsByLink).length) {
786
      case 0:
787
        return self.respondError(404, 'Not Found')
788
      case 1:
789
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
790
        return self.res.end()
791
      default:
792
        return self.respond(300, {choices: authorsByLink})
793
    }
794
  }
795
}
796
797
var localhosts = {
798
  '::1': true,
799
  '127.0.0.1': true,
800
  '::ffff:127.0.0.1': true,
801
}
802
803
Req.prototype.publishPkg = function (pkgName) {
804
  var self = this
805
  var remoteAddress = self.req.socket.remoteAddress
806
  if (!(remoteAddress in localhosts)) {
807
    return self.respondError(403, 'You may not publish as this user.')
808
  }
809
810
  var chunks = []
811
  self.req.on('data', function (data) {
812
    chunks.push(data)
813
  })
814
  self.req.on('end', function () {
815
    var data
816
    try {
817
      data = JSON.parse(Buffer.concat(chunks))
818
    } catch(e) {
819
      return self.respondError(400, e.stack)
820
    }
821
    return self.publishPkg2(pkgName, data || {})
822
  })
823
}
824
825
Req.prototype.publishPkg2 = function (name, data) {
826
  var self = this
827
  if (data.users) console.trace('[npm-registry] users property is not supported')
828
  var attachments = data._attachments || {}
829
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
830
  var waiting = 0
831
  Object.keys(attachments).forEach(function (filename) {
832
    waiting++
833
    var tarball = new Buffer(attachments[filename].data, 'base64')
834
    var length = attachments[filename].length
835
    if (length && length !== tarball.length) return self.respondError(400,
836
      'Length mismatch for attachment \'' + filename + '\'')
837
    self.server.sbot.blobs.add(function (err, id) {
838
      if (err) return self.respondError(500,
839
        'Adding attachment \'' + filename + '\' as blob failed')
840
      self.blobsToPush.push(id)
841
      links[filename] = {link: id, size: tarball.length}
842
      if (!--waiting) next()
843
    })(pull.once(tarball))
844
  })
845
  function next() {
846
    try {
847
      self.publishPkg3(name, data, links)
848
    } catch(e) {
849
      self.respondError(500, e.stack || e)
850
    }
851
  }
852
}
853
854
Req.prototype.publishPkg3 = function (name, data, links) {
855
  var self = this
856
  var versions = data.versions || {}
857
  var linksByVersion = {/* <version>: link */}
858
859
  // associate tarball blobs with versions
860
  for (var version in versions) {
861
    var pkg = versions[version]
862
    if (!pkg) return self.respondError(400, 'Bad package object')
863
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
864
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
865
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
866
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
867
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
868
    var filename = m[1]
869
    var link = links[filename]
870
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
871
    // TODO?: try to find missing tarball mentioned in other messages
872
    if (pkg.version && pkg.version !== version)
873
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
874
    linksByVersion[version] = link
875
    link.version = version
876
    link.dependencies = pkg.dependencies || {}
877
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
878
  }
879
880
  // associate blobs with dist-tags
881
  var tags = data['dist-tags'] || {}
882
  for (var tag in tags) {
883
    var version = tags[tag]
884
    var link = linksByVersion[version]
885
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
886
    // TODO?: support setting dist-tag without version,
887
    // by looking up a tarball blob for the version
888
    link.tag = tag
889
  }
890
891
  // compute blob links to publish
892
  var mentions = []
893
  for (var filename in links) {
894
    var link = links[filename] || {}
895
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
896
    mentions.push({
897
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
898
      link: link.link,
899
      size: link.size,
900
      dependencies: link.dependencies,
901
      bundledDependencies: link.bundledDependencies,
902
    })
903
  }
904
  return self.publishPkgs(mentions)
905
}
906
907
Req.prototype.publishPkgs = function (mentions) {
908
  var self = this
909
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
910
    if (err) self.respondError(500, err.stack || err)
911
    self.server.pushBlobs(self.blobsToPush, function (err) {
912
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
913
      self.respond(201)
914
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
915
    })
916
  })
917
}
918
919
Req.prototype.populatePackageJson = function (obj, cb) {
920
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
921
  var self = this
922
  var deps, bundledDeps
923
924
  // look for dependencies in links.
925
  // then fallback to getting it from the tarball blob
926
927
  pull(
928
    self.getMentionLinks(blobId),
929
    pull.drain(function (link) {
930
      if (link.dependencies) deps = link.dependencies
931
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
932
      // how to handle multiple assignments of dependencies to a package?
933
    }, function (err) {
934
      if (err) return cb(new Error(err.stack || err))
935
      if (deps) {
936
        // assume that the dependencies in the links to the blob are
937
        // correct.
938
        obj.dependencies = deps
939
        obj.bundledDependencies = bundledDeps
940
        cb(null, obj)
941
      } else {
942
        // get dependencies from the tarball
943
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
944
          if (err) return cb(err)
945
          pkg.dist = obj.dist
946
          pkg.dist.shasum = pkg._shasum
947
          pkg.author = pkg.author || obj.author
948
          pkg.version = pkg.version || obj.version
949
          pkg.name = pkg.name || obj.name
950
          cb(null, pkg)
951
        })
952
      }
953
    })
954
  )
955
}
956
957
function getPackageJsonFromTarballBlob(sbot, id, cb) {
958
  var self = this
959
  getBlob(sbot, id, function (err, readBlob) {
960
    if (err) return cb(err)
961
    cb = once(cb)
962
    var extract = tar.extract()
963
    var pkg, shasum
964
    extract.on('entry', function (header, stream, next) {
965
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
966
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
967
          if (err) return cb(err)
968
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
969
          catch(e) { return cb(e) }
970
          next()
971
        }))
972
      } else {
973
        stream.on('end', next)
974
        stream.resume()
975
      }
976
    })
977
    extract.on('finish', function () {
978
      pkg._shasum = shasum
979
      cb(null, pkg)
980
    })
981
    pull(
982
      readBlob,
983
      hash('sha1', 'hex', function (err, sum) {
984
        if (err) return cb(err)
985
        shasum = sum
986
      }),
987
      toPull(zlib.createGunzip()),
988
      toPull(extract)
989
    )
990
  })
991
}
992