git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function onceify(fn, self) {
25
  var cbs = [], err, data
26
  return function (cb) {
27
    if (fn) {
28
      cbs.push(cb)
29
      fn.call(self, function (_err, _data) {
30
        err = _err, data = _data
31
        var _cbs = cbs
32
        cbs = null
33
        while (_cbs.length) _cbs.shift()(err, data)
34
      })
35
      fn = null
36
    } else if (cbs) {
37
      cbs.push(cb)
38
    } else {
39
      cb(err, data)
40
    }
41
  }
42
}
43
44
function once(cb) {
45
  var done
46
  return function (err, result) {
47
    if (done) {
48
      if (err) console.trace(err)
49
    } else {
50
      done = true
51
      cb(err, result)
52
    }
53
  }
54
}
55
56
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57
  // convert a package-lock.json file into data for serving as an npm registry
58
  var hasNonBlobUrl = false
59
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60
  var pkgs = {}
61
  var queue = [pkgLock, pkgLock.name]
62
  while (queue.length) {
63
    var dep = queue.shift(), name = queue.shift()
64
    if (name) {
65
      var pkg = pkgs[name] || (pkgs[name] = {
66
        _id: name,
67
        name: name,
68
        versions: {}
69
      })
70
      if (dep.version && dep.integrity && dep.resolved) {
71
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72
        pkg.versions[dep.version] = {
73
          name: name,
74
          version: dep.version,
75
          dist: {
76
            integrity: dep.integrity,
77
            tarball: dep.resolved
78
          }
79
        }
80
      }
81
    }
82
    if (dep.dependencies) for (var depName in dep.dependencies) {
83
      queue.push(dep.dependencies[depName], depName)
84
    }
85
  }
86
  pkgs._hasNonBlobUrl = hasNonBlobUrl
87
  return pkgs
88
}
89
90
function npmLogin(registryAddress, cb) {
91
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
92
  var filename = path.join(os.homedir(), '.npmrc')
93
  fs.readFile(filename, 'utf8', function (err, data) {
94
    if (err && err.code === 'ENOENT') data = ''
95
    else if (err) return cb(new Error(err.stack))
96
    var lines = data ? data.split('\n') : []
97
    if (lines.indexOf(tokenLine) > -1) return cb()
98
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
99
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
100
    fs.appendFile(filename, line, cb)
101
  })
102
}
103
104
function formatHost(host) {
105
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
106
}
107
108
exports.name = 'npm-registry'
109
exports.version = '1.0.0'
110
exports.manifest = {
111
  getAddress: 'async'
112
}
113
exports.init = function (sbot, config) {
114
  var conf = config.npm || {}
115
  var port = conf.port || 8043
116
  var host = conf.host || null
117
  var autoAuth = conf.autoAuth !== false
118
119
  var server = http.createServer(exports.respond(sbot, config))
120
  var getAddress = onceify(function (cb) {
121
    server.on('error', cb)
122
    server.listen(port, host, function () {
123
      server.removeListener('error', cb)
124
      var regHost = formatHost(host || 'localhost')
125
      var regPort = this.address().port
126
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
127
      if (autoAuth) npmLogin(regUrl, next)
128
      else next()
129
      function next(err) {
130
        cb(err, regUrl)
131
      }
132
    })
133
    sbot.on('close', function () {
134
      server.close()
135
    })
136
  })
137
138
  getAddress(function (err, addr) {
139
    if (err) return console.error(err)
140
    console.log('[npm-registry] Listening on ' + addr)
141
  })
142
143
  return {
144
    getAddress: getAddress
145
  }
146
}
147
148
exports.respond = function (sbot, config) {
149
  var reg = new SsbNpmRegistryServer(sbot, config)
150
  return function (req, res) {
151
    new Req(reg, req, res).serve()
152
  }
153
}
154
155
function publishMsg(sbot, value, cb) {
156
  var gotExpectedPrevious = false
157
  sbot.publish(value, function next(err, msg) {
158
    if (err && /^expected previous:/.test(err.message)) {
159
      // retry once on this error
160
      if (gotExpectedPrevious) return cb(err)
161
      gotExpectedPrevious = true
162
      return sbot.publish(value, next)
163
    }
164
    cb(err, msg)
165
  })
166
}
167
168
function getDependencyBranches(sbot, id, cb) {
169
  // get ids of heads of tree of dependencyBranch message links to include all
170
  // the dependencies of the given tarball id.
171
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
172
  var msgs = {}
173
  var branched = {}
174
  var blobs = {}
175
176
  function addPkgById(id, cb) {
177
    if (blobs[id]) return cb()
178
    blobs[id] = true
179
    getPackageJsonCached(id, function (err, pkg) {
180
      if (err) return cb(err)
181
      var done = multicb()
182
      for (var name in pkg.dependencies || {}) {
183
        addPkgBySpec(name, pkg.dependencies[name], done())
184
      }
185
      done(cb)
186
    })
187
  }
188
189
  function addPkgBySpec(name, spec, cb) {
190
    var done = multicb()
191
    pull(
192
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
193
      pull.map(function (mention) {
194
        addPkgById(mention.link, done())
195
        return packageLinks(sbot, mention.author, mention.link, name, spec)
196
      }),
197
      pull.flatten(),
198
      pull.drain(function (msg) {
199
        var c = msg && msg.value && msg.value.content
200
        if (!c) return
201
        msgs[msg.key] = msg.value
202
        if (Array.isArray(c.dependencyBranch)) {
203
          for (var k = 0; k < c.dependencyBranch.length; k++) {
204
            branched[c.dependencyBranch[k]] = true
205
          }
206
        }
207
      }, function (err) {
208
        if (err) return cb(err)
209
        done(cb)
210
      })
211
    )
212
  }
213
214
  addPkgById(id, function (err) {
215
    if (err) return cb(err)
216
    var ids = []
217
    for (var key in msgs) {
218
      if (!branched[key]) ids.push(key)
219
    }
220
    cb(null, ids)
221
  })
222
}
223
224
function packageLinks(sbot, feed, id, name, spec) {
225
  function matches(mention) {
226
    var data = mention
227
      && mention.link === id
228
      && decodeName(mention.name)
229
    return data
230
      && data.name === name
231
      && (spec ? semver.satisfies(data.version, spec) : true)
232
  }
233
  return pull(
234
    sbot.links({
235
      source: feed,
236
      dest: id,
237
      rel: 'mentions',
238
      values: true,
239
    }),
240
    pull.filter(function (msg) {
241
      var c = msg && msg.value && msg.value.content
242
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
243
    })
244
  )
245
}
246
247
function getVersionBranches(sbot, link, cb) {
248
  var data = decodeName(link.name)
249
  var msgs = {}, branched = {}
250
  pull(
251
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
252
    pull.map(function (mention) {
253
      return packageLinks(sbot, mention.author, mention.link, data.name)
254
    }),
255
    pull.flatten(),
256
    pull.drain(function (msg) {
257
      var c = msg && msg.value && msg.value.content
258
      if (!c) return
259
      msgs[msg.key] = msg.value
260
      if (Array.isArray(c.versionBranch)) {
261
        for (var k = 0; k < c.versionBranch.length; k++) {
262
          branched[c.versionBranch[k]] = true
263
        }
264
      }
265
    }, function (err) {
266
      if (err) return cb(err)
267
      var ids = []
268
      for (var key in msgs) {
269
        if (!branched[key]) ids.push(key)
270
      }
271
      cb(null, ids)
272
    })
273
  )
274
}
275
276
// For each dependency that is not a bundledDependency, get message ids for
277
// that dependency name + version.
278
function publishSingleMention(sbot, mention, cb) {
279
  // Calculate dependencyBranch and versionBranch message ids.
280
  var value = {
281
    type: 'npm-packages',
282
    mentions: [mention]
283
  }
284
  var done = multicb({pluck: 1, spread: true})
285
  getDependencyBranches(sbot, mention.link, done())
286
  getVersionBranches(sbot, mention, done())
287
  done(function (err, dependencyBranches, versionBranches) {
288
    if (err) return cb(err)
289
    value.dependencyBranch = dependencyBranches || undefined
290
    value.versionBranch = versionBranches || undefined
291
    publishMsg(sbot, value, cb)
292
  })
293
}
294
295
function publishMentions(sbot, mentions, cb) {
296
  // console.error("publishing %s mentions", mentions.length)
297
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
298
  // if it is just one mention, fetch and add useful metadata
299
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
300
  publishMsg(sbot, {
301
    type: 'npm-packages',
302
    mentions: mentions,
303
  }, cb)
304
}
305
306
exports.publishPkgMentions = function (sbot, mentions, cb) {
307
  // try to fit the mentions into as few messages as possible,
308
  // while fitting under the message size limit.
309
  var msgs = []
310
  ;(function next(i, chunks) {
311
    if (i >= mentions.length) return cb(null, msgs)
312
    var chunkLen = Math.ceil(mentions.length / chunks)
313
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
314
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
315
      if (err && msgs.length) return onPartialPublish(err)
316
      if (err) return cb(err)
317
      msgs.push(msg)
318
      next(i + chunkLen, chunks)
319
    })
320
  })(0, 1)
321
  function onPartialPublish(err) {
322
    var remaining = mentions.length - i
323
    return cb(new Error('Published messages ' +
324
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
325
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
326
  }
327
}
328
329
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
330
  cb = once(cb)
331
  var waiting = 0
332
  var expandedMentions = mentions.map(function (link) {
333
    var id = link && link.link
334
    if (!id) return link
335
    waiting++
336
    var newLink = {}
337
    for (var k in link) newLink[k] = link[k]
338
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
339
      if (err) return cb(err)
340
      for (var k in props) newLink[k] = pkg[k]
341
      if (!--waiting) next()
342
    })
343
    return newLink
344
  })
345
  if (!waiting) next()
346
  function next() {
347
    cb(null, expandedMentions)
348
  }
349
}
350
351
function SsbNpmRegistryServer(sbot, config) {
352
  this.sbot = sbot
353
  this.config = config
354
  this.npmConfig = config.npm || {}
355
  this.host = this.npmConfig.host || 'localhost'
356
  this.fetchAll = this.npmConfig.fetchAll
357
  this.links2 = sbot.links2
358
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
359
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
360
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
361
    + this.wsPort + '/blobs/get/'
362
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
363
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
364
}
365
366
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
367
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
368
369
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
370
  var self = this
371
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
372
  ;(function next(i) {
373
    if (i >= ids.length) return cb()
374
    self.sbot.blobs.push(ids[i], function (err) {
375
      if (err) return cb(err)
376
      next(i+1)
377
    })
378
  })(0)
379
}
380
381
function getBlob(sbot, id, cb) {
382
  var blobs = sbot.blobs
383
  blobs.size(id, function (err, size) {
384
    if (typeof size === 'number') cb(null, blobs.get(id))
385
    else blobs.want(id, function (err, got) {
386
      if (err) cb(err)
387
      else if (!got) cb('missing blob ' + id)
388
      else cb(null, blobs.get(id))
389
    })
390
  })
391
}
392
393
SsbNpmRegistryServer.prototype.blobDist = function (id) {
394
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
395
  if (!m) throw new Error('bad blob id: ' + id)
396
  return {
397
    integrity: m[2] + '-' + m[1],
398
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
399
  }
400
}
401
402
function getMentions(links2, name) {
403
  return links2.read({
404
    query: [
405
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
406
      {$filter: {dest: {$prefix: '&'}}},
407
      {$map: {
408
        name: ['rel', 1],
409
        size: ['rel', 2],
410
        link: 'dest',
411
        author: 'source',
412
        ts: 'ts'
413
      }}
414
    ]
415
  })
416
}
417
418
SsbNpmRegistryServer.prototype.getMentions = function (name) {
419
  return getMentions(this.links2, name)
420
}
421
422
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
423
  var self = this
424
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
425
  var ids = {}
426
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
427
  fs.readdir(prebuildsDir, function (err, filenames) {
428
    if (err) return cb(new Error(err.stack || err))
429
    ;(function next(i) {
430
      if (i >= filenames.length) return cb(null, ids)
431
      var m = nameRegex.exec(filenames[i])
432
      if (!m) return next(i+1)
433
      var name = m[1]
434
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
435
        if (err) return cb(new Error(err.stack || err))
436
        self.sbot.blobs.add(function (err, id) {
437
          if (err) return cb(new Error(err.stack || err))
438
          ids[name] = id
439
          next(i+1)
440
        })(pull.once(data))
441
      })
442
    })(0)
443
  })
444
}
445
446
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
447
  var self = this
448
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
449
450
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
451
    if (err) return cb(new Error(err.stack || err))
452
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
453
    if (pkgs._hasNonBlobUrl) {
454
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
455
    }
456
457
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
458
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
459
460
    var waiting = 2
461
462
    self.sbot.blobs.add(function (err, id) {
463
      if (err) return next(new Error(err.stack || err))
464
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
465
      var versions = pkg.versions || (pkg.versions = {})
466
      pkg.versions[sbotPkgLock.version] = {
467
        name: sbotPkgLock.name,
468
        version: sbotPkgLock.version,
469
        dist: self.blobDist(id)
470
      }
471
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
472
      distTags.latest = sbotPkgLock.version
473
      next()
474
    })(self.sbot.bootstrap.pack())
475
476
    var prebuilds
477
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
478
      if (err) return next(err)
479
      prebuilds = _prebuilds
480
      next()
481
    })
482
483
    function next(err) {
484
      if (err) return waiting = 0, cb(err)
485
      if (--waiting) return
486
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
487
        encoding: 'utf8'
488
      }, function (err, bootstrapScript) {
489
        if (err) return cb(err)
490
        var script = bootstrapScript + '\n' +
491
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
492
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
493
494
        self.sbot.blobs.add(function (err, id) {
495
          if (err) return cb(new Error(err.stack || err))
496
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
497
          if (!m) return cb(new Error('bad blob id: ' + id))
498
          cb(null, {
499
            name: sbotPkgLock.name,
500
            blob: id,
501
            hashType: m[2],
502
            hashBuf: Buffer.from(m[1], 'base64'),
503
          })
504
        })(pull.once(script))
505
      })
506
    }
507
  })
508
}
509
510
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
511
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
512
  else this.sbot.get(id, function (err, value) {
513
    if (err) return cb(err)
514
    cb(null, {key: id, value: value})
515
  })
516
}
517
518
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
519
  var self = this
520
  var stack = heads.slice()
521
  var seen = {}
522
  return function (abort, cb) {
523
    if (abort) return cb(abort)
524
    for (var id; stack.length && seen[id = stack.pop()];);
525
    if (!id) return cb(true)
526
    seen[id] = true
527
    // TODO: use DFS
528
    self.getMsg(id, function (err, msg) {
529
      if (err) return cb(new Error(err.stack || err))
530
      var c = msg && msg.value && msg.value.content
531
      var links = c && c[prop]
532
      if (Array.isArray(links)) {
533
        stack.push.apply(stack, links)
534
      } else if (links) {
535
        stack.push(links)
536
      }
537
      cb(null, msg)
538
    })
539
  }
540
}
541
542
function Req(server, req, res) {
543
  this.server = server
544
  this.req = req
545
  this.res = res
546
  this.blobsToPush = []
547
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
548
}
549
550
Req.prototype.serve = function () {
551
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
552
  var pathname = this.req.url.replace(/\?.*/, '')
553
  var m
554
  if ((m = /^\/(%25.*sha256)(\/.*)$/.exec(pathname))) {
555
    try {
556
      this.headMsgId = decodeURIComponent(m[1])
557
    } catch(e) {
558
      return this.respondError(400, e.stack || e)
559
    }
560
    pathname = m[2]
561
  }
562
  if (pathname === '/') return this.serveHome()
563
  if (pathname === '/bootstrap') return this.serveBootstrap()
564
  if (pathname === '/-/whoami') return this.serveWhoami()
565
  if (pathname === '/-/ping') return this.respond(200, true)
566
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
567
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
568
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
569
  return this.respond(404)
570
}
571
572
Req.prototype.respond = function (status, message) {
573
  this.res.writeHead(status, {'content-type': 'application/json'})
574
  this.res.end(message && JSON.stringify(message, 0, 2))
575
}
576
577
Req.prototype.respondError = function (status, message) {
578
  this.respond(status, {error: message})
579
}
580
581
var bootstrapName = 'ssb-npm-bootstrap'
582
583
Req.prototype.serveHome = function () {
584
  var self = this
585
  self.res.writeHead(200, {'content-type': 'text/html'})
586
  var port = 8044
587
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
588
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
589
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
590
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
591
    '</body></html>')
592
}
593
594
Req.prototype.serveBootstrap = function () {
595
  var self = this
596
  self.server.getBootstrapInfo(function (err, info) {
597
    if (err) return self.respondError(err.stack || err)
598
    var pkgNameText = info.name
599
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
600
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
601
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
602
    var blobsHostname = httpHost + ':' + self.server.wsPort
603
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
604
    var pkgHashText = info.hashBuf.toString('hex')
605
    var hashCmd = info.hashType + 'sum'
606
607
    var script =
608
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
609
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
610
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
611
        'npm install -g ' + info.name + ' &&\n' +
612
      'sbot server'
613
614
    self.res.writeHead(200, {'content-type': 'text/plain'})
615
    self.res.end(script)
616
  })
617
}
618
619
Req.prototype.serveWhoami = function () {
620
  var self = this
621
  self.server.sbot.whoami(function (err, feed) {
622
    if (err) return self.respondError(err.stack || err)
623
    self.respond(200, {username: feed.id})
624
  })
625
}
626
627
Req.prototype.serveUser1 = function () {
628
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
629
}
630
631
function decodeName(name) {
632
  var parts = name.replace(/\.tgz$/, '').split(':')
633
  return {
634
    name: parts[1],
635
    version: parts[2],
636
    distTag: parts[3],
637
  }
638
}
639
640
Req.prototype.getMentions = function (name) {
641
  var serverMentions = this.server.getMentions(name)
642
  var msgMentions = this.headMsgId ? pull(
643
    this.server.streamTree([this.headMsgId], 'dependencyBranch'),
644
    // decryption could be done here
645
    pull.map(function (msg) {
646
      var c = msg.value && msg.value.content
647
      if (!c.mentions || !Array.isArray(c.mentions)) return []
648
      return c.mentions.map(function (mention) {
649
        return {
650
          name: mention.name,
651
          size: mention.size,
652
          link: mention.link,
653
          author: msg.value.author,
654
          ts: msg.value.timestamp,
655
        }
656
      })
657
    }),
658
    pull.flatten(),
659
    pull.filter(typeof name === 'string' ? function (link) {
660
      return link.name === name
661
    } : name && name.$prefix ? function (link) {
662
      return link.name.substr(0, name.$prefix.length) === name.$prefix
663
    } : function () {
664
      throw new TypeError('unsupported name filter')
665
    }),
666
  ) : pull.empty()
667
  return cat([msgMentions, serverMentions])
668
}
669
670
Req.prototype.getMentionLinks = function (blobId) {
671
  return pull(
672
    this.headMsgId
673
      ? this.server.streamTree([this.headMsgId], 'dependencyBranch')
674
      : this.server.sbot.links({
675
        dest: blobId,
676
        rel: 'mentions',
677
        values: true,
678
      }),
679
    // decryption could be done here
680
    pull.map(function (msg) {
681
      var c = msg.value && msg.value.content
682
      return c && Array.isArray(c.mentions) || []
683
    }),
684
    pull.flatten(),
685
    pull.filter(function (link) {
686
      return link && link.link === blobId
687
    })
688
  )
689
}
690
691
Req.prototype.servePkg = function (pathname) {
692
  var self = this
693
  var parts = pathname.split('/')
694
  var pkgName = parts.shift().replace(/%2f/i, '/')
695
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
696
  var spec = parts.shift()
697
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
698
  if (parts.length > 0) return this.respondError(404)
699
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
700
  var obj = {
701
    _id: pkgName,
702
    name: pkgName,
703
    'dist-tags': {},
704
    versions: {}
705
  }
706
  var distTags = {/* <tag>: {version, ts}*/}
707
  pull(
708
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
709
    pull.drain(function (mention) {
710
      var data = decodeName(mention.name)
711
      if (!data.version) return
712
      if (data.distTag) {
713
        var tag = distTags[data.distTag]
714
        if (!tag || mention.ts > tag.ts) {
715
          /* TODO: sort by causal order (versionBranch links) instead of just
716
          * by timestamps */
717
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
718
        }
719
      }
720
      obj.versions[data.version] = {
721
        author: {
722
          url: mention.author
723
        },
724
        name: pkgName,
725
        version: data.version,
726
        dist: self.server.blobDist(mention.link)
727
      }
728
    }, function (err) {
729
      if (err) return self.respondError(500, err.stack || err)
730
      for (var tag in distTags) {
731
        obj['dist-tags'][tag] = distTags[tag].version
732
      }
733
      if (spec) resolveSpec()
734
      else if (self.fetchAll) resolveAll()
735
      else done()
736
    })
737
  )
738
  function resolveSpec() {
739
    var version = obj['dist-tags'][spec]
740
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
741
    obj = obj.versions[version]
742
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
743
    self.populatePackageJson(obj, function (err, pkg) {
744
      if (err) return self.respondError(500, err.stack || err)
745
      obj = pkg || obj
746
      done()
747
    })
748
  }
749
  function resolveVersion(version, cb) {
750
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
751
      if (err) return cb(err)
752
      if (pkg) obj.versions[version] = pkg
753
      cb()
754
    })
755
  }
756
  function resolveAll() {
757
    var done = multicb()
758
    for (var version in obj.versions) {
759
      resolveVersion(version, done())
760
    }
761
    done(resolved)
762
  }
763
  function resolved(err) {
764
    if (err) return self.respondError(500, err.stack || err)
765
    self.respond(200, obj)
766
  }
767
}
768
769
Req.prototype.servePrebuild = function (name) {
770
  var self = this
771
  var getMention = self.getMentions('prebuild:' + name)
772
  var blobsByAuthor = {/* <author>: BlobId */}
773
  getMention(null, function next(err, link) {
774
    if (err === true) return done()
775
    if (err) return self.respondError(500, err.stack || err)
776
    blobsByAuthor[link.author] = link.link
777
    getMention(null, next)
778
  })
779
  function done() {
780
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
781
    var blobId
782
    for (var feed in blobsByAuthor) {
783
      var blob = blobId = blobsByAuthor[feed]
784
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
785
      feeds.push(feed)
786
    }
787
    switch (Object.keys(authorsByLink).length) {
788
      case 0:
789
        return self.respondError(404, 'Not Found')
790
      case 1:
791
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
792
        return self.res.end()
793
      default:
794
        return self.respond(300, {choices: authorsByLink})
795
    }
796
  }
797
}
798
799
var localhosts = {
800
  '::1': true,
801
  '127.0.0.1': true,
802
  '::ffff:127.0.0.1': true,
803
}
804
805
Req.prototype.publishPkg = function (pkgName) {
806
  var self = this
807
  var remoteAddress = self.req.socket.remoteAddress
808
  if (!(remoteAddress in localhosts)) {
809
    return self.respondError(403, 'You may not publish as this user.')
810
  }
811
812
  var chunks = []
813
  self.req.on('data', function (data) {
814
    chunks.push(data)
815
  })
816
  self.req.on('end', function () {
817
    var data
818
    try {
819
      data = JSON.parse(Buffer.concat(chunks))
820
    } catch(e) {
821
      return self.respondError(400, e.stack)
822
    }
823
    return self.publishPkg2(pkgName, data || {})
824
  })
825
}
826
827
Req.prototype.publishPkg2 = function (name, data) {
828
  var self = this
829
  if (data.users) console.trace('[npm-registry] users property is not supported')
830
  var attachments = data._attachments || {}
831
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
832
  var done = multicb()
833
  function addAttachmentAsBlob(filename, cb) {
834
    var tarball = new Buffer(attachments[filename].data, 'base64')
835
    var length = attachments[filename].length
836
    if (length && length !== tarball.length) return self.respondError(400,
837
      'Length mismatch for attachment \'' + filename + '\'')
838
    self.server.sbot.blobs.add(function (err, id) {
839
      if (err) return cb(err)
840
      self.blobsToPush.push(id)
841
      links[filename] = {link: id, size: tarball.length}
842
      cb()
843
    })(pull.once(tarball))
844
  }
845
  for (var filename in attachments) {
846
    addAttachmentAsBlob(filename, done())
847
  }
848
  done(function (err) {
849
    if (err) return self.respondError(500, err.stack || err)
850
    try {
851
      self.publishPkg3(name, data, links)
852
    } catch(e) {
853
      self.respondError(500, e.stack || e)
854
    }
855
  })
856
}
857
858
Req.prototype.publishPkg3 = function (name, data, links) {
859
  var self = this
860
  var versions = data.versions || {}
861
  var linksByVersion = {/* <version>: link */}
862
863
  // associate tarball blobs with versions
864
  for (var version in versions) {
865
    var pkg = versions[version]
866
    if (!pkg) return self.respondError(400, 'Bad package object')
867
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
868
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
869
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
870
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
871
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
872
    var filename = m[1]
873
    var link = links[filename]
874
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
875
    // TODO?: try to find missing tarball mentioned in other messages
876
    if (pkg.version && pkg.version !== version)
877
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
878
    linksByVersion[version] = link
879
    link.version = version
880
    link.dependencies = pkg.dependencies || {}
881
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
882
  }
883
884
  // associate blobs with dist-tags
885
  var tags = data['dist-tags'] || {}
886
  for (var tag in tags) {
887
    var version = tags[tag]
888
    var link = linksByVersion[version]
889
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
890
    // TODO?: support setting dist-tag without version,
891
    // by looking up a tarball blob for the version
892
    link.tag = tag
893
  }
894
895
  // compute blob links to publish
896
  var mentions = []
897
  for (var filename in links) {
898
    var link = links[filename] || {}
899
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
900
    mentions.push({
901
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
902
      link: link.link,
903
      size: link.size,
904
      dependencies: link.dependencies,
905
      bundledDependencies: link.bundledDependencies,
906
    })
907
  }
908
  return self.publishPkgs(mentions)
909
}
910
911
Req.prototype.publishPkgs = function (mentions) {
912
  var self = this
913
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
914
    if (err) self.respondError(500, err.stack || err)
915
    self.server.pushBlobs(self.blobsToPush, function (err) {
916
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
917
      self.respond(201)
918
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
919
    })
920
  })
921
}
922
923
Req.prototype.populatePackageJson = function (obj, cb) {
924
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
925
  var self = this
926
  var deps, bundledDeps
927
928
  // look for dependencies in links.
929
  // then fallback to getting it from the tarball blob
930
931
  pull(
932
    self.getMentionLinks(blobId),
933
    pull.drain(function (link) {
934
      if (link.dependencies) deps = link.dependencies
935
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
936
      // how to handle multiple assignments of dependencies to a package?
937
    }, function (err) {
938
      if (err) return cb(new Error(err.stack || err))
939
      if (deps) {
940
        // assume that the dependencies in the links to the blob are
941
        // correct.
942
        obj.dependencies = deps
943
        obj.bundledDependencies = bundledDeps
944
        cb(null, obj)
945
      } else {
946
        // get dependencies from the tarball
947
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
948
          if (err) return cb(err)
949
          pkg.dist = obj.dist
950
          pkg.dist.shasum = pkg._shasum
951
          pkg.author = pkg.author || obj.author
952
          pkg.version = pkg.version || obj.version
953
          pkg.name = pkg.name || obj.name
954
          cb(null, pkg)
955
        })
956
      }
957
    })
958
  )
959
}
960
961
function getPackageJsonFromTarballBlob(sbot, id, cb) {
962
  var self = this
963
  getBlob(sbot, id, function (err, readBlob) {
964
    if (err) return cb(err)
965
    cb = once(cb)
966
    var extract = tar.extract()
967
    var pkg, shasum
968
    extract.on('entry', function (header, stream, next) {
969
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
970
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
971
          if (err) return cb(err)
972
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
973
          catch(e) { return cb(e) }
974
          next()
975
        }))
976
      } else {
977
        stream.on('end', next)
978
        stream.resume()
979
      }
980
    })
981
    extract.on('finish', function () {
982
      pkg._shasum = shasum
983
      cb(null, pkg)
984
    })
985
    pull(
986
      readBlob,
987
      hash('sha1', 'hex', function (err, sum) {
988
        if (err) return cb(err)
989
        shasum = sum
990
      }),
991
      toPull(zlib.createGunzip()),
992
      toPull(extract)
993
    )
994
  })
995
}
996