git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var hash = require('pull-hash')
13
var multicb = require('multicb')
14
var memo = require('asyncmemo')
15
var lru = require('hashlru')
16
17
function escapeHTML(str) {
18
  return String(str)
19
    .replace(/</g, '&lt;')
20
    .replace(/>/g, '&gt;')
21
}
22
23
function onceify(fn, self) {
24
  var cbs = [], err, data
25
  return function (cb) {
26
    if (fn) {
27
      cbs.push(cb)
28
      fn.call(self, function (_err, _data) {
29
        err = _err, data = _data
30
        var _cbs = cbs
31
        cbs = null
32
        while (_cbs.length) _cbs.shift()(err, data)
33
      })
34
      fn = null
35
    } else if (cbs) {
36
      cbs.push(cb)
37
    } else {
38
      cb(err, data)
39
    }
40
  }
41
}
42
43
function once(cb) {
44
  var done
45
  return function (err, result) {
46
    if (done) {
47
      if (err) console.trace(err)
48
    } else {
49
      done = true
50
      cb(err, result)
51
    }
52
  }
53
}
54
55
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
56
  // convert a package-lock.json file into data for serving as an npm registry
57
  var hasNonBlobUrl = false
58
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
59
  var pkgs = {}
60
  var queue = [pkgLock, pkgLock.name]
61
  while (queue.length) {
62
    var dep = queue.shift(), name = queue.shift()
63
    if (name) {
64
      var pkg = pkgs[name] || (pkgs[name] = {
65
        _id: name,
66
        name: name,
67
        versions: {}
68
      })
69
      if (dep.version && dep.integrity && dep.resolved) {
70
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
71
        pkg.versions[dep.version] = {
72
          name: name,
73
          version: dep.version,
74
          dist: {
75
            integrity: dep.integrity,
76
            tarball: dep.resolved
77
          }
78
        }
79
      }
80
    }
81
    if (dep.dependencies) for (var depName in dep.dependencies) {
82
      queue.push(dep.dependencies[depName], depName)
83
    }
84
  }
85
  pkgs._hasNonBlobUrl = hasNonBlobUrl
86
  return pkgs
87
}
88
89
function npmLogin(registryAddress, cb) {
90
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
91
  var filename = path.join(os.homedir(), '.npmrc')
92
  fs.readFile(filename, 'utf8', function (err, data) {
93
    if (err && err.code === 'ENOENT') data = ''
94
    else if (err) return cb(new Error(err.stack))
95
    var lines = data ? data.split('\n') : []
96
    if (lines.indexOf(tokenLine) > -1) return cb()
97
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
98
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
99
    fs.appendFile(filename, line, cb)
100
  })
101
}
102
103
function formatHost(host) {
104
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
105
}
106
107
exports.name = 'npm-registry'
108
exports.version = '1.0.0'
109
exports.manifest = {
110
  getAddress: 'async'
111
}
112
exports.init = function (sbot, config) {
113
  var conf = config.npm || {}
114
  var port = conf.port || 8043
115
  var host = conf.host || null
116
  var autoAuth = conf.autoAuth !== false
117
118
  var server = http.createServer(exports.respond(sbot, config))
119
  var getAddress = onceify(function (cb) {
120
    server.on('error', cb)
121
    server.listen(port, host, function () {
122
      server.removeListener('error', cb)
123
      var regHost = formatHost(host || 'localhost')
124
      var regPort = this.address().port
125
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
126
      if (autoAuth) npmLogin(regUrl, next)
127
      else next()
128
      function next(err) {
129
        cb(err, regUrl)
130
      }
131
    })
132
    sbot.on('close', function () {
133
      server.close()
134
    })
135
  })
136
137
  getAddress(function (err, addr) {
138
    if (err) return console.error(err)
139
    console.log('[npm-registry] Listening on ' + addr)
140
  })
141
142
  return {
143
    getAddress: getAddress
144
  }
145
}
146
147
exports.respond = function (sbot, config) {
148
  var reg = new SsbNpmRegistryServer(sbot, config)
149
  return function (req, res) {
150
    new Req(reg, req, res).serve()
151
  }
152
}
153
154
function publishMsg(sbot, value, cb) {
155
  var gotExpectedPrevious = false
156
  sbot.publish(value, function next(err, msg) {
157
    if (err && /^expected previous:/.test(err.message)) {
158
      // retry once on this error
159
      if (gotExpectedPrevious) return cb(err)
160
      gotExpectedPrevious = true
161
      return sbot.publish(value, next)
162
    }
163
    cb(err, msg)
164
  })
165
}
166
167
function getDependencyBranches(sbot, id, cb) {
168
  // get ids of heads of tree of dependencyBranch message links to include all
169
  // the dependencies of the given tarball id.
170
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
171
  var msgs = {}
172
  var branched = {}
173
  var blobs = {}
174
175
  function addPkgById(id, cb) {
176
    if (blobs[id]) return cb()
177
    blobs[id] = true
178
    getPackageJsonCached(id, function (err, pkg) {
179
      if (err) return cb(err)
180
      var done = multicb()
181
      for (var name in pkg.dependencies || {}) {
182
        addPkgBySpec(name, pkg.dependencies[name], done())
183
      }
184
      done(cb)
185
    })
186
  }
187
188
  function addPkgBySpec(name, spec, cb) {
189
    var done = multicb()
190
    pull(
191
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
192
      pull.map(function (mention) {
193
        addPkgById(mention.link, done())
194
        return packageLinks(sbot, mention.author, mention.link, name, spec)
195
      }),
196
      pull.flatten(),
197
      pull.drain(function (msg) {
198
        var c = msg && msg.value && msg.value.content
199
        if (!c) return
200
        msgs[msg.key] = msg.value
201
        if (Array.isArray(c.dependencyBranch)) {
202
          for (var k = 0; k < c.dependencyBranch.length; k++) {
203
            branched[c.dependencyBranch[k]] = true
204
          }
205
        }
206
      }, function (err) {
207
        if (err) return cb(err)
208
        done(cb)
209
      })
210
    )
211
  }
212
213
  addPkgById(id, function (err) {
214
    if (err) return cb(err)
215
    var ids = []
216
    for (var key in msgs) {
217
      if (!branched[key]) ids.push(key)
218
    }
219
    cb(null, ids)
220
  })
221
}
222
223
function packageLinks(sbot, feed, id, name, spec) {
224
  function matches(mention) {
225
    var data = mention
226
      && mention.link === id
227
      && decodeName(mention.name)
228
    return data
229
      && data.name === name
230
      && (spec ? semver.satisfies(data.version, spec) : true)
231
  }
232
  return pull(
233
    sbot.links({
234
      source: feed,
235
      dest: id,
236
      rel: 'mentions',
237
      values: true,
238
    }),
239
    pull.filter(function (msg) {
240
      var c = msg && msg.value && msg.value.content
241
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
242
    })
243
  )
244
}
245
246
function getVersionBranches(sbot, link, cb) {
247
  var data = decodeName(link.name)
248
  var msgs = {}, branched = {}
249
  pull(
250
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
251
    pull.map(function (mention) {
252
      return packageLinks(sbot, mention.author, mention.link, data.name)
253
    }),
254
    pull.flatten(),
255
    pull.drain(function (msg) {
256
      var c = msg && msg.value && msg.value.content
257
      if (!c) return
258
      msgs[msg.key] = msg.value
259
      if (Array.isArray(c.versionBranch)) {
260
        for (var k = 0; k < c.versionBranch.length; k++) {
261
          branched[c.versionBranch[k]] = true
262
        }
263
      }
264
    }, function (err) {
265
      if (err) return cb(err)
266
      var ids = []
267
      for (var key in msgs) {
268
        if (!branched[key]) ids.push(key)
269
      }
270
      cb(null, ids)
271
    })
272
  )
273
}
274
275
// For each dependency that is not a bundledDependency, get message ids for
276
// that dependency name + version.
277
function publishSingleMention(sbot, mention, cb) {
278
  // Calculate dependencyBranch and versionBranch message ids.
279
  var value = {
280
    type: 'npm-packages',
281
    mentions: [mention]
282
  }
283
  var done = multicb({pluck: 1, spread: true})
284
  getDependencyBranches(sbot, mention.link, done())
285
  getVersionBranches(sbot, mention, done())
286
  done(function (err, dependencyBranches, versionBranches) {
287
    if (err) return cb(err)
288
    value.dependencyBranch = dependencyBranches || undefined
289
    value.versionBranch = versionBranches || undefined
290
    publishMsg(sbot, value, cb)
291
  })
292
}
293
294
function publishMentions(sbot, mentions, cb) {
295
  // console.error("publishing %s mentions", mentions.length)
296
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
297
  // if it is just one mention, fetch and add useful metadata
298
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
299
  publishMsg(sbot, {
300
    type: 'npm-packages',
301
    mentions: mentions,
302
  }, cb)
303
}
304
305
exports.publishPkgMentions = function (sbot, mentions, cb) {
306
  // try to fit the mentions into as few messages as possible,
307
  // while fitting under the message size limit.
308
  var msgs = []
309
  ;(function next(i, chunks) {
310
    if (i >= mentions.length) return cb(null, msgs)
311
    var chunkLen = Math.ceil(mentions.length / chunks)
312
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
313
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
314
      if (err && msgs.length) return onPartialPublish(err)
315
      if (err) return cb(err)
316
      msgs.push(msg)
317
      next(i + chunkLen, chunks)
318
    })
319
  })(0, 1)
320
  function onPartialPublish(err) {
321
    var remaining = mentions.length - i
322
    return cb(new Error('Published messages ' +
323
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
324
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
325
  }
326
}
327
328
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
329
  cb = once(cb)
330
  var waiting = 0
331
  var expandedMentions = mentions.map(function (link) {
332
    var id = link && link.link
333
    if (!id) return link
334
    waiting++
335
    var newLink = {}
336
    for (var k in link) newLink[k] = link[k]
337
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
338
      if (err) return cb(err)
339
      for (var k in props) newLink[k] = pkg[k]
340
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
341
      if (!--waiting) next()
342
    })
343
    return newLink
344
  })
345
  if (!waiting) next()
346
  function next() {
347
    cb(null, expandedMentions)
348
  }
349
}
350
351
function SsbNpmRegistryServer(sbot, config) {
352
  this.sbot = sbot
353
  this.config = config
354
  this.npmConfig = config.npm || {}
355
  this.host = this.npmConfig.host || 'localhost'
356
  this.fetchAll = this.npmConfig.fetchAll
357
  this.needShasum = this.npmConfig.needShasum
358
  this.links2 = sbot.links2
359
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
360
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
361
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
362
    + this.wsPort + '/blobs/get/'
363
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
364
}
365
366
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
367
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
368
369
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
370
  var self = this
371
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
372
  ;(function next(i) {
373
    if (i >= ids.length) return cb()
374
    self.sbot.blobs.push(ids[i], function (err) {
375
      if (err) return cb(err)
376
      next(i+1)
377
    })
378
  })(0)
379
}
380
381
function getBlob(sbot, id, cb) {
382
  var blobs = sbot.blobs
383
  blobs.size(id, function (err, size) {
384
    if (typeof size === 'number') cb(null, blobs.get(id))
385
    else blobs.want(id, function (err, got) {
386
      if (err) cb(err)
387
      else if (!got) cb('missing blob ' + id)
388
      else cb(null, blobs.get(id))
389
    })
390
  })
391
}
392
393
SsbNpmRegistryServer.prototype.blobDist = function (id) {
394
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
395
  if (!m) throw new Error('bad blob id: ' + id)
396
  return {
397
    integrity: m[2] + '-' + m[1],
398
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
399
  }
400
}
401
402
function getMentions(links2, name) {
403
  return links2.read({
404
    query: [
405
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
406
      {$filter: {dest: {$prefix: '&'}}},
407
      {$map: {
408
        name: ['rel', 1],
409
        size: ['rel', 2],
410
        link: 'dest',
411
        author: 'source',
412
        ts: 'ts'
413
      }}
414
    ]
415
  })
416
}
417
418
SsbNpmRegistryServer.prototype.getMentions = function (name) {
419
  return getMentions(this.links2, name)
420
}
421
422
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
423
  var self = this
424
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
425
  var ids = {}
426
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
427
  fs.readdir(prebuildsDir, function (err, filenames) {
428
    if (err) return cb(new Error(err.stack || err))
429
    ;(function next(i) {
430
      if (i >= filenames.length) return cb(null, ids)
431
      var m = nameRegex.exec(filenames[i])
432
      if (!m) return next(i+1)
433
      var name = m[1]
434
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
435
        if (err) return cb(new Error(err.stack || err))
436
        self.sbot.blobs.add(function (err, id) {
437
          if (err) return cb(new Error(err.stack || err))
438
          ids[name] = id
439
          next(i+1)
440
        })(pull.once(data))
441
      })
442
    })(0)
443
  })
444
}
445
446
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
447
  var self = this
448
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
449
450
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
451
    if (err) return cb(new Error(err.stack || err))
452
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
453
    if (pkgs._hasNonBlobUrl) {
454
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
455
    }
456
457
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
458
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
459
460
    var waiting = 2
461
462
    self.sbot.blobs.add(function (err, id) {
463
      if (err) return next(new Error(err.stack || err))
464
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
465
      var versions = pkg.versions || (pkg.versions = {})
466
      pkg.versions[sbotPkgLock.version] = {
467
        name: sbotPkgLock.name,
468
        version: sbotPkgLock.version,
469
        dist: self.blobDist(id)
470
      }
471
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
472
      distTags.latest = sbotPkgLock.version
473
      next()
474
    })(self.sbot.bootstrap.pack())
475
476
    var prebuilds
477
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
478
      if (err) return next(err)
479
      prebuilds = _prebuilds
480
      next()
481
    })
482
483
    function next(err) {
484
      if (err) return waiting = 0, cb(err)
485
      if (--waiting) return
486
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
487
        encoding: 'utf8'
488
      }, function (err, bootstrapScript) {
489
        if (err) return cb(err)
490
        var script = bootstrapScript + '\n' +
491
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
492
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
493
494
        self.sbot.blobs.add(function (err, id) {
495
          if (err) return cb(new Error(err.stack || err))
496
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
497
          if (!m) return cb(new Error('bad blob id: ' + id))
498
          cb(null, {
499
            name: sbotPkgLock.name,
500
            blob: id,
501
            hashType: m[2],
502
            hashBuf: Buffer.from(m[1], 'base64'),
503
          })
504
        })(pull.once(script))
505
      })
506
    }
507
  })
508
}
509
510
function Req(server, req, res) {
511
  this.server = server
512
  this.req = req
513
  this.res = res
514
  this.blobsToPush = []
515
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
516
  var ua = this.req.headers['user-agent']
517
  var m = /\bnpm\/([0-9]*)/.exec(ua)
518
  var npmVersion = m && m[1]
519
  this.needShasum = server.needShasum != null ? server.needShasum :
520
    (npmVersion && npmVersion < 5)
521
}
522
523
Req.prototype.serve = function () {
524
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
525
  var pathname = this.req.url.replace(/\?.*/, '')
526
  var m
527
  if (pathname === '/') return this.serveHome()
528
  if (pathname === '/bootstrap') return this.serveBootstrap()
529
  if (pathname === '/-/whoami') return this.serveWhoami()
530
  if (pathname === '/-/ping') return this.respond(200, true)
531
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
532
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
533
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
534
  return this.respond(404)
535
}
536
537
Req.prototype.respond = function (status, message) {
538
  this.res.writeHead(status, {'content-type': 'application/json'})
539
  this.res.end(message && JSON.stringify(message, 0, 2))
540
}
541
542
Req.prototype.respondError = function (status, message) {
543
  this.respond(status, {error: message})
544
}
545
546
var bootstrapName = 'ssb-npm-bootstrap'
547
548
Req.prototype.serveHome = function () {
549
  var self = this
550
  self.res.writeHead(200, {'content-type': 'text/html'})
551
  var port = 8044
552
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
553
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
554
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
555
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
556
    '</body></html>')
557
}
558
559
Req.prototype.serveBootstrap = function () {
560
  var self = this
561
  self.server.getBootstrapInfo(function (err, info) {
562
    if (err) return self.respondError(err.stack || err)
563
    var pkgNameText = info.name
564
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
565
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
566
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
567
    var blobsHostname = httpHost + ':' + self.server.wsPort
568
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
569
    var pkgHashText = info.hashBuf.toString('hex')
570
    var hashCmd = info.hashType + 'sum'
571
572
    var script =
573
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
574
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
575
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
576
        'npm install -g ' + info.name + ' &&\n' +
577
      'sbot server'
578
579
    self.res.writeHead(200, {'content-type': 'text/plain'})
580
    self.res.end(script)
581
  })
582
}
583
584
Req.prototype.serveWhoami = function () {
585
  var self = this
586
  self.server.sbot.whoami(function (err, feed) {
587
    if (err) return self.respondError(err.stack || err)
588
    self.respond(200, {username: feed.id})
589
  })
590
}
591
592
Req.prototype.serveUser1 = function () {
593
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
594
}
595
596
function decodeName(name) {
597
  var parts = name.replace(/\.tgz$/, '').split(':')
598
  return {
599
    name: parts[1],
600
    version: parts[2],
601
    distTag: parts[3],
602
  }
603
}
604
605
Req.prototype.servePkg = function (pathname) {
606
  var self = this
607
  var parts = pathname.split('/')
608
  var pkgName = parts.shift().replace(/%2f/i, '/')
609
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
610
  var spec = parts.shift()
611
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
612
  if (parts.length > 0) return this.respondError(404)
613
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
614
  var obj = {
615
    _id: pkgName,
616
    name: pkgName,
617
    'dist-tags': {},
618
    versions: {}
619
  }
620
  var distTags = {/* <tag>: {version, ts}*/}
621
  pull(
622
    self.server.getMentions({$prefix: 'npm:' + pkgName + ':'}),
623
    pull.drain(function (mention) {
624
      var data = decodeName(mention.name)
625
      if (!data.version) return
626
      if (data.distTag) {
627
        var tag = distTags[data.distTag]
628
        if (!tag || mention.ts > tag.ts) {
629
          /* TODO: sort by causal order (versionBranch links) instead of just
630
          * by timestamps */
631
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
632
        }
633
      }
634
      obj.versions[data.version] = {
635
        author: {
636
          url: mention.author
637
        },
638
        name: pkgName,
639
        version: data.version,
640
        dist: self.server.blobDist(mention.link)
641
      }
642
    }, function (err) {
643
      if (err) return self.respondError(500, err.stack || err)
644
      for (var tag in distTags) {
645
        obj['dist-tags'][tag] = distTags[tag].version
646
      }
647
      if (spec) resolveSpec()
648
      else if (self.fetchAll) resolveAll()
649
      else done()
650
    })
651
  )
652
  function resolveSpec() {
653
    var version = obj['dist-tags'][spec]
654
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
655
    obj = obj.versions[version]
656
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
657
    self.populatePackageJson(obj, function (err, pkg) {
658
      if (err) return self.respondError(500, err.stack || err)
659
      obj = pkg || obj
660
      done()
661
    })
662
  }
663
  function resolveAll() {
664
    var waiting = 0
665
    for (var version in obj.versions) (function (version) {
666
      waiting++
667
      self.populatePackageJson(obj.versions[version], function (err, pkg) {
668
        if (err && waiting <= 0) return console.trace(err)
669
        if (err) return waiting = 0, self.respondError(500, err.stack || err)
670
        if (pkg) obj.versions[version] = pkg
671
        if (!--waiting) done()
672
      })
673
    }(version))
674
    if (!waiting) done()
675
  }
676
  function done() {
677
    self.respond(200, obj)
678
  }
679
}
680
681
Req.prototype.servePrebuild = function (name) {
682
  var self = this
683
  var getMention = self.server.getMentions('prebuild:' + name)
684
  var blobsByAuthor = {/* <author>: BlobId */}
685
  getMention(null, function next(err, link) {
686
    if (err === true) return done()
687
    if (err) return self.respondError(500, err.stack || err)
688
    blobsByAuthor[link.author] = link.link
689
    getMention(null, next)
690
  })
691
  function done() {
692
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
693
    var blobId
694
    for (var feed in blobsByAuthor) {
695
      var blob = blobId = blobsByAuthor[feed]
696
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
697
      feeds.push(feed)
698
    }
699
    switch (Object.keys(authorsByLink).length) {
700
      case 0:
701
        return self.respondError(404, 'Not Found')
702
      case 1:
703
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
704
        return self.res.end()
705
      default:
706
        return self.respond(300, {choices: authorsByLink})
707
    }
708
  }
709
}
710
711
var localhosts = {
712
  '::1': true,
713
  '127.0.0.1': true,
714
  '::ffff:127.0.0.1': true,
715
}
716
717
Req.prototype.publishPkg = function (pkgName) {
718
  var self = this
719
  var remoteAddress = self.req.socket.remoteAddress
720
  if (!(remoteAddress in localhosts)) {
721
    return self.respondError(403, 'You may not publish as this user.')
722
  }
723
724
  var chunks = []
725
  self.req.on('data', function (data) {
726
    chunks.push(data)
727
  })
728
  self.req.on('end', function () {
729
    var data
730
    try {
731
      data = JSON.parse(Buffer.concat(chunks))
732
    } catch(e) {
733
      return self.respondError(400, e.stack)
734
    }
735
    return self.publishPkg2(pkgName, data || {})
736
  })
737
}
738
739
Req.prototype.publishPkg2 = function (name, data) {
740
  var self = this
741
  if (data.users) console.trace('[npm-registry] users property is not supported')
742
  var attachments = data._attachments || {}
743
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
744
  var waiting = 0
745
  Object.keys(attachments).forEach(function (filename) {
746
    waiting++
747
    var data = attachments[filename].data
748
    var tarball = new Buffer(data, 'base64')
749
    var length = attachments[filename].length
750
    if (length && length !== tarball.length) return self.respondError(400,
751
      'Length mismatch for attachment \'' + filename + '\'')
752
    self.server.sbot.blobs.add(function (err, id) {
753
      if (err) return self.respondError(500,
754
        'Adding attachment \'' + filename + '\' as blob failed')
755
      self.blobsToPush.push(id)
756
      var shasum = crypto.createHash('sha1').update(data).digest('hex')
757
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
758
      if (!--waiting) next()
759
    })(pull.once(tarball))
760
  })
761
  function next() {
762
    try {
763
      self.publishPkg3(name, data, links)
764
    } catch(e) {
765
      self.respondError(500, e.stack || e)
766
    }
767
  }
768
}
769
770
Req.prototype.publishPkg3 = function (name, data, links) {
771
  var self = this
772
  var versions = data.versions || {}
773
  var linksByVersion = {/* <version>: link */}
774
775
  // associate tarball blobs with versions
776
  for (var version in versions) {
777
    var pkg = versions[version]
778
    if (!pkg) return self.respondError(400, 'Bad package object')
779
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
780
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
781
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
782
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
783
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
784
    var filename = m[1]
785
    var link = links[filename]
786
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
787
    // TODO?: try to find missing tarball mentioned in other messages
788
    if (pkg.version && pkg.version !== version)
789
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
790
    linksByVersion[version] = link
791
    link.version = version
792
    link.dependencies = pkg.dependencies || {}
793
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
794
  }
795
796
  // associate blobs with dist-tags
797
  var tags = data['dist-tags'] || {}
798
  for (var tag in tags) {
799
    var version = tags[tag]
800
    var link = linksByVersion[version]
801
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
802
    // TODO?: support setting dist-tag without version,
803
    // by looking up a tarball blob for the version
804
    link.tag = tag
805
  }
806
807
  // compute blob links to publish
808
  var mentions = []
809
  for (var filename in links) {
810
    var link = links[filename] || {}
811
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
812
    mentions.push({
813
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
814
      link: link.link,
815
      size: link.size,
816
      shasum: link.shasum,
817
      dependencies: link.dependencies,
818
      bundledDependencies: link.bundledDependencies,
819
    })
820
  }
821
  return self.publishPkgs(mentions)
822
}
823
824
Req.prototype.publishPkgs = function (mentions) {
825
  var self = this
826
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
827
    if (err) self.respondError(500, err.stack || err)
828
    self.server.pushBlobs(self.blobsToPush, function (err) {
829
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
830
      self.respond(201)
831
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
832
    })
833
  })
834
}
835
836
Req.prototype.populatePackageJson = function (obj, cb) {
837
  var self = this
838
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
839
  var deps, bundledDeps, shasum
840
841
  // look for dependencies in links.
842
  // then fallback to getting it from the tarball blob
843
844
  pull(
845
    self.server.sbot.links({
846
      dest: blobId,
847
      rel: 'mentions',
848
      values: true,
849
    }),
850
    // decryption could be done here
851
    pull.map(function (msg) {
852
      var c = msg.value && msg.value.content
853
      var mentions = c && c.mentions
854
      return Array.isArray(mentions) ? mentions : []
855
    }),
856
    pull.flatten(),
857
    pull.filter(function (link) {
858
      return link && link.link === blobId
859
    }),
860
    pull.drain(function (link) {
861
      if (link.dependencies) deps = link.dependencies
862
      if (link.shasum) shasum = link.shasum
863
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
864
      // how to handle multiple assignments of dependencies to a package?
865
    }, function (err) {
866
      if (err) return cb(new Error(err.stack || err))
867
      if (deps && (shasum || !self.needShasum)) {
868
        // assume that the dependencies in the links to the blob are
869
        // correct.
870
        obj.dependencies = deps
871
        obj.bundledDependencies = bundledDeps
872
        if (shasum) obj.dist.shasum = obj._shasum = shasum
873
        cb(null, obj)
874
      } else {
875
        // get dependencies from the tarball
876
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
877
          if (err) return cb(err)
878
          pkg.dist = obj.dist
879
          pkg.dist.shasum = pkg._shasum
880
          pkg.author = pkg.author || obj.author
881
          pkg.version = pkg.version || obj.version
882
          pkg.name = pkg.name || obj.name
883
          cb(null, pkg)
884
        })
885
      }
886
    })
887
  )
888
}
889
890
function getPackageJsonFromTarballBlob(sbot, id, cb) {
891
  var self = this
892
  getBlob(sbot, id, function (err, readBlob) {
893
    if (err) return cb(err)
894
    cb = once(cb)
895
    var extract = tar.extract()
896
    var pkg, shasum
897
    extract.on('entry', function (header, stream, next) {
898
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
899
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
900
          if (err) return cb(err)
901
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
902
          catch(e) { return cb(e) }
903
          next()
904
        }))
905
      } else {
906
        stream.on('end', next)
907
        stream.resume()
908
      }
909
    })
910
    extract.on('finish', function () {
911
      pkg._shasum = shasum
912
      cb(null, pkg)
913
    })
914
    pull(
915
      readBlob,
916
      hash('sha1', 'hex', function (err, sum) {
917
        if (err) return cb(err)
918
        shasum = sum
919
      }),
920
      toPull(zlib.createGunzip()),
921
      toPull(extract)
922
    )
923
  })
924
}
925