git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var hash = require('pull-hash')
13
var multicb = require('multicb')
14
var memo = require('asyncmemo')
15
var lru = require('hashlru')
16
17
function escapeHTML(str) {
18
  return String(str)
19
    .replace(/</g, '&lt;')
20
    .replace(/>/g, '&gt;')
21
}
22
23
function onceify(fn, self) {
24
  var cbs = [], err, data
25
  return function (cb) {
26
    if (fn) {
27
      cbs.push(cb)
28
      fn.call(self, function (_err, _data) {
29
        err = _err, data = _data
30
        var _cbs = cbs
31
        cbs = null
32
        while (_cbs.length) _cbs.shift()(err, data)
33
      })
34
      fn = null
35
    } else if (cbs) {
36
      cbs.push(cb)
37
    } else {
38
      cb(err, data)
39
    }
40
  }
41
}
42
43
function once(cb) {
44
  var done
45
  return function (err, result) {
46
    if (done) {
47
      if (err) console.trace(err)
48
    } else {
49
      done = true
50
      cb(err, result)
51
    }
52
  }
53
}
54
55
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
56
  // convert a package-lock.json file into data for serving as an npm registry
57
  var hasNonBlobUrl = false
58
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
59
  var pkgs = {}
60
  var queue = [pkgLock, pkgLock.name]
61
  while (queue.length) {
62
    var dep = queue.shift(), name = queue.shift()
63
    if (name) {
64
      var pkg = pkgs[name] || (pkgs[name] = {
65
        _id: name,
66
        name: name,
67
        versions: {}
68
      })
69
      if (dep.version && dep.integrity && dep.resolved) {
70
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
71
        pkg.versions[dep.version] = {
72
          name: name,
73
          version: dep.version,
74
          dist: {
75
            integrity: dep.integrity,
76
            tarball: dep.resolved
77
          }
78
        }
79
      }
80
    }
81
    if (dep.dependencies) for (var depName in dep.dependencies) {
82
      queue.push(dep.dependencies[depName], depName)
83
    }
84
  }
85
  pkgs._hasNonBlobUrl = hasNonBlobUrl
86
  return pkgs
87
}
88
89
function npmLogin(registryAddress, cb) {
90
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
91
  var filename = path.join(os.homedir(), '.npmrc')
92
  fs.readFile(filename, 'utf8', function (err, data) {
93
    if (err && err.code === 'ENOENT') data = ''
94
    else if (err) return cb(new Error(err.stack))
95
    var lines = data ? data.split('\n') : []
96
    if (lines.indexOf(tokenLine) > -1) return cb()
97
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
98
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
99
    fs.appendFile(filename, line, cb)
100
  })
101
}
102
103
function formatHost(host) {
104
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
105
}
106
107
exports.name = 'npm-registry'
108
exports.version = '1.0.0'
109
exports.manifest = {
110
  getAddress: 'async'
111
}
112
exports.init = function (sbot, config) {
113
  var conf = config.npm || {}
114
  var port = conf.port || 8043
115
  var host = conf.host || null
116
  var autoAuth = conf.autoAuth !== false
117
118
  var server = http.createServer(exports.respond(sbot, config))
119
  var getAddress = onceify(function (cb) {
120
    server.on('error', cb)
121
    server.listen(port, host, function () {
122
      server.removeListener('error', cb)
123
      var regHost = formatHost(host || 'localhost')
124
      var regPort = this.address().port
125
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
126
      if (autoAuth) npmLogin(regUrl, next)
127
      else next()
128
      function next(err) {
129
        cb(err, regUrl)
130
      }
131
    })
132
    sbot.on('close', function () {
133
      server.close()
134
    })
135
  })
136
137
  getAddress(function (err, addr) {
138
    if (err) return console.error(err)
139
    console.log('[npm-registry] Listening on ' + addr)
140
  })
141
142
  return {
143
    getAddress: getAddress
144
  }
145
}
146
147
exports.respond = function (sbot, config) {
148
  var reg = new SsbNpmRegistryServer(sbot, config)
149
  return function (req, res) {
150
    new Req(reg, req, res).serve()
151
  }
152
}
153
154
function publishMsg(sbot, value, cb) {
155
  var gotExpectedPrevious = false
156
  sbot.publish(value, function next(err, msg) {
157
    if (err && /^expected previous:/.test(err.message)) {
158
      // retry once on this error
159
      if (gotExpectedPrevious) return cb(err)
160
      gotExpectedPrevious = true
161
      return sbot.publish(value, next)
162
    }
163
    cb(err, msg)
164
  })
165
}
166
167
function getDependencyBranches(sbot, id, cb) {
168
  // get ids of heads of tree of dependencyBranch message links to include all
169
  // the dependencies of the given tarball id.
170
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
171
  var msgs = {}
172
  var branched = {}
173
  var blobs = {}
174
175
  function addPkgById(id, cb) {
176
    if (blobs[id]) return cb()
177
    blobs[id] = true
178
    getPackageJsonCached(id, function (err, pkg) {
179
      if (err) return cb(err)
180
      var done = multicb()
181
      for (var name in pkg.dependencies || {}) {
182
        addPkgBySpec(name, pkg.dependencies[name], done())
183
      }
184
      done(cb)
185
    })
186
  }
187
188
  function addPkgBySpec(name, spec, cb) {
189
    var done = multicb()
190
    pull(
191
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
192
      pull.map(function (mention) {
193
        addPkgById(mention.link, done())
194
        return packageLinks(sbot, mention.author, mention.link, name, spec)
195
      }),
196
      pull.flatten(),
197
      pull.drain(function (msg) {
198
        var c = msg && msg.value && msg.value.content
199
        if (!c) return
200
        msgs[msg.key] = msg.value
201
        if (Array.isArray(c.dependencyBranch)) {
202
          for (var k = 0; k < c.dependencyBranch.length; k++) {
203
            branched[c.dependencyBranch[k]] = true
204
          }
205
        }
206
      }, function (err) {
207
        if (err) return cb(err)
208
        done(cb)
209
      })
210
    )
211
  }
212
213
  addPkgById(id, function (err) {
214
    if (err) return cb(err)
215
    var ids = []
216
    for (var key in msgs) {
217
      if (!branched[key]) ids.push(key)
218
    }
219
    cb(null, ids)
220
  })
221
}
222
223
function packageLinks(sbot, feed, id, name, spec) {
224
  function matches(mention) {
225
    var data = mention
226
      && mention.link === id
227
      && decodeName(mention.name)
228
    return data
229
      && data.name === name
230
      && (spec ? semver.satisfies(data.version, spec) : true)
231
  }
232
  return pull(
233
    sbot.links({
234
      source: feed,
235
      dest: id,
236
      rel: 'mentions',
237
      values: true,
238
    }),
239
    pull.filter(function (msg) {
240
      var c = msg && msg.value && msg.value.content
241
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
242
    })
243
  )
244
}
245
246
function getVersionBranches(sbot, link, cb) {
247
  var data = decodeName(link.name)
248
  var msgs = {}, branched = {}
249
  pull(
250
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
251
    pull.map(function (mention) {
252
      return packageLinks(sbot, mention.author, mention.link, data.name)
253
    }),
254
    pull.flatten(),
255
    pull.drain(function (msg) {
256
      var c = msg && msg.value && msg.value.content
257
      if (!c) return
258
      msgs[msg.key] = msg.value
259
      if (Array.isArray(c.versionBranch)) {
260
        for (var k = 0; k < c.versionBranch.length; k++) {
261
          branched[c.versionBranch[k]] = true
262
        }
263
      }
264
    }, function (err) {
265
      if (err) return cb(err)
266
      var ids = []
267
      for (var key in msgs) {
268
        if (!branched[key]) ids.push(key)
269
      }
270
      cb(null, ids)
271
    })
272
  )
273
}
274
275
// For each dependency that is not a bundledDependency, get message ids for
276
// that dependency name + version.
277
function publishSingleMention(sbot, mention, cb) {
278
  // Calculate dependencyBranch and versionBranch message ids.
279
  var value = {
280
    type: 'npm-packages',
281
    mentions: [mention]
282
  }
283
  var done = multicb({pluck: 1, spread: true})
284
  getDependencyBranches(sbot, mention.link, done())
285
  getVersionBranches(sbot, mention, done())
286
  done(function (err, dependencyBranches, versionBranches) {
287
    if (err) return cb(err)
288
    value.dependencyBranch = dependencyBranches || undefined
289
    value.versionBranch = versionBranches || undefined
290
    publishMsg(sbot, value, cb)
291
  })
292
}
293
294
function publishMentions(sbot, mentions, cb) {
295
  // console.error("publishing %s mentions", mentions.length)
296
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
297
  // if it is just one mention, fetch and add useful metadata
298
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
299
  publishMsg(sbot, {
300
    type: 'npm-packages',
301
    mentions: mentions,
302
  }, cb)
303
}
304
305
exports.publishPkgMentions = function (sbot, mentions, cb) {
306
  // try to fit the mentions into as few messages as possible,
307
  // while fitting under the message size limit.
308
  var msgs = []
309
  ;(function next(i, chunks) {
310
    if (i >= mentions.length) return cb(null, msgs)
311
    var chunkLen = Math.ceil(mentions.length / chunks)
312
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
313
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
314
      if (err && msgs.length) return onPartialPublish(err)
315
      if (err) return cb(err)
316
      msgs.push(msg)
317
      next(i + chunkLen, chunks)
318
    })
319
  })(0, 1)
320
  function onPartialPublish(err) {
321
    var remaining = mentions.length - i
322
    return cb(new Error('Published messages ' +
323
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
324
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
325
  }
326
}
327
328
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
329
  cb = once(cb)
330
  var waiting = 0
331
  var expandedMentions = mentions.map(function (link) {
332
    var id = link && link.link
333
    if (!id) return link
334
    waiting++
335
    var newLink = {}
336
    for (var k in link) newLink[k] = link[k]
337
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
338
      if (err) return cb(err)
339
      for (var k in props) newLink[k] = pkg[k]
340
      if (!--waiting) next()
341
    })
342
    return newLink
343
  })
344
  if (!waiting) next()
345
  function next() {
346
    cb(null, expandedMentions)
347
  }
348
}
349
350
function SsbNpmRegistryServer(sbot, config) {
351
  this.sbot = sbot
352
  this.config = config
353
  this.npmConfig = config.npm || {}
354
  this.host = this.npmConfig.host || 'localhost'
355
  this.fetchAll = this.npmConfig.fetchAll
356
  this.links2 = sbot.links2
357
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
358
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
359
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
360
    + this.wsPort + '/blobs/get/'
361
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
362
}
363
364
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
365
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
366
367
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
368
  var self = this
369
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
370
  ;(function next(i) {
371
    if (i >= ids.length) return cb()
372
    self.sbot.blobs.push(ids[i], function (err) {
373
      if (err) return cb(err)
374
      next(i+1)
375
    })
376
  })(0)
377
}
378
379
function getBlob(sbot, id, cb) {
380
  var blobs = sbot.blobs
381
  blobs.size(id, function (err, size) {
382
    if (typeof size === 'number') cb(null, blobs.get(id))
383
    else blobs.want(id, function (err, got) {
384
      if (err) cb(err)
385
      else if (!got) cb('missing blob ' + id)
386
      else cb(null, blobs.get(id))
387
    })
388
  })
389
}
390
391
SsbNpmRegistryServer.prototype.blobDist = function (id) {
392
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
393
  if (!m) throw new Error('bad blob id: ' + id)
394
  return {
395
    integrity: m[2] + '-' + m[1],
396
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
397
  }
398
}
399
400
function getMentions(links2, name) {
401
  return links2.read({
402
    query: [
403
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
404
      {$filter: {dest: {$prefix: '&'}}},
405
      {$map: {
406
        name: ['rel', 1],
407
        size: ['rel', 2],
408
        link: 'dest',
409
        author: 'source',
410
        ts: 'ts'
411
      }}
412
    ]
413
  })
414
}
415
416
SsbNpmRegistryServer.prototype.getMentions = function (name) {
417
  return getMentions(this.links2, name)
418
}
419
420
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
421
  var self = this
422
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
423
  var ids = {}
424
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
425
  fs.readdir(prebuildsDir, function (err, filenames) {
426
    if (err) return cb(new Error(err.stack || err))
427
    ;(function next(i) {
428
      if (i >= filenames.length) return cb(null, ids)
429
      var m = nameRegex.exec(filenames[i])
430
      if (!m) return next(i+1)
431
      var name = m[1]
432
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
433
        if (err) return cb(new Error(err.stack || err))
434
        self.sbot.blobs.add(function (err, id) {
435
          if (err) return cb(new Error(err.stack || err))
436
          ids[name] = id
437
          next(i+1)
438
        })(pull.once(data))
439
      })
440
    })(0)
441
  })
442
}
443
444
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
445
  var self = this
446
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
447
448
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
449
    if (err) return cb(new Error(err.stack || err))
450
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
451
    if (pkgs._hasNonBlobUrl) {
452
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
453
    }
454
455
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
456
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
457
458
    var waiting = 2
459
460
    self.sbot.blobs.add(function (err, id) {
461
      if (err) return next(new Error(err.stack || err))
462
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
463
      var versions = pkg.versions || (pkg.versions = {})
464
      pkg.versions[sbotPkgLock.version] = {
465
        name: sbotPkgLock.name,
466
        version: sbotPkgLock.version,
467
        dist: self.blobDist(id)
468
      }
469
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
470
      distTags.latest = sbotPkgLock.version
471
      next()
472
    })(self.sbot.bootstrap.pack())
473
474
    var prebuilds
475
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
476
      if (err) return next(err)
477
      prebuilds = _prebuilds
478
      next()
479
    })
480
481
    function next(err) {
482
      if (err) return waiting = 0, cb(err)
483
      if (--waiting) return
484
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
485
        encoding: 'utf8'
486
      }, function (err, bootstrapScript) {
487
        if (err) return cb(err)
488
        var script = bootstrapScript + '\n' +
489
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
490
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
491
492
        self.sbot.blobs.add(function (err, id) {
493
          if (err) return cb(new Error(err.stack || err))
494
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
495
          if (!m) return cb(new Error('bad blob id: ' + id))
496
          cb(null, {
497
            name: sbotPkgLock.name,
498
            blob: id,
499
            hashType: m[2],
500
            hashBuf: Buffer.from(m[1], 'base64'),
501
          })
502
        })(pull.once(script))
503
      })
504
    }
505
  })
506
}
507
508
function Req(server, req, res) {
509
  this.server = server
510
  this.req = req
511
  this.res = res
512
  this.blobsToPush = []
513
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
514
}
515
516
Req.prototype.serve = function () {
517
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
518
  var pathname = this.req.url.replace(/\?.*/, '')
519
  var m
520
  if (pathname === '/') return this.serveHome()
521
  if (pathname === '/bootstrap') return this.serveBootstrap()
522
  if (pathname === '/-/whoami') return this.serveWhoami()
523
  if (pathname === '/-/ping') return this.respond(200, true)
524
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
525
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
526
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
527
  return this.respond(404)
528
}
529
530
Req.prototype.respond = function (status, message) {
531
  this.res.writeHead(status, {'content-type': 'application/json'})
532
  this.res.end(message && JSON.stringify(message, 0, 2))
533
}
534
535
Req.prototype.respondError = function (status, message) {
536
  this.respond(status, {error: message})
537
}
538
539
var bootstrapName = 'ssb-npm-bootstrap'
540
541
Req.prototype.serveHome = function () {
542
  var self = this
543
  self.res.writeHead(200, {'content-type': 'text/html'})
544
  var port = 8044
545
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
546
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
547
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
548
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
549
    '</body></html>')
550
}
551
552
Req.prototype.serveBootstrap = function () {
553
  var self = this
554
  self.server.getBootstrapInfo(function (err, info) {
555
    if (err) return self.respondError(err.stack || err)
556
    var pkgNameText = info.name
557
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
558
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
559
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
560
    var blobsHostname = httpHost + ':' + self.server.wsPort
561
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
562
    var pkgHashText = info.hashBuf.toString('hex')
563
    var hashCmd = info.hashType + 'sum'
564
565
    var script =
566
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
567
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
568
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
569
        'npm install -g ' + info.name + ' &&\n' +
570
      'sbot server'
571
572
    self.res.writeHead(200, {'content-type': 'text/plain'})
573
    self.res.end(script)
574
  })
575
}
576
577
Req.prototype.serveWhoami = function () {
578
  var self = this
579
  self.server.sbot.whoami(function (err, feed) {
580
    if (err) return self.respondError(err.stack || err)
581
    self.respond(200, {username: feed.id})
582
  })
583
}
584
585
Req.prototype.serveUser1 = function () {
586
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
587
}
588
589
function decodeName(name) {
590
  var parts = name.replace(/\.tgz$/, '').split(':')
591
  return {
592
    name: parts[1],
593
    version: parts[2],
594
    distTag: parts[3],
595
  }
596
}
597
598
Req.prototype.servePkg = function (pathname) {
599
  var self = this
600
  var parts = pathname.split('/')
601
  var pkgName = parts.shift().replace(/%2f/i, '/')
602
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
603
  var spec = parts.shift()
604
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
605
  if (parts.length > 0) return this.respondError(404)
606
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
607
  var obj = {
608
    _id: pkgName,
609
    name: pkgName,
610
    'dist-tags': {},
611
    versions: {}
612
  }
613
  pull(
614
    self.server.getMentions({$prefix: 'npm:' + pkgName + ':'}),
615
    pull.drain(function (mention) {
616
      var data = decodeName(mention.name)
617
      if (!data.version) return
618
      if (data.distTag) obj['dist-tags'][data.distTag] = data.version
619
      obj.versions[data.version] = {
620
        author: {
621
          url: mention.author
622
        },
623
        name: pkgName,
624
        version: data.version,
625
        dist: self.server.blobDist(mention.link)
626
      }
627
    }, function (err) {
628
      if (err) return self.respondError(500, err.stack || err)
629
      if (spec) resolveSpec()
630
      else if (self.fetchAll) resolveAll()
631
      else done()
632
    })
633
  )
634
  function resolveSpec() {
635
    var version = obj['dist-tags'][spec]
636
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
637
    obj = obj.versions[version]
638
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
639
    populatePackageJson(self.server.sbot, obj, function (err, pkg) {
640
      if (err) return self.respondError(500, err.stack || err)
641
      obj = pkg || obj
642
      done()
643
    })
644
  }
645
  function resolveAll() {
646
    var waiting = 0
647
    for (var version in obj.versions) (function (version) {
648
      waiting++
649
      populatePackageJson(self.server.sbot, obj.versions[version], function (err, pkg) {
650
        if (err && waiting <= 0) return console.trace(err)
651
        if (err) return waiting = 0, self.respondError(500, err.stack || err)
652
        if (pkg) obj.versions[version] = pkg
653
        if (!--waiting) done()
654
      })
655
    }(version))
656
    if (!waiting) done()
657
  }
658
  function done() {
659
    self.respond(200, obj)
660
  }
661
}
662
663
Req.prototype.servePrebuild = function (name) {
664
  var self = this
665
  var getMention = self.server.getMentions('prebuild:' + name)
666
  var blobsByAuthor = {/* <author>: BlobId */}
667
  getMention(null, function next(err, link) {
668
    if (err === true) return done()
669
    if (err) return self.respondError(500, err.stack || err)
670
    blobsByAuthor[link.author] = link.link
671
    getMention(null, next)
672
  })
673
  function done() {
674
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
675
    var blobId
676
    for (var feed in blobsByAuthor) {
677
      var blob = blobId = blobsByAuthor[feed]
678
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
679
      feeds.push(feed)
680
    }
681
    switch (Object.keys(authorsByLink).length) {
682
      case 0:
683
        return self.respondError(404, 'Not Found')
684
      case 1:
685
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
686
        return self.res.end()
687
      default:
688
        return self.respond(300, {choices: authorsByLink})
689
    }
690
  }
691
}
692
693
var localhosts = {
694
  '::1': true,
695
  '127.0.0.1': true,
696
  '::ffff:127.0.0.1': true,
697
}
698
699
Req.prototype.publishPkg = function (pkgName) {
700
  var self = this
701
  var remoteAddress = self.req.socket.remoteAddress
702
  if (!(remoteAddress in localhosts)) {
703
    return self.respondError(403, 'You may not publish as this user.')
704
  }
705
706
  var chunks = []
707
  self.req.on('data', function (data) {
708
    chunks.push(data)
709
  })
710
  self.req.on('end', function () {
711
    var data
712
    try {
713
      data = JSON.parse(Buffer.concat(chunks))
714
    } catch(e) {
715
      return self.respondError(400, e.stack)
716
    }
717
    return self.publishPkg2(pkgName, data || {})
718
  })
719
}
720
721
Req.prototype.publishPkg2 = function (name, data) {
722
  var self = this
723
  if (data.users) console.trace('[npm-registry] users property is not supported')
724
  var attachments = data._attachments || {}
725
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
726
  var waiting = 0
727
  Object.keys(attachments).forEach(function (filename) {
728
    waiting++
729
    var tarball = new Buffer(attachments[filename].data, 'base64')
730
    var length = attachments[filename].length
731
    if (length && length !== tarball.length) return self.respondError(400,
732
      'Length mismatch for attachment \'' + filename + '\'')
733
    self.server.sbot.blobs.add(function (err, id) {
734
      if (err) return self.respondError(500,
735
        'Adding attachment \'' + filename + '\' as blob failed')
736
      self.blobsToPush.push(id)
737
      links[filename] = {link: id, size: tarball.length}
738
      if (!--waiting) next()
739
    })(pull.once(tarball))
740
  })
741
  function next() {
742
    try {
743
      self.publishPkg3(name, data, links)
744
    } catch(e) {
745
      self.respondError(500, e.stack || e)
746
    }
747
  }
748
}
749
750
Req.prototype.publishPkg3 = function (name, data, links) {
751
  var self = this
752
  var versions = data.versions || {}
753
  var linksByVersion = {/* <version>: link */}
754
755
  // associate tarball blobs with versions
756
  for (var version in versions) {
757
    var pkg = versions[version]
758
    if (!pkg) return self.respondError(400, 'Bad package object')
759
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
760
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
761
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
762
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
763
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
764
    var filename = m[1]
765
    var link = links[filename]
766
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
767
    // TODO?: try to find missing tarball mentioned in other messages
768
    if (pkg.version && pkg.version !== version)
769
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
770
    linksByVersion[version] = link
771
    link.version = version
772
    link.dependencies = pkg.dependencies || {}
773
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
774
  }
775
776
  // associate blobs with dist-tags
777
  var tags = data['dist-tags'] || {}
778
  for (var tag in tags) {
779
    var version = tags[tag]
780
    var link = linksByVersion[version]
781
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
782
    // TODO?: support setting dist-tag without version,
783
    // by looking up a tarball blob for the version
784
    link.tag = tag
785
  }
786
787
  // compute blob links to publish
788
  var mentions = []
789
  for (var filename in links) {
790
    var link = links[filename] || {}
791
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
792
    mentions.push({
793
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
794
      link: link.link,
795
      size: link.size,
796
      dependencies: link.dependencies,
797
      bundledDependencies: link.bundledDependencies,
798
    })
799
  }
800
  return self.publishPkgs(mentions)
801
}
802
803
Req.prototype.publishPkgs = function (mentions) {
804
  var self = this
805
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
806
    if (err) self.respondError(500, err.stack || err)
807
    self.server.pushBlobs(self.blobsToPush, function (err) {
808
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
809
      self.respond(201)
810
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
811
    })
812
  })
813
}
814
815
function populatePackageJson(sbot, obj, cb) {
816
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
817
  var deps, bundledDeps
818
819
  // look for dependencies in links.
820
  // then fallback to getting it from the tarball blob
821
822
  pull(
823
    sbot.links({
824
      dest: blobId,
825
      rel: 'mentions',
826
      values: true,
827
    }),
828
    // decryption could be done here
829
    pull.map(function (msg) {
830
      var c = msg.value && msg.value.content
831
      var mentions = c && c.mentions
832
      return Array.isArray(mentions) ? mentions : []
833
    }),
834
    pull.flatten(),
835
    pull.filter(function (link) {
836
      return link && link.link === blobId
837
    }),
838
    pull.drain(function (link) {
839
      if (link.dependencies) deps = link.dependencies
840
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
841
      // how to handle multiple assignments of dependencies to a package?
842
    }, function (err) {
843
      if (err) return cb(new Error(err.stack || err))
844
      if (deps) {
845
        // assume that the dependencies in the links to the blob are
846
        // correct.
847
        obj.dependencies = deps
848
        obj.bundledDependencies = bundledDeps
849
        cb(null, obj)
850
      } else {
851
        // get dependencies from the tarball
852
        getPackageJsonFromTarballBlob(sbot, blobId, function (err, pkg) {
853
          if (err) return cb(err)
854
          pkg.dist = obj.dist
855
          pkg.dist.shasum = pkg._shasum
856
          pkg.author = pkg.author || obj.author
857
          pkg.version = pkg.version || obj.version
858
          pkg.name = pkg.name || obj.name
859
          cb(null, pkg)
860
        })
861
      }
862
    })
863
  )
864
}
865
866
function getPackageJsonFromTarballBlob(sbot, id, cb) {
867
  var self = this
868
  getBlob(sbot, id, function (err, readBlob) {
869
    if (err) return cb(err)
870
    cb = once(cb)
871
    var extract = tar.extract()
872
    var pkg, shasum
873
    extract.on('entry', function (header, stream, next) {
874
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
875
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
876
          if (err) return cb(err)
877
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
878
          catch(e) { return cb(e) }
879
          next()
880
        }))
881
      } else {
882
        stream.on('end', next)
883
        stream.resume()
884
      }
885
    })
886
    extract.on('finish', function () {
887
      pkg._shasum = shasum
888
      cb(null, pkg)
889
    })
890
    pull(
891
      readBlob,
892
      hash('sha1', 'hex', function (err, sum) {
893
        if (err) return cb(err)
894
        shasum = sum
895
      }),
896
      toPull(zlib.createGunzip()),
897
      toPull(extract)
898
    )
899
  })
900
}
901