git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var hash = require('pull-hash')
13
var multicb = require('multicb')
14
var memo = require('asyncmemo')
15
var lru = require('hashlru')
16
17
function escapeHTML(str) {
18
  return String(str)
19
    .replace(/</g, '&lt;')
20
    .replace(/>/g, '&gt;')
21
}
22
23
function onceify(fn, self) {
24
  var cbs = [], err, data
25
  return function (cb) {
26
    if (fn) {
27
      cbs.push(cb)
28
      fn.call(self, function (_err, _data) {
29
        err = _err, data = _data
30
        var _cbs = cbs
31
        cbs = null
32
        while (_cbs.length) _cbs.shift()(err, data)
33
      })
34
      fn = null
35
    } else if (cbs) {
36
      cbs.push(cb)
37
    } else {
38
      cb(err, data)
39
    }
40
  }
41
}
42
43
function once(cb) {
44
  var done
45
  return function (err, result) {
46
    if (done) {
47
      if (err) console.trace(err)
48
    } else {
49
      done = true
50
      cb(err, result)
51
    }
52
  }
53
}
54
55
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
56
  // convert a package-lock.json file into data for serving as an npm registry
57
  var hasNonBlobUrl = false
58
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
59
  var pkgs = {}
60
  var queue = [pkgLock, pkgLock.name]
61
  while (queue.length) {
62
    var dep = queue.shift(), name = queue.shift()
63
    if (name) {
64
      var pkg = pkgs[name] || (pkgs[name] = {
65
        _id: name,
66
        name: name,
67
        versions: {}
68
      })
69
      if (dep.version && dep.integrity && dep.resolved) {
70
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
71
        pkg.versions[dep.version] = {
72
          name: name,
73
          version: dep.version,
74
          dist: {
75
            integrity: dep.integrity,
76
            tarball: dep.resolved
77
          }
78
        }
79
      }
80
    }
81
    if (dep.dependencies) for (var depName in dep.dependencies) {
82
      queue.push(dep.dependencies[depName], depName)
83
    }
84
  }
85
  pkgs._hasNonBlobUrl = hasNonBlobUrl
86
  return pkgs
87
}
88
89
function npmLogin(registryAddress, cb) {
90
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
91
  var filename = path.join(os.homedir(), '.npmrc')
92
  fs.readFile(filename, 'utf8', function (err, data) {
93
    if (err && err.code === 'ENOENT') data = ''
94
    else if (err) return cb(new Error(err.stack))
95
    var lines = data ? data.split('\n') : []
96
    if (lines.indexOf(tokenLine) > -1) return cb()
97
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
98
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
99
    fs.appendFile(filename, line, cb)
100
  })
101
}
102
103
function formatHost(host) {
104
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
105
}
106
107
exports.name = 'npm-registry'
108
exports.version = '1.0.0'
109
exports.manifest = {
110
  getAddress: 'async'
111
}
112
exports.init = function (sbot, config) {
113
  var conf = config.npm || {}
114
  var port = conf.port || 8043
115
  var host = conf.host || null
116
  var autoAuth = conf.autoAuth !== false
117
118
  var server = http.createServer(exports.respond(sbot, config))
119
  var getAddress = onceify(function (cb) {
120
    server.on('error', cb)
121
    server.listen(port, host, function () {
122
      server.removeListener('error', cb)
123
      var regHost = formatHost(host || 'localhost')
124
      var regPort = this.address().port
125
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
126
      if (autoAuth) npmLogin(regUrl, next)
127
      else next()
128
      function next(err) {
129
        cb(err, regUrl)
130
      }
131
    })
132
    sbot.on('close', function () {
133
      server.close()
134
    })
135
  })
136
137
  getAddress(function (err, addr) {
138
    if (err) return console.error(err)
139
    console.log('[npm-registry] Listening on ' + addr)
140
  })
141
142
  return {
143
    getAddress: getAddress
144
  }
145
}
146
147
exports.respond = function (sbot, config) {
148
  var reg = new SsbNpmRegistryServer(sbot, config)
149
  return function (req, res) {
150
    new Req(reg, req, res).serve()
151
  }
152
}
153
154
function publishMsg(sbot, value, cb) {
155
  var gotExpectedPrevious = false
156
  sbot.publish(value, function next(err, msg) {
157
    if (err && /^expected previous:/.test(err.message)) {
158
      // retry once on this error
159
      if (gotExpectedPrevious) return cb(err)
160
      gotExpectedPrevious = true
161
      return sbot.publish(value, next)
162
    }
163
    cb(err, msg)
164
  })
165
}
166
167
function getDependencyBranches(sbot, id, cb) {
168
  // get ids of heads of tree of dependencyBranch message links to include all
169
  // the dependencies of the given tarball id.
170
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
171
  var msgs = {}
172
  var branched = {}
173
  var blobs = {}
174
175
  function addPkgById(id, cb) {
176
    if (blobs[id]) return cb()
177
    blobs[id] = true
178
    getPackageJsonCached(id, function (err, pkg) {
179
      if (err) return cb(err)
180
      var done = multicb()
181
      for (var name in pkg.dependencies || {}) {
182
        addPkgBySpec(name, pkg.dependencies[name], done())
183
      }
184
      done(cb)
185
    })
186
  }
187
188
  function addPkgBySpec(name, spec, cb) {
189
    var done = multicb()
190
    pull(
191
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
192
      pull.map(function (mention) {
193
        addPkgById(mention.link, done())
194
        return packageLinks(sbot, mention.author, mention.link, name, spec)
195
      }),
196
      pull.flatten(),
197
      pull.drain(function (msg) {
198
        var c = msg && msg.value && msg.value.content
199
        if (!c) return
200
        msgs[msg.key] = msg.value
201
        if (Array.isArray(c.dependencyBranch)) {
202
          for (var k = 0; k < c.dependencyBranch.length; k++) {
203
            branched[c.dependencyBranch[k]] = true
204
          }
205
        }
206
      }, function (err) {
207
        if (err) return cb(err)
208
        done(cb)
209
      })
210
    )
211
  }
212
213
  addPkgById(id, function (err) {
214
    if (err) return cb(err)
215
    var ids = []
216
    for (var key in msgs) {
217
      if (!branched[key]) ids.push(key)
218
    }
219
    cb(null, ids)
220
  })
221
}
222
223
function packageLinks(sbot, feed, id, name, spec) {
224
  function matches(mention) {
225
    var data = mention
226
      && mention.link === id
227
      && decodeName(mention.name)
228
    return data
229
      && data.name === name
230
      && (spec ? semver.satisfies(data.version, spec) : true)
231
  }
232
  return pull(
233
    sbot.links({
234
      source: feed,
235
      dest: id,
236
      rel: 'mentions',
237
      values: true,
238
    }),
239
    pull.filter(function (msg) {
240
      var c = msg && msg.value && msg.value.content
241
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
242
    })
243
  )
244
}
245
246
function getVersionBranches(sbot, link, cb) {
247
  var data = decodeName(link.name)
248
  var msgs = {}, branched = {}
249
  pull(
250
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
251
    pull.map(function (mention) {
252
      return packageLinks(sbot, mention.author, mention.link, data.name)
253
    }),
254
    pull.flatten(),
255
    pull.drain(function (msg) {
256
      var c = msg && msg.value && msg.value.content
257
      if (!c) return
258
      msgs[msg.key] = msg.value
259
      if (Array.isArray(c.versionBranch)) {
260
        for (var k = 0; k < c.versionBranch.length; k++) {
261
          branched[c.versionBranch[k]] = true
262
        }
263
      }
264
    }, function (err) {
265
      if (err) return cb(err)
266
      var ids = []
267
      for (var key in msgs) {
268
        if (!branched[key]) ids.push(key)
269
      }
270
      cb(null, ids)
271
    })
272
  )
273
}
274
275
// For each dependency that is not a bundledDependency, get message ids for
276
// that dependency name + version.
277
function publishSingleMention(sbot, mention, cb) {
278
  // Calculate dependencyBranch and versionBranch message ids.
279
  var value = {
280
    type: 'npm-packages',
281
    mentions: [mention]
282
  }
283
  var done = multicb({pluck: 1, spread: true})
284
  getDependencyBranches(sbot, mention.link, done())
285
  getVersionBranches(sbot, mention, done())
286
  done(function (err, dependencyBranches, versionBranches) {
287
    if (err) return cb(err)
288
    value.dependencyBranch = dependencyBranches || undefined
289
    value.versionBranch = versionBranches || undefined
290
    publishMsg(sbot, value, cb)
291
  })
292
}
293
294
function publishMentions(sbot, mentions, cb) {
295
  // console.error("publishing %s mentions", mentions.length)
296
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
297
  // if it is just one mention, fetch and add useful metadata
298
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
299
  publishMsg(sbot, {
300
    type: 'npm-packages',
301
    mentions: mentions,
302
  }, cb)
303
}
304
305
exports.publishPkgMentions = function (sbot, mentions, cb) {
306
  // try to fit the mentions into as few messages as possible,
307
  // while fitting under the message size limit.
308
  var msgs = []
309
  ;(function next(i, chunks) {
310
    if (i >= mentions.length) return cb(null, msgs)
311
    var chunkLen = Math.ceil(mentions.length / chunks)
312
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
313
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
314
      if (err && msgs.length) return onPartialPublish(err)
315
      if (err) return cb(err)
316
      msgs.push(msg)
317
      next(i + chunkLen, chunks)
318
    })
319
  })(0, 1)
320
  function onPartialPublish(err) {
321
    var remaining = mentions.length - i
322
    return cb(new Error('Published messages ' +
323
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
324
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
325
  }
326
}
327
328
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
329
  cb = once(cb)
330
  var waiting = 0
331
  var expandedMentions = mentions.map(function (link) {
332
    var id = link && link.link
333
    if (!id) return link
334
    waiting++
335
    var newLink = {}
336
    for (var k in link) newLink[k] = link[k]
337
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
338
      if (err) return cb(err)
339
      for (var k in props) newLink[k] = pkg[k]
340
      if (!--waiting) next()
341
    })
342
    return newLink
343
  })
344
  if (!waiting) next()
345
  function next() {
346
    cb(null, expandedMentions)
347
  }
348
}
349
350
function SsbNpmRegistryServer(sbot, config) {
351
  this.sbot = sbot
352
  this.config = config
353
  this.npmConfig = config.npm || {}
354
  this.host = this.npmConfig.host || 'localhost'
355
  this.fetchAll = this.npmConfig.fetchAll
356
  this.links2 = sbot.links2
357
  if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
358
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
359
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
360
    + this.wsPort + '/blobs/get/'
361
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
362
}
363
364
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
365
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
366
367
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
368
  var self = this
369
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
370
  ;(function next(i) {
371
    if (i >= ids.length) return cb()
372
    self.sbot.blobs.push(ids[i], function (err) {
373
      if (err) return cb(err)
374
      next(i+1)
375
    })
376
  })(0)
377
}
378
379
function getBlob(sbot, id, cb) {
380
  var blobs = sbot.blobs
381
  blobs.size(id, function (err, size) {
382
    if (typeof size === 'number') cb(null, blobs.get(id))
383
    else blobs.want(id, function (err, got) {
384
      if (err) cb(err)
385
      else if (!got) cb('missing blob ' + id)
386
      else cb(null, blobs.get(id))
387
    })
388
  })
389
}
390
391
SsbNpmRegistryServer.prototype.blobDist = function (id) {
392
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
393
  if (!m) throw new Error('bad blob id: ' + id)
394
  return {
395
    integrity: m[2] + '-' + m[1],
396
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
397
  }
398
}
399
400
function getMentions(links2, name) {
401
  return links2.read({
402
    query: [
403
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
404
      {$filter: {dest: {$prefix: '&'}}},
405
      {$map: {
406
        name: ['rel', 1],
407
        size: ['rel', 2],
408
        link: 'dest',
409
        author: 'source',
410
        ts: 'ts'
411
      }}
412
    ]
413
  })
414
}
415
416
SsbNpmRegistryServer.prototype.getMentions = function (name) {
417
  return getMentions(this.links2, name)
418
}
419
420
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
421
  var self = this
422
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
423
  var ids = {}
424
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
425
  fs.readdir(prebuildsDir, function (err, filenames) {
426
    if (err) return cb(new Error(err.stack || err))
427
    ;(function next(i) {
428
      if (i >= filenames.length) return cb(null, ids)
429
      var m = nameRegex.exec(filenames[i])
430
      if (!m) return next(i+1)
431
      var name = m[1]
432
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
433
        if (err) return cb(new Error(err.stack || err))
434
        self.sbot.blobs.add(function (err, id) {
435
          if (err) return cb(new Error(err.stack || err))
436
          ids[name] = id
437
          next(i+1)
438
        })(pull.once(data))
439
      })
440
    })(0)
441
  })
442
}
443
444
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
445
  var self = this
446
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
447
448
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
449
    if (err) return cb(new Error(err.stack || err))
450
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
451
    if (pkgs._hasNonBlobUrl) {
452
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
453
    }
454
455
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
456
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
457
458
    var waiting = 2
459
460
    self.sbot.blobs.add(function (err, id) {
461
      if (err) return next(new Error(err.stack || err))
462
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
463
      var versions = pkg.versions || (pkg.versions = {})
464
      pkg.versions[sbotPkgLock.version] = {
465
        name: sbotPkgLock.name,
466
        version: sbotPkgLock.version,
467
        dist: self.blobDist(id)
468
      }
469
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
470
      distTags.latest = sbotPkgLock.version
471
      next()
472
    })(self.sbot.bootstrap.pack())
473
474
    var prebuilds
475
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
476
      if (err) return next(err)
477
      prebuilds = _prebuilds
478
      next()
479
    })
480
481
    function next(err) {
482
      if (err) return waiting = 0, cb(err)
483
      if (--waiting) return
484
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
485
        encoding: 'utf8'
486
      }, function (err, bootstrapScript) {
487
        if (err) return cb(err)
488
        var script = bootstrapScript + '\n' +
489
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
490
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
491
492
        self.sbot.blobs.add(function (err, id) {
493
          if (err) return cb(new Error(err.stack || err))
494
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
495
          if (!m) return cb(new Error('bad blob id: ' + id))
496
          cb(null, {
497
            name: sbotPkgLock.name,
498
            blob: id,
499
            hashType: m[2],
500
            hashBuf: Buffer.from(m[1], 'base64'),
501
          })
502
        })(pull.once(script))
503
      })
504
    }
505
  })
506
}
507
508
function Req(server, req, res) {
509
  this.server = server
510
  this.req = req
511
  this.res = res
512
  this.blobsToPush = []
513
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
514
}
515
516
Req.prototype.serve = function () {
517
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
518
  var pathname = this.req.url.replace(/\?.*/, '')
519
  var m
520
  if (pathname === '/') return this.serveHome()
521
  if (pathname === '/bootstrap') return this.serveBootstrap()
522
  if (pathname === '/-/whoami') return this.serveWhoami()
523
  if (pathname === '/-/ping') return this.respond(200, true)
524
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
525
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
526
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
527
  return this.respond(404)
528
}
529
530
Req.prototype.respond = function (status, message) {
531
  this.res.writeHead(status, {'content-type': 'application/json'})
532
  this.res.end(message && JSON.stringify(message, 0, 2))
533
}
534
535
Req.prototype.respondError = function (status, message) {
536
  this.respond(status, {error: message})
537
}
538
539
var bootstrapName = 'ssb-npm-bootstrap'
540
541
Req.prototype.serveHome = function () {
542
  var self = this
543
  self.res.writeHead(200, {'content-type': 'text/html'})
544
  var port = 8044
545
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
546
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
547
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
548
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
549
    '</body></html>')
550
}
551
552
Req.prototype.serveBootstrap = function () {
553
  var self = this
554
  self.server.getBootstrapInfo(function (err, info) {
555
    if (err) return self.respondError(err.stack || err)
556
    var pkgNameText = info.name
557
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
558
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
559
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
560
    var blobsHostname = httpHost + ':' + self.server.wsPort
561
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
562
    var pkgHashText = info.hashBuf.toString('hex')
563
    var hashCmd = info.hashType + 'sum'
564
565
    var script =
566
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
567
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
568
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
569
        'npm install -g ' + info.name + ' &&\n' +
570
      'sbot server'
571
572
    self.res.writeHead(200, {'content-type': 'text/plain'})
573
    self.res.end(script)
574
  })
575
}
576
577
Req.prototype.serveWhoami = function () {
578
  var self = this
579
  self.server.sbot.whoami(function (err, feed) {
580
    if (err) return self.respondError(err.stack || err)
581
    self.respond(200, {username: feed.id})
582
  })
583
}
584
585
Req.prototype.serveUser1 = function () {
586
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
587
}
588
589
function decodeName(name) {
590
  var parts = name.replace(/\.tgz$/, '').split(':')
591
  return {
592
    name: parts[1],
593
    version: parts[2],
594
    distTag: parts[3],
595
  }
596
}
597
598
Req.prototype.servePkg = function (pathname) {
599
  var self = this
600
  var parts = pathname.split('/')
601
  var pkgName = parts.shift().replace(/%2f/i, '/')
602
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
603
  var spec = parts.shift()
604
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
605
  if (parts.length > 0) return this.respondError(404)
606
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
607
  var obj = {
608
    _id: pkgName,
609
    name: pkgName,
610
    'dist-tags': {},
611
    versions: {}
612
  }
613
  var distTags = {/* <tag>: {version, ts}*/}
614
  pull(
615
    self.server.getMentions({$prefix: 'npm:' + pkgName + ':'}),
616
    pull.drain(function (mention) {
617
      var data = decodeName(mention.name)
618
      if (!data.version) return
619
      if (data.distTag) {
620
        var tag = distTags[data.distTag]
621
        if (!tag || mention.ts > tag.ts) {
622
          /* TODO: sort by causal order (versionBranch links) instead of just
623
          * by timestamps */
624
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
625
        }
626
      }
627
      obj.versions[data.version] = {
628
        author: {
629
          url: mention.author
630
        },
631
        name: pkgName,
632
        version: data.version,
633
        dist: self.server.blobDist(mention.link)
634
      }
635
    }, function (err) {
636
      if (err) return self.respondError(500, err.stack || err)
637
      for (var tag in distTags) {
638
        obj['dist-tags'][tag] = distTags[tag].version
639
      }
640
      if (spec) resolveSpec()
641
      else if (self.fetchAll) resolveAll()
642
      else done()
643
    })
644
  )
645
  function resolveSpec() {
646
    var version = obj['dist-tags'][spec]
647
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
648
    obj = obj.versions[version]
649
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
650
    populatePackageJson(self.server.sbot, obj, function (err, pkg) {
651
      if (err) return self.respondError(500, err.stack || err)
652
      obj = pkg || obj
653
      done()
654
    })
655
  }
656
  function resolveAll() {
657
    var waiting = 0
658
    for (var version in obj.versions) (function (version) {
659
      waiting++
660
      populatePackageJson(self.server.sbot, obj.versions[version], function (err, pkg) {
661
        if (err && waiting <= 0) return console.trace(err)
662
        if (err) return waiting = 0, self.respondError(500, err.stack || err)
663
        if (pkg) obj.versions[version] = pkg
664
        if (!--waiting) done()
665
      })
666
    }(version))
667
    if (!waiting) done()
668
  }
669
  function done() {
670
    self.respond(200, obj)
671
  }
672
}
673
674
Req.prototype.servePrebuild = function (name) {
675
  var self = this
676
  var getMention = self.server.getMentions('prebuild:' + name)
677
  var blobsByAuthor = {/* <author>: BlobId */}
678
  getMention(null, function next(err, link) {
679
    if (err === true) return done()
680
    if (err) return self.respondError(500, err.stack || err)
681
    blobsByAuthor[link.author] = link.link
682
    getMention(null, next)
683
  })
684
  function done() {
685
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
686
    var blobId
687
    for (var feed in blobsByAuthor) {
688
      var blob = blobId = blobsByAuthor[feed]
689
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
690
      feeds.push(feed)
691
    }
692
    switch (Object.keys(authorsByLink).length) {
693
      case 0:
694
        return self.respondError(404, 'Not Found')
695
      case 1:
696
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
697
        return self.res.end()
698
      default:
699
        return self.respond(300, {choices: authorsByLink})
700
    }
701
  }
702
}
703
704
var localhosts = {
705
  '::1': true,
706
  '127.0.0.1': true,
707
  '::ffff:127.0.0.1': true,
708
}
709
710
Req.prototype.publishPkg = function (pkgName) {
711
  var self = this
712
  var remoteAddress = self.req.socket.remoteAddress
713
  if (!(remoteAddress in localhosts)) {
714
    return self.respondError(403, 'You may not publish as this user.')
715
  }
716
717
  var chunks = []
718
  self.req.on('data', function (data) {
719
    chunks.push(data)
720
  })
721
  self.req.on('end', function () {
722
    var data
723
    try {
724
      data = JSON.parse(Buffer.concat(chunks))
725
    } catch(e) {
726
      return self.respondError(400, e.stack)
727
    }
728
    return self.publishPkg2(pkgName, data || {})
729
  })
730
}
731
732
Req.prototype.publishPkg2 = function (name, data) {
733
  var self = this
734
  if (data.users) console.trace('[npm-registry] users property is not supported')
735
  var attachments = data._attachments || {}
736
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
737
  var waiting = 0
738
  Object.keys(attachments).forEach(function (filename) {
739
    waiting++
740
    var tarball = new Buffer(attachments[filename].data, 'base64')
741
    var length = attachments[filename].length
742
    if (length && length !== tarball.length) return self.respondError(400,
743
      'Length mismatch for attachment \'' + filename + '\'')
744
    self.server.sbot.blobs.add(function (err, id) {
745
      if (err) return self.respondError(500,
746
        'Adding attachment \'' + filename + '\' as blob failed')
747
      self.blobsToPush.push(id)
748
      links[filename] = {link: id, size: tarball.length}
749
      if (!--waiting) next()
750
    })(pull.once(tarball))
751
  })
752
  function next() {
753
    try {
754
      self.publishPkg3(name, data, links)
755
    } catch(e) {
756
      self.respondError(500, e.stack || e)
757
    }
758
  }
759
}
760
761
Req.prototype.publishPkg3 = function (name, data, links) {
762
  var self = this
763
  var versions = data.versions || {}
764
  var linksByVersion = {/* <version>: link */}
765
766
  // associate tarball blobs with versions
767
  for (var version in versions) {
768
    var pkg = versions[version]
769
    if (!pkg) return self.respondError(400, 'Bad package object')
770
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
771
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
772
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
773
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
774
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
775
    var filename = m[1]
776
    var link = links[filename]
777
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
778
    // TODO?: try to find missing tarball mentioned in other messages
779
    if (pkg.version && pkg.version !== version)
780
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
781
    linksByVersion[version] = link
782
    link.version = version
783
    link.dependencies = pkg.dependencies || {}
784
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
785
  }
786
787
  // associate blobs with dist-tags
788
  var tags = data['dist-tags'] || {}
789
  for (var tag in tags) {
790
    var version = tags[tag]
791
    var link = linksByVersion[version]
792
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
793
    // TODO?: support setting dist-tag without version,
794
    // by looking up a tarball blob for the version
795
    link.tag = tag
796
  }
797
798
  // compute blob links to publish
799
  var mentions = []
800
  for (var filename in links) {
801
    var link = links[filename] || {}
802
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
803
    mentions.push({
804
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
805
      link: link.link,
806
      size: link.size,
807
      dependencies: link.dependencies,
808
      bundledDependencies: link.bundledDependencies,
809
    })
810
  }
811
  return self.publishPkgs(mentions)
812
}
813
814
Req.prototype.publishPkgs = function (mentions) {
815
  var self = this
816
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
817
    if (err) self.respondError(500, err.stack || err)
818
    self.server.pushBlobs(self.blobsToPush, function (err) {
819
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
820
      self.respond(201)
821
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
822
    })
823
  })
824
}
825
826
function populatePackageJson(sbot, obj, cb) {
827
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
828
  var deps, bundledDeps
829
830
  // look for dependencies in links.
831
  // then fallback to getting it from the tarball blob
832
833
  pull(
834
    sbot.links({
835
      dest: blobId,
836
      rel: 'mentions',
837
      values: true,
838
    }),
839
    // decryption could be done here
840
    pull.map(function (msg) {
841
      var c = msg.value && msg.value.content
842
      var mentions = c && c.mentions
843
      return Array.isArray(mentions) ? mentions : []
844
    }),
845
    pull.flatten(),
846
    pull.filter(function (link) {
847
      return link && link.link === blobId
848
    }),
849
    pull.drain(function (link) {
850
      if (link.dependencies) deps = link.dependencies
851
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
852
      // how to handle multiple assignments of dependencies to a package?
853
    }, function (err) {
854
      if (err) return cb(new Error(err.stack || err))
855
      if (deps) {
856
        // assume that the dependencies in the links to the blob are
857
        // correct.
858
        obj.dependencies = deps
859
        obj.bundledDependencies = bundledDeps
860
        cb(null, obj)
861
      } else {
862
        // get dependencies from the tarball
863
        getPackageJsonFromTarballBlob(sbot, blobId, function (err, pkg) {
864
          if (err) return cb(err)
865
          pkg.dist = obj.dist
866
          pkg.dist.shasum = pkg._shasum
867
          pkg.author = pkg.author || obj.author
868
          pkg.version = pkg.version || obj.version
869
          pkg.name = pkg.name || obj.name
870
          cb(null, pkg)
871
        })
872
      }
873
    })
874
  )
875
}
876
877
function getPackageJsonFromTarballBlob(sbot, id, cb) {
878
  var self = this
879
  getBlob(sbot, id, function (err, readBlob) {
880
    if (err) return cb(err)
881
    cb = once(cb)
882
    var extract = tar.extract()
883
    var pkg, shasum
884
    extract.on('entry', function (header, stream, next) {
885
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
886
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
887
          if (err) return cb(err)
888
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
889
          catch(e) { return cb(e) }
890
          next()
891
        }))
892
      } else {
893
        stream.on('end', next)
894
        stream.resume()
895
      }
896
    })
897
    extract.on('finish', function () {
898
      pkg._shasum = shasum
899
      cb(null, pkg)
900
    })
901
    pull(
902
      readBlob,
903
      hash('sha1', 'hex', function (err, sum) {
904
        if (err) return cb(err)
905
        shasum = sum
906
      }),
907
      toPull(zlib.createGunzip()),
908
      toPull(extract)
909
    )
910
  })
911
}
912