git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function onceify(fn, self) {
25
  var cbs = [], err, data
26
  return function (cb) {
27
    if (fn) {
28
      cbs.push(cb)
29
      fn.call(self, function (_err, _data) {
30
        err = _err, data = _data
31
        var _cbs = cbs
32
        cbs = null
33
        while (_cbs.length) _cbs.shift()(err, data)
34
      })
35
      fn = null
36
    } else if (cbs) {
37
      cbs.push(cb)
38
    } else {
39
      cb(err, data)
40
    }
41
  }
42
}
43
44
function once(cb) {
45
  var done
46
  return function (err, result) {
47
    if (done) {
48
      if (err) console.trace(err)
49
    } else {
50
      done = true
51
      cb(err, result)
52
    }
53
  }
54
}
55
56
function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57
  // convert a package-lock.json file into data for serving as an npm registry
58
  var hasNonBlobUrl = false
59
  var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60
  var pkgs = {}
61
  var queue = [pkgLock, pkgLock.name]
62
  while (queue.length) {
63
    var dep = queue.shift(), name = queue.shift()
64
    if (name) {
65
      var pkg = pkgs[name] || (pkgs[name] = {
66
        _id: name,
67
        name: name,
68
        versions: {}
69
      })
70
      if (dep.version && dep.integrity && dep.resolved) {
71
        if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72
        pkg.versions[dep.version] = {
73
          name: name,
74
          version: dep.version,
75
          dist: {
76
            integrity: dep.integrity,
77
            tarball: dep.resolved
78
          }
79
        }
80
      }
81
    }
82
    if (dep.dependencies) for (var depName in dep.dependencies) {
83
      queue.push(dep.dependencies[depName], depName)
84
    }
85
  }
86
  pkgs._hasNonBlobUrl = hasNonBlobUrl
87
  return pkgs
88
}
89
90
function npmLogin(registryAddress, cb) {
91
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
92
  var filename = path.join(os.homedir(), '.npmrc')
93
  fs.readFile(filename, 'utf8', function (err, data) {
94
    if (err && err.code === 'ENOENT') data = ''
95
    else if (err) return cb(new Error(err.stack))
96
    var lines = data ? data.split('\n') : []
97
    if (lines.indexOf(tokenLine) > -1) return cb()
98
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
99
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
100
    fs.appendFile(filename, line, cb)
101
  })
102
}
103
104
function formatHost(host) {
105
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
106
}
107
108
exports.name = 'npm-registry'
109
exports.version = '1.0.0'
110
exports.manifest = {
111
  getAddress: 'async'
112
}
113
exports.init = function (sbot, config) {
114
  var conf = config.npm || {}
115
  var port = conf.port || 8043
116
  var host = conf.host || null
117
  var autoAuth = conf.autoAuth !== false
118
119
  var server = http.createServer(exports.respond(sbot, config))
120
  var getAddress = onceify(function (cb) {
121
    server.on('error', cb)
122
    server.listen(port, host, function () {
123
      server.removeListener('error', cb)
124
      var regHost = formatHost(host || 'localhost')
125
      var regPort = this.address().port
126
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
127
      if (autoAuth) npmLogin(regUrl, next)
128
      else next()
129
      function next(err) {
130
        cb(err, regUrl)
131
      }
132
    })
133
    sbot.on('close', function () {
134
      server.close()
135
    })
136
  })
137
138
  getAddress(function (err, addr) {
139
    if (err) return console.error(err)
140
    console.log('[npm-registry] Listening on ' + addr)
141
  })
142
143
  return {
144
    getAddress: getAddress
145
  }
146
}
147
148
exports.respond = function (sbot, config) {
149
  var reg = new SsbNpmRegistryServer(sbot, config)
150
  return function (req, res) {
151
    new Req(reg, req, res).serve()
152
  }
153
}
154
155
function publishMsg(sbot, value, cb) {
156
  var gotExpectedPrevious = false
157
  sbot.publish(value, function next(err, msg) {
158
    if (err && /^expected previous:/.test(err.message)) {
159
      // retry once on this error
160
      if (gotExpectedPrevious) return cb(err)
161
      gotExpectedPrevious = true
162
      return sbot.publish(value, next)
163
    }
164
    cb(err, msg)
165
  })
166
}
167
168
function getDependencyBranches(sbot, id, cb) {
169
  // get ids of heads of tree of dependencyBranch message links to include all
170
  // the dependencies of the given tarball id.
171
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
172
  var msgs = {}
173
  var branched = {}
174
  var blobs = {}
175
176
  function addPkgById(id, cb) {
177
    if (blobs[id]) return cb()
178
    blobs[id] = true
179
    getPackageJsonCached(id, function (err, pkg) {
180
      if (err) return cb(err)
181
      var done = multicb()
182
      for (var name in pkg.dependencies || {}) {
183
        addPkgBySpec(name, pkg.dependencies[name], done())
184
      }
185
      done(cb)
186
    })
187
  }
188
189
  function addPkgBySpec(name, spec, cb) {
190
    var versions = {}
191
    var distTags = {}
192
    pull(
193
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
194
      pull.filter(mentionMatches(null, name, spec)),
195
      pull.map(function (mention) {
196
        // query to get the messages since links2 does not include the msg id
197
        return sbot.links({
198
          source: mention.author,
199
          dest: mention.link,
200
          rel: 'mentions',
201
          values: true,
202
        })
203
      }),
204
      pull.flatten(),
205
      pull.drain(function (msg) {
206
        var c = msg && msg.value && msg.value.content
207
        if (!c || !Array.isArray(c.mentions)) return
208
        c.mentions.forEach(function (link) {
209
          var data = link && link.name && decodeName(link.name)
210
          if (!data || data.name !== name) return
211
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
212
          if (data.distTag) distTags[data.distTag] = data.version
213
        })
214
      }, function (err) {
215
        if (err) return cb(err)
216
        var version = distTags[spec]
217
          || semver.maxSatisfying(Object.keys(versions), spec)
218
        var item = versions[version]
219
        if (!item) return console.error('version not found: ' + name + '@' + spec)
220
        msgs[item.msg.key] = item.msg.value
221
        var c = item.msg.value.content
222
        if (Array.isArray(c.dependencyBranch)) {
223
          for (var k = 0; k < c.dependencyBranch.length; k++) {
224
            branched[c.dependencyBranch[k]] = true
225
          }
226
        }
227
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
228
        addPkgById(item.mention.link, cb)
229
      })
230
    )
231
  }
232
233
  addPkgById(id, function (err) {
234
    if (err) return cb(err)
235
    var ids = []
236
    for (var key in msgs) {
237
      if (!branched[key]) ids.push(key)
238
    }
239
    cb(null, ids)
240
  })
241
}
242
243
function packageLinks(sbot, feed, id, name, spec) {
244
  var matches = mentionMatches(id, name, spec)
245
  return pull(
246
    sbot.links({
247
      source: feed,
248
      dest: id,
249
      rel: 'mentions',
250
      values: true,
251
    }),
252
    pull.filter(function (msg) {
253
      var c = msg && msg.value && msg.value.content
254
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
255
    })
256
  )
257
}
258
259
function getVersionBranches(sbot, link, cb) {
260
  var data = decodeName(link.name)
261
  var msgs = {}, branched = {}
262
  pull(
263
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
264
    pull.map(function (mention) {
265
      return packageLinks(sbot, mention.author, mention.link, data.name)
266
    }),
267
    pull.flatten(),
268
    pull.drain(function (msg) {
269
      var c = msg && msg.value && msg.value.content
270
      if (!c) return
271
      msgs[msg.key] = msg.value
272
      if (Array.isArray(c.versionBranch)) {
273
        for (var k = 0; k < c.versionBranch.length; k++) {
274
          branched[c.versionBranch[k]] = true
275
        }
276
      }
277
    }, function (err) {
278
      if (err) return cb(err)
279
      var ids = []
280
      for (var key in msgs) {
281
        if (!branched[key]) ids.push(key)
282
      }
283
      cb(null, ids)
284
    })
285
  )
286
}
287
288
// For each dependency that is not a bundledDependency, get message ids for
289
// that dependency name + version.
290
function publishSingleMention(sbot, mention, cb) {
291
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
292
  // Calculate dependencyBranch and versionBranch message ids.
293
  var value = {
294
    type: 'npm-packages',
295
    mentions: [mention]
296
  }
297
  var done = multicb({pluck: 1, spread: true})
298
  getDependencyBranches(sbot, mention.link, done())
299
  getVersionBranches(sbot, mention, done())
300
  done(function (err, dependencyBranches, versionBranches) {
301
    if (err) return cb(err)
302
    value.dependencyBranch = dependencyBranches || undefined
303
    value.versionBranch = versionBranches || undefined
304
    publishMsg(sbot, value, cb)
305
  })
306
}
307
308
function publishMentions(sbot, mentions, cb) {
309
  // console.error("publishing %s mentions", mentions.length)
310
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
311
  // if it is just one mention, fetch and add useful metadata
312
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
313
  publishMsg(sbot, {
314
    type: 'npm-packages',
315
    mentions: mentions,
316
  }, cb)
317
}
318
319
exports.publishPkgMentions = function (sbot, mentions, cb) {
320
  // try to fit the mentions into as few messages as possible,
321
  // while fitting under the message size limit.
322
  var msgs = []
323
  ;(function next(i, chunks) {
324
    if (i >= mentions.length) return cb(null, msgs)
325
    var chunkLen = Math.ceil(mentions.length / chunks)
326
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
327
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
328
      if (err && msgs.length) return onPartialPublish(err)
329
      if (err) return cb(err)
330
      msgs.push(msg)
331
      next(i + chunkLen, chunks)
332
    })
333
  })(0, 1)
334
  function onPartialPublish(err) {
335
    var remaining = mentions.length - i
336
    return cb(new Error('Published messages ' +
337
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
338
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
339
  }
340
}
341
342
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
343
  cb = once(cb)
344
  var waiting = 0
345
  var expandedMentions = mentions.map(function (link) {
346
    var id = link && link.link
347
    if (!id) return link
348
    waiting++
349
    var newLink = {}
350
    for (var k in link) newLink[k] = link[k]
351
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
352
      if (err) return cb(err)
353
      for (var k in props) newLink[k] = pkg[k]
354
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
355
      if (!--waiting) next()
356
    })
357
    return newLink
358
  })
359
  if (!waiting) next()
360
  function next() {
361
    cb(null, expandedMentions)
362
  }
363
}
364
365
function SsbNpmRegistryServer(sbot, config) {
366
  this.sbot = sbot
367
  this.config = config
368
  this.npmConfig = config.npm || {}
369
  this.host = this.npmConfig.host || 'localhost'
370
  this.fetchAll = this.npmConfig.fetchAll
371
  this.needShasum = this.npmConfig.needShasum
372
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
373
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
374
    + this.wsPort + '/blobs/get/'
375
  this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
376
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
377
}
378
379
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
380
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
381
382
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
383
  var self = this
384
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
385
  ;(function next(i) {
386
    if (i >= ids.length) return cb()
387
    self.sbot.blobs.push(ids[i], function (err) {
388
      if (err) return cb(err)
389
      next(i+1)
390
    })
391
  })(0)
392
}
393
394
function getBlob(sbot, id, cb) {
395
  var blobs = sbot.blobs
396
  blobs.size(id, function (err, size) {
397
    if (typeof size === 'number') cb(null, blobs.get(id))
398
    else blobs.want(id, function (err, got) {
399
      if (err) cb(err)
400
      else if (!got) cb('missing blob ' + id)
401
      else cb(null, blobs.get(id))
402
    })
403
  })
404
}
405
406
SsbNpmRegistryServer.prototype.blobDist = function (id) {
407
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
408
  if (!m) throw new Error('bad blob id: ' + id)
409
  return {
410
    integrity: m[2] + '-' + m[1],
411
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
412
  }
413
}
414
415
function getMentions(links2, name) {
416
  return links2.read({
417
    query: [
418
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
419
      {$filter: {dest: {$prefix: '&'}}},
420
      {$map: {
421
        name: ['rel', 1],
422
        size: ['rel', 2],
423
        link: 'dest',
424
        author: 'source',
425
        ts: 'ts'
426
      }}
427
    ]
428
  })
429
}
430
431
function mentionMatches(id, name, spec) {
432
  return function (mention) {
433
    var data = mention
434
      && (!id || id === mention.link)
435
      && mention.name
436
      && decodeName(mention.name)
437
    return data
438
      && data.name === name
439
      && (spec ? semver.satisfies(data.version, spec) : true)
440
  }
441
}
442
443
SsbNpmRegistryServer.prototype.getMentions = function (name) {
444
  if (!this.sbot.links2) return pull.empty()
445
  return getMentions(this.sbot.links2, name)
446
}
447
448
SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
449
  var self = this
450
  var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
451
  var ids = {}
452
  var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
453
  fs.readdir(prebuildsDir, function (err, filenames) {
454
    if (err) return cb(new Error(err.stack || err))
455
    ;(function next(i) {
456
      if (i >= filenames.length) return cb(null, ids)
457
      var m = nameRegex.exec(filenames[i])
458
      if (!m) return next(i+1)
459
      var name = m[1]
460
      fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
461
        if (err) return cb(new Error(err.stack || err))
462
        self.sbot.blobs.add(function (err, id) {
463
          if (err) return cb(new Error(err.stack || err))
464
          ids[name] = id
465
          next(i+1)
466
        })(pull.once(data))
467
      })
468
    })(0)
469
  })
470
}
471
472
SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
473
  var self = this
474
  if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
475
476
  self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
477
    if (err) return cb(new Error(err.stack || err))
478
    var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
479
    if (pkgs._hasNonBlobUrl) {
480
      console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
481
    }
482
483
    if (!sbotPkgLock.name) console.trace('missing pkg lock name')
484
    if (!sbotPkgLock.version) console.trace('missing pkg lock version')
485
486
    var waiting = 2
487
488
    self.sbot.blobs.add(function (err, id) {
489
      if (err) return next(new Error(err.stack || err))
490
      var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
491
      var versions = pkg.versions || (pkg.versions = {})
492
      pkg.versions[sbotPkgLock.version] = {
493
        name: sbotPkgLock.name,
494
        version: sbotPkgLock.version,
495
        dist: self.blobDist(id)
496
      }
497
      var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
498
      distTags.latest = sbotPkgLock.version
499
      next()
500
    })(self.sbot.bootstrap.pack())
501
502
    var prebuilds
503
    self.getLocalPrebuildsLinks(function (err, _prebuilds) {
504
      if (err) return next(err)
505
      prebuilds = _prebuilds
506
      next()
507
    })
508
509
    function next(err) {
510
      if (err) return waiting = 0, cb(err)
511
      if (--waiting) return
512
      fs.readFile(path.join(__dirname, 'bootstrap.js'), {
513
        encoding: 'utf8'
514
      }, function (err, bootstrapScript) {
515
        if (err) return cb(err)
516
        var script = bootstrapScript + '\n' +
517
          'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
518
          'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
519
520
        self.sbot.blobs.add(function (err, id) {
521
          if (err) return cb(new Error(err.stack || err))
522
          var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
523
          if (!m) return cb(new Error('bad blob id: ' + id))
524
          cb(null, {
525
            name: sbotPkgLock.name,
526
            blob: id,
527
            hashType: m[2],
528
            hashBuf: Buffer.from(m[1], 'base64'),
529
          })
530
        })(pull.once(script))
531
      })
532
    }
533
  })
534
}
535
536
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
537
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
538
  else this.sbot.get(id, function (err, value) {
539
    if (err) return cb(err)
540
    cb(null, {key: id, value: value})
541
  })
542
}
543
544
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
545
  var self = this
546
  var stack = heads.slice()
547
  var seen = {}
548
  return function (abort, cb) {
549
    if (abort) return cb(abort)
550
    for (var id; stack.length && seen[id = stack.pop()];);
551
    if (!id) return cb(true)
552
    seen[id] = true
553
    // TODO: use DFS
554
    self.getMsg(id, function (err, msg) {
555
      if (err) return cb(new Error(err.stack || err))
556
      var c = msg && msg.value && msg.value.content
557
      var links = c && c[prop]
558
      if (Array.isArray(links)) {
559
        stack.push.apply(stack, links)
560
      } else if (links) {
561
        stack.push(links)
562
      }
563
      cb(null, msg)
564
    })
565
  }
566
}
567
568
function Req(server, req, res) {
569
  this.server = server
570
  this.req = req
571
  this.res = res
572
  this.blobsToPush = []
573
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
574
  var ua = this.req.headers['user-agent']
575
  var m = /\bnpm\/([0-9]*)/.exec(ua)
576
  var npmVersion = m && m[1]
577
  this.needShasum = server.needShasum != null ? server.needShasum :
578
    (npmVersion && npmVersion < 5)
579
}
580
581
Req.prototype.serve = function () {
582
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
583
  this.res.setTimeout(0)
584
  var pathname = this.req.url.replace(/\?.*/, '')
585
  var m
586
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)(\/.*)$/.exec(pathname))) {
587
    try {
588
      this.headMsgId = decodeURIComponent(m[2])
589
      this.headMsgPlus = !!m[1]
590
      // ^ means also include packages published after the head message id
591
    } catch(e) {
592
      return this.respondError(400, e.stack || e)
593
    }
594
    pathname = m[3]
595
  }
596
  if (pathname === '/') return this.serveHome()
597
  if (pathname === '/bootstrap') return this.serveBootstrap()
598
  if (pathname === '/-/whoami') return this.serveWhoami()
599
  if (pathname === '/-/ping') return this.respond(200, true)
600
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
601
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
602
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
603
  return this.respond(404)
604
}
605
606
Req.prototype.respond = function (status, message) {
607
  this.res.writeHead(status, {'content-type': 'application/json'})
608
  this.res.end(message && JSON.stringify(message, 0, 2))
609
}
610
611
Req.prototype.respondError = function (status, message) {
612
  this.respond(status, {error: message})
613
}
614
615
var bootstrapName = 'ssb-npm-bootstrap'
616
617
Req.prototype.serveHome = function () {
618
  var self = this
619
  self.res.writeHead(200, {'content-type': 'text/html'})
620
  var port = 8044
621
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
622
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
623
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
624
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
625
    '</body></html>')
626
}
627
628
Req.prototype.serveBootstrap = function () {
629
  var self = this
630
  self.server.getBootstrapInfo(function (err, info) {
631
    if (err) return self.respondError(err.stack || err)
632
    var pkgNameText = info.name
633
    var pkgTmpText = '/tmp/' + bootstrapName + '.js'
634
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
635
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
636
    var blobsHostname = httpHost + ':' + self.server.wsPort
637
    var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
638
    var pkgHashText = info.hashBuf.toString('hex')
639
    var hashCmd = info.hashType + 'sum'
640
641
    var script =
642
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
643
      'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
644
      'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
645
        'npm install -g ' + info.name + ' &&\n' +
646
      'sbot server'
647
648
    self.res.writeHead(200, {'content-type': 'text/plain'})
649
    self.res.end(script)
650
  })
651
}
652
653
Req.prototype.serveWhoami = function () {
654
  var self = this
655
  self.server.sbot.whoami(function (err, feed) {
656
    if (err) return self.respondError(err.stack || err)
657
    self.respond(200, {username: feed.id})
658
  })
659
}
660
661
Req.prototype.serveUser1 = function () {
662
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
663
}
664
665
function decodeName(name) {
666
  var parts = String(name).replace(/\.tgz$/, '').split(':')
667
  return {
668
    name: parts[1],
669
    version: parts[2],
670
    distTag: parts[3],
671
  }
672
}
673
674
Req.prototype.getMsgMentions = function (name) {
675
  return pull(
676
    this.server.streamTree([this.headMsgId], 'dependencyBranch'),
677
    // decryption could be done here
678
    pull.map(function (msg) {
679
      var c = msg.value && msg.value.content
680
      if (!c.mentions || !Array.isArray(c.mentions)) return []
681
      return c.mentions.map(function (mention) {
682
        return {
683
          name: mention.name,
684
          size: mention.size,
685
          link: mention.link,
686
          author: msg.value.author,
687
          ts: msg.value.timestamp,
688
        }
689
      })
690
    }),
691
    pull.flatten(),
692
    pull.filter(typeof name === 'string' ? function (link) {
693
      return link.name === name
694
    } : name && name.$prefix ? function (link) {
695
      return link.name.substr(0, name.$prefix.length) === name.$prefix
696
    } : function () {
697
      throw new TypeError('unsupported name filter')
698
    })
699
  )
700
}
701
702
Req.prototype.getMentions = function (name) {
703
  var useMsgMentions = this.headMsgId
704
  var useServerMentions = !this.headMsgId || this.headMsgPlus
705
  if (useServerMentions && !this.server.sbot.links2) {
706
    return this.headMsgPlus
707
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
708
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
709
  }
710
  return cat([
711
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
712
    useServerMentions ? this.server.getMentions(name) : pull.empty()
713
  ])
714
}
715
716
Req.prototype.getMentionLinks = function (blobId) {
717
  return pull(
718
    this.headMsgId
719
      ? this.server.streamTree([this.headMsgId], 'dependencyBranch')
720
      : this.server.sbot.links({
721
        dest: blobId,
722
        rel: 'mentions',
723
        values: true,
724
      }),
725
    // decryption could be done here
726
    pull.map(function (msg) {
727
      var c = msg.value && msg.value.content
728
      return c && Array.isArray(c.mentions) || []
729
    }),
730
    pull.flatten(),
731
    pull.filter(function (link) {
732
      return link && link.link === blobId
733
    })
734
  )
735
}
736
737
Req.prototype.servePkg = function (pathname) {
738
  var self = this
739
  var parts = pathname.split('/')
740
  var pkgName = parts.shift().replace(/%2f/i, '/')
741
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
742
  var spec = parts.shift()
743
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
744
  if (parts.length > 0) return this.respondError(404)
745
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
746
  var obj = {
747
    _id: pkgName,
748
    name: pkgName,
749
    'dist-tags': {},
750
    versions: {}
751
  }
752
  var distTags = {/* <tag>: {version, ts}*/}
753
  pull(
754
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
755
    pull.drain(function (mention) {
756
      var data = decodeName(mention.name)
757
      if (!data.version) return
758
      if (data.distTag) {
759
        var tag = distTags[data.distTag]
760
        if (!tag || mention.ts > tag.ts) {
761
          /* TODO: sort by causal order (versionBranch links) instead of just
762
          * by timestamps */
763
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
764
        }
765
      }
766
      obj.versions[data.version] = {
767
        author: {
768
          url: mention.author
769
        },
770
        name: pkgName,
771
        version: data.version,
772
        dist: self.server.blobDist(mention.link)
773
      }
774
    }, function (err) {
775
      if (err) return self.respondError(500, err.stack || err)
776
      for (var tag in distTags) {
777
        obj['dist-tags'][tag] = distTags[tag].version
778
      }
779
      if (spec) resolveSpec()
780
      else if (self.fetchAll) resolveAll()
781
      else resolved()
782
    })
783
  )
784
  function resolveSpec() {
785
    var version = obj['dist-tags'][spec]
786
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
787
    obj = obj.versions[version]
788
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
789
    self.populatePackageJson(obj, function (err, pkg) {
790
      if (err) return resolved(err)
791
      obj = pkg || obj
792
      resolved()
793
    })
794
  }
795
  function resolveVersion(version, cb) {
796
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
797
      if (err) return cb(err)
798
      if (pkg) obj.versions[version] = pkg
799
      cb()
800
    })
801
  }
802
  function resolveAll() {
803
    var done = multicb()
804
    for (var version in obj.versions) {
805
      resolveVersion(version, done())
806
    }
807
    done(resolved)
808
  }
809
  function resolved(err) {
810
    if (err) return self.respondError(500, err.stack || err)
811
    self.respond(200, obj)
812
  }
813
}
814
815
Req.prototype.servePrebuild = function (name) {
816
  var self = this
817
  var getMention = self.getMentions('prebuild:' + name)
818
  var blobsByAuthor = {/* <author>: BlobId */}
819
  getMention(null, function next(err, link) {
820
    if (err === true) return done()
821
    if (err) return self.respondError(500, err.stack || err)
822
    blobsByAuthor[link.author] = link.link
823
    getMention(null, next)
824
  })
825
  function done() {
826
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
827
    var blobId
828
    for (var feed in blobsByAuthor) {
829
      var blob = blobId = blobsByAuthor[feed]
830
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
831
      feeds.push(feed)
832
    }
833
    switch (Object.keys(authorsByLink).length) {
834
      case 0:
835
        return self.respondError(404, 'Not Found')
836
      case 1:
837
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
838
        return self.res.end()
839
      default:
840
        return self.respond(300, {choices: authorsByLink})
841
    }
842
  }
843
}
844
845
var localhosts = {
846
  '::1': true,
847
  '127.0.0.1': true,
848
  '::ffff:127.0.0.1': true,
849
}
850
851
Req.prototype.publishPkg = function (pkgName) {
852
  var self = this
853
  var remoteAddress = self.req.socket.remoteAddress
854
  if (!(remoteAddress in localhosts)) {
855
    return self.respondError(403, 'You may not publish as this user.')
856
  }
857
858
  var chunks = []
859
  self.req.on('data', function (data) {
860
    chunks.push(data)
861
  })
862
  self.req.on('end', function () {
863
    var data
864
    try {
865
      data = JSON.parse(Buffer.concat(chunks))
866
    } catch(e) {
867
      return self.respondError(400, e.stack)
868
    }
869
    return self.publishPkg2(pkgName, data || {})
870
  })
871
}
872
873
Req.prototype.publishPkg2 = function (name, data) {
874
  var self = this
875
  if (data.users) console.trace('[npm-registry] users property is not supported')
876
  var attachments = data._attachments || {}
877
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
878
  var done = multicb()
879
  function addAttachmentAsBlob(filename, cb) {
880
    var data = attachments[filename].data
881
    var tarball = new Buffer(data, 'base64')
882
    var length = attachments[filename].length
883
    if (length && length !== tarball.length) return self.respondError(400,
884
      'Length mismatch for attachment \'' + filename + '\'')
885
    self.server.sbot.blobs.add(function (err, id) {
886
      if (err) return cb(err)
887
      self.blobsToPush.push(id)
888
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
889
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
890
      cb()
891
    })(pull.once(tarball))
892
  }
893
  for (var filename in attachments) {
894
    addAttachmentAsBlob(filename, done())
895
  }
896
  done(function (err) {
897
    if (err) return self.respondError(500, err.stack || err)
898
    try {
899
      self.publishPkg3(name, data, links)
900
    } catch(e) {
901
      self.respondError(500, e.stack || e)
902
    }
903
  })
904
}
905
906
Req.prototype.publishPkg3 = function (name, data, links) {
907
  var self = this
908
  var versions = data.versions || {}
909
  var linksByVersion = {/* <version>: link */}
910
911
  // associate tarball blobs with versions
912
  for (var version in versions) {
913
    var pkg = versions[version]
914
    if (!pkg) return self.respondError(400, 'Bad package object')
915
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
916
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
917
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
918
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
919
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
920
    var filename = m[1]
921
    var link = links[filename]
922
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
923
    // TODO?: try to find missing tarball mentioned in other messages
924
    if (pkg.version && pkg.version !== version)
925
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
926
    linksByVersion[version] = link
927
    link.version = version
928
    link.dependencies = pkg.dependencies || {}
929
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
930
  }
931
932
  // associate blobs with dist-tags
933
  var tags = data['dist-tags'] || {}
934
  for (var tag in tags) {
935
    var version = tags[tag]
936
    var link = linksByVersion[version]
937
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
938
    // TODO?: support setting dist-tag without version,
939
    // by looking up a tarball blob for the version
940
    link.tag = tag
941
  }
942
943
  // compute blob links to publish
944
  var mentions = []
945
  for (var filename in links) {
946
    var link = links[filename] || {}
947
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
948
    mentions.push({
949
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
950
      link: link.link,
951
      size: link.size,
952
      shasum: link.shasum,
953
      dependencies: link.dependencies,
954
      bundledDependencies: link.bundledDependencies,
955
    })
956
  }
957
  return self.publishPkgs(mentions)
958
}
959
960
Req.prototype.publishPkgs = function (mentions) {
961
  var self = this
962
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
963
    if (err) return self.respondError(500, err.stack || err)
964
    self.server.pushBlobs(self.blobsToPush, function (err) {
965
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
966
      self.respond(201)
967
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
968
    })
969
  })
970
}
971
972
Req.prototype.populatePackageJson = function (obj, cb) {
973
  var self = this
974
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
975
  var deps, bundledDeps, shasum
976
977
  // look for dependencies in links.
978
  // then fallback to getting it from the tarball blob
979
980
  pull(
981
    self.getMentionLinks(blobId),
982
    pull.drain(function (link) {
983
      if (link.dependencies) deps = link.dependencies
984
      if (link.shasum) shasum = link.shasum
985
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
986
      // how to handle multiple assignments of dependencies to a package?
987
    }, function (err) {
988
      if (err) return cb(new Error(err.stack || err))
989
      if (deps && (shasum || !self.needShasum)) {
990
        // assume that the dependencies in the links to the blob are
991
        // correct.
992
        obj.dependencies = deps
993
        obj.bundledDependencies = bundledDeps
994
        if (shasum) obj.dist.shasum = obj._shasum = shasum
995
        cb(null, obj)
996
      } else {
997
        // get dependencies from the tarball
998
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
999
          if (err) return cb(err)
1000
          pkg.dist = obj.dist
1001
          pkg.dist.shasum = pkg._shasum
1002
          pkg.author = pkg.author || obj.author
1003
          pkg.version = pkg.version || obj.version
1004
          pkg.name = pkg.name || obj.name
1005
          cb(null, pkg)
1006
        })
1007
      }
1008
    })
1009
  )
1010
}
1011
1012
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1013
  var self = this
1014
  getBlob(sbot, id, function (err, readBlob) {
1015
    if (err) return cb(err)
1016
    cb = once(cb)
1017
    var extract = tar.extract()
1018
    var pkg, shasum
1019
    extract.on('entry', function (header, stream, next) {
1020
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1021
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1022
          if (err) return cb(err)
1023
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1024
          catch(e) { return cb(e) }
1025
          next()
1026
        }))
1027
      } else {
1028
        stream.on('end', next)
1029
        stream.resume()
1030
      }
1031
    })
1032
    extract.on('finish', function () {
1033
      pkg._shasum = shasum
1034
      cb(null, pkg)
1035
    })
1036
    pull(
1037
      readBlob,
1038
      hash('sha1', 'hex', function (err, sum) {
1039
        if (err) return cb(err)
1040
        shasum = sum
1041
      }),
1042
      toPull(zlib.createGunzip()),
1043
      toPull(extract)
1044
    )
1045
  })
1046
}
1047