git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
77
}
78
79
exports.name = 'npm-registry'
80
exports.version = '1.0.0'
81
exports.manifest = {
82
  getAddress: 'async'
83
}
84
exports.init = function (sbot, config) {
85
  var conf = config.npm || {}
86
  var port = conf.port || 8043
87
  var host = conf.host || null
88
  var autoAuth = conf.autoAuth !== false
89
90
  var server = http.createServer(exports.respond(sbot, config))
91
  var getAddress = onceify(function (cb) {
92
    server.on('error', cb)
93
    server.listen(port, host, function () {
94
      server.removeListener('error', cb)
95
      var regHost = formatHost(host || 'localhost')
96
      var regPort = this.address().port
97
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
98
      if (autoAuth) npmLogin(regUrl, next)
99
      else next()
100
      function next(err) {
101
        cb(err, regUrl)
102
      }
103
    })
104
    sbot.on('close', function () {
105
      server.close()
106
    })
107
  })
108
109
  getAddress(function (err, addr) {
110
    if (err) return console.error(err)
111
    console.log('[npm-registry] Listening on ' + addr)
112
  })
113
114
  return {
115
    getAddress: getAddress
116
  }
117
}
118
119
exports.respond = function (sbot, config) {
120
  var reg = new SsbNpmRegistryServer(sbot, config)
121
  return function (req, res) {
122
    new Req(reg, req, res).serve()
123
  }
124
}
125
126
function publishMsg(sbot, value, cb) {
127
  var gotExpectedPrevious = false
128
  sbot.publish(value, function next(err, msg) {
129
    if (err && /^expected previous:/.test(err.message)) {
130
      // retry once on this error
131
      if (gotExpectedPrevious) return cb(err)
132
      gotExpectedPrevious = true
133
      return sbot.publish(value, next)
134
    }
135
    cb(err, msg)
136
  })
137
}
138
139
function getDependencyBranches(sbot, link, cb) {
140
  var id = link.link
141
142
  // get ids of heads of tree of dependencyBranch message links to include all
143
  // the dependencies of the given tarball id.
144
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
145
  var msgs = {}
146
  var branched = {}
147
  var blobs = {}
148
149
  function addPkgById(id, cb) {
150
    if (blobs[id]) return cb()
151
    blobs[id] = true
152
    getPackageJsonCached(id, function (err, pkg) {
153
      if (err) return cb(err)
154
      var done = multicb()
155
      for (var name in pkg.dependencies || {}) {
156
        addPkgBySpec(name, pkg.dependencies[name], done())
157
      }
158
      done(cb)
159
    })
160
  }
161
162
  function addPkgBySpec(name, spec, cb) {
163
    var versions = {}
164
    var distTags = {}
165
    pull(
166
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
167
      pull.filter(mentionMatches(null, name, spec)),
168
      pull.map(function (mention) {
169
        // query to get the messages since links2 does not include the msg id
170
        return sbot.links({
171
          source: mention.author,
172
          dest: mention.link,
173
          rel: 'mentions',
174
          values: true,
175
        })
176
      }),
177
      pull.flatten(),
178
      pull.drain(function (msg) {
179
        var c = msg && msg.value && msg.value.content
180
        if (!c || !Array.isArray(c.mentions)) return
181
        c.mentions.forEach(function (link) {
182
          var data = link && link.name && decodeName(link.name)
183
          if (!data || data.name !== name) return
184
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
185
          if (data.distTag) distTags[data.distTag] = data.version
186
        })
187
      }, function (err) {
188
        if (err) return cb(err)
189
        var version = distTags[spec]
190
          || semver.maxSatisfying(Object.keys(versions), spec)
191
        var item = versions[version]
192
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
193
        addDepBranchMsg(item.msg)
194
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
195
        addPkgById(item.mention.link, cb)
196
      })
197
    )
198
  }
199
200
  function addDepBranchMsg(msg) {
201
    var c = msg && msg.value && msg.value.content
202
    if (!c) return
203
    if (msgs[msg.key]) return
204
    msgs[msg.key] = msg.value
205
    if (Array.isArray(c.dependencyBranch)) {
206
      for (var k = 0; k < c.dependencyBranch.length; k++) {
207
        branched[c.dependencyBranch[k]] = true
208
      }
209
    }
210
    if (Array.isArray(c.versionBranch)) {
211
      for (var k = 0; k < c.versionBranch.length; k++) {
212
        branched[c.versionBranch[k]] = true
213
      }
214
    }
215
  }
216
217
  function addPkgsByName(name, cb) {
218
    var data = decodeName(name)
219
    pull(
220
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
221
      pull.map(function (mention) {
222
        return packageLinks(sbot, mention.author, mention.link, data.name)
223
      }),
224
      pull.flatten(),
225
      pull.drain(addDepBranchMsg, cb)
226
    )
227
  }
228
229
  var done = multicb()
230
  addPkgsByName(link.name, done())
231
  addPkgById(id, done())
232
  done(function (err) {
233
    if (err) return cb(err)
234
    var ids = []
235
    for (var key in msgs) {
236
      if (!branched[key]) ids.push(key)
237
    }
238
    cb(null, ids)
239
  })
240
}
241
242
function packageLinks(sbot, feed, id, name, spec) {
243
  var matches = mentionMatches(id, name, spec)
244
  return pull(
245
    sbot.links({
246
      source: feed,
247
      dest: id,
248
      rel: 'mentions',
249
      values: true,
250
    }),
251
    pull.filter(function (msg) {
252
      var c = msg && msg.value && msg.value.content
253
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
254
    })
255
  )
256
}
257
258
// For each dependency that is not a bundledDependency, get message ids for
259
// that dependency name + version.
260
function publishSingleMention(sbot, mention, cb) {
261
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
262
  // Calculate dependencyBranch message ids.
263
  var value = {
264
    type: 'npm-packages',
265
    mentions: [mention]
266
  }
267
  var done = multicb({pluck: 1, spread: true})
268
  getDependencyBranches(sbot, mention, function (err, branches) {
269
    if (err) return cb(err)
270
    value.dependencyBranch = branches || undefined
271
    publishMsg(sbot, value, cb)
272
  })
273
}
274
275
function publishMentions(sbot, mentions, cb) {
276
  // console.error("publishing %s mentions", mentions.length)
277
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
278
  // if it is just one mention, fetch and add useful metadata
279
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
280
  publishMsg(sbot, {
281
    type: 'npm-packages',
282
    mentions: mentions,
283
  }, cb)
284
}
285
286
exports.publishPkgMentions = function (sbot, mentions, cb) {
287
  // try to fit the mentions into as few messages as possible,
288
  // while fitting under the message size limit.
289
  var msgs = []
290
  ;(function next(i, chunks) {
291
    if (i >= mentions.length) return cb(null, msgs)
292
    var chunkLen = Math.ceil(mentions.length / chunks)
293
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
294
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
295
      if (err && msgs.length) return onPartialPublish(err)
296
      if (err) return cb(err)
297
      msgs.push(msg)
298
      next(i + chunkLen, chunks)
299
    })
300
  })(0, 1)
301
  function onPartialPublish(err) {
302
    var remaining = mentions.length - i
303
    return cb(new Error('Published messages ' +
304
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
305
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
306
  }
307
}
308
309
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
310
  cb = once(cb)
311
  var waiting = 0
312
  var expandedMentions = mentions.map(function (link) {
313
    var id = link && link.link
314
    if (!id) return link
315
    waiting++
316
    var newLink = {}
317
    for (var k in link) newLink[k] = link[k]
318
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
319
      if (err) return cb(err)
320
      for (var k in props) newLink[k] = pkg[k]
321
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
322
      if (!--waiting) next()
323
    })
324
    return newLink
325
  })
326
  if (!waiting) next()
327
  function next() {
328
    cb(null, expandedMentions)
329
  }
330
}
331
332
function SsbNpmRegistryServer(sbot, config) {
333
  this.sbot = sbot
334
  this.config = config
335
  this.npmConfig = config.npm || {}
336
  this.host = this.npmConfig.host || 'localhost'
337
  this.fetchAll = this.npmConfig.fetchAll
338
  this.needShasum = this.npmConfig.needShasum
339
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
340
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
341
    + this.wsPort + '/blobs/get/'
342
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
343
}
344
345
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
346
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
347
348
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
349
  var self = this
350
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
351
  ;(function next(i) {
352
    if (i >= ids.length) return cb()
353
    self.sbot.blobs.push(ids[i], function (err) {
354
      if (err) return cb(err)
355
      next(i+1)
356
    })
357
  })(0)
358
}
359
360
function getBlob(sbot, id, cb) {
361
  var blobs = sbot.blobs
362
  blobs.size(id, function (err, size) {
363
    if (typeof size === 'number') cb(null, blobs.get(id))
364
    else blobs.want(id, function (err, got) {
365
      if (err) cb(err)
366
      else if (!got) cb('missing blob ' + id)
367
      else cb(null, blobs.get(id))
368
    })
369
  })
370
}
371
372
SsbNpmRegistryServer.prototype.blobDist = function (id) {
373
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
374
  if (!m) throw new Error('bad blob id: ' + id)
375
  return {
376
    integrity: m[2] + '-' + m[1],
377
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
378
  }
379
}
380
381
function getMentions(links2, name) {
382
  return links2.read({
383
    query: [
384
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
385
      {$filter: {dest: {$prefix: '&'}}},
386
      {$map: {
387
        name: ['rel', 1],
388
        size: ['rel', 2],
389
        link: 'dest',
390
        author: 'source',
391
        ts: 'ts'
392
      }}
393
    ]
394
  })
395
}
396
397
function mentionMatches(id, name, spec) {
398
  return function (mention) {
399
    var data = mention
400
      && (!id || id === mention.link)
401
      && mention.name
402
      && decodeName(mention.name)
403
    return data
404
      && data.name === name
405
      && (spec ? semver.satisfies(data.version, spec) : true)
406
  }
407
}
408
409
SsbNpmRegistryServer.prototype.getMentions = function (name) {
410
  if (!this.sbot.links2) return pull.empty()
411
  return getMentions(this.sbot.links2, name)
412
}
413
414
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
415
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
416
  else this.sbot.get(id, function (err, value) {
417
    if (err) return cb(err)
418
    cb(null, {key: id, value: value})
419
  })
420
}
421
422
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
423
  var self = this
424
  var stack = heads.slice()
425
  var seen = {}
426
  return function (abort, cb) {
427
    if (abort) return cb(abort)
428
    for (var id; stack.length && seen[id = stack.pop()];);
429
    if (!id) return cb(true)
430
    seen[id] = true
431
    // TODO: use DFS
432
    self.getMsg(id, function (err, msg) {
433
      if (err) return cb(new Error(err.stack || err))
434
      var c = msg && msg.value && msg.value.content
435
      var links = c && c[prop]
436
      if (Array.isArray(links)) {
437
        stack.push.apply(stack, links)
438
      } else if (links) {
439
        stack.push(links)
440
      }
441
      cb(null, msg)
442
    })
443
  }
444
}
445
446
function Req(server, req, res) {
447
  this.server = server
448
  this.req = req
449
  this.res = res
450
  this.blobsToPush = []
451
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
452
  var ua = this.req.headers['user-agent']
453
  var m = /\bnpm\/([0-9]*)/.exec(ua)
454
  var npmVersion = m && m[1]
455
  this.needShasum = server.needShasum != null ? server.needShasum :
456
    (npmVersion && npmVersion < 5)
457
}
458
459
Req.prototype.serve = function () {
460
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
461
  this.res.setTimeout(0)
462
  var pathname = this.req.url.replace(/\?.*/, '')
463
  var m
464
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
465
    try {
466
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
467
      this.headMsgPlus = !!m[1]
468
      // ^ means also include packages published after the head message id
469
    } catch(e) {
470
      return this.respondError(400, e.stack || e)
471
    }
472
    pathname = m[3]
473
  }
474
  if (pathname === '/') return this.serveHome()
475
  if (pathname === '/bootstrap') return this.serveBootstrap()
476
  if (pathname === '/robots.txt') return this.serveRobots()
477
  if (pathname === '/-/whoami') return this.serveWhoami()
478
  if (pathname === '/-/ping') return this.respond(200, true)
479
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
480
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
481
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
482
  return this.respond(404)
483
}
484
485
Req.prototype.respond = function (status, message) {
486
  this.res.writeHead(status, {'content-type': 'application/json'})
487
  this.res.end(message && JSON.stringify(message, 0, 2))
488
}
489
490
Req.prototype.respondError = function (status, message) {
491
  this.respond(status, {error: message})
492
}
493
494
Req.prototype.respondErrorStr = function (status, err) {
495
  this.res.writeHead(status, {'content-type': 'text/plain'})
496
  this.res.end(err.stack || err)
497
}
498
499
Req.prototype.serveHome = function () {
500
  var self = this
501
  self.res.writeHead(200, {'content-type': 'text/html'})
502
  var port = 8044
503
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
504
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
505
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
506
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
507
    '</body></html>')
508
}
509
510
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
511
  var self = this
512
  pull(
513
    self.server.sbot.links({
514
      source: feedId,
515
      dest: blobId,
516
      rel: 'mentions',
517
      values: true,
518
    }),
519
    pull.filter(function (msg) {
520
      var c = msg && msg.value && msg.value.content
521
      return c.type === 'npm-packages'
522
    }),
523
    pull.collect(function (err, msgs) {
524
      if (err) return cb(err)
525
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
526
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
527
      // TODO: make a smarter decision about which message id to use
528
      cb(null, msgs.pop().key)
529
    })
530
  )
531
}
532
533
Req.prototype.resolvePkg = function (pkgSpec, cb) {
534
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
535
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
536
  var self = this
537
  var pkgName = m[1]
538
  var spec = m[2] || '*'
539
  var versions = {}
540
  var distTags = {}
541
  pull(
542
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
543
    pull.drain(function (mention) {
544
      var data = decodeName(mention.name)
545
      if (!data.version) return
546
      if (data.distTag) {
547
        distTags[data.distTag] = data.version
548
      }
549
      versions[data.version] = {
550
        author: mention.author,
551
        name: pkgName,
552
        version: data.version,
553
        blobId: mention.link
554
      }
555
    }, function (err) {
556
      if (err) return cb(err)
557
      var version = distTags[spec]
558
        || semver.maxSatisfying(Object.keys(versions), spec)
559
      var item = versions[version]
560
      if (!item) return cb(new Error('Version not found: ' + name + '@' + spec))
561
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
562
        if (err) return cb(err)
563
        item.msgId = id
564
        cb(null, item)
565
      })
566
    })
567
  )
568
}
569
570
Req.prototype.resolvePkgs = function (specs, cb) {
571
  var done = multicb({pluck: 1})
572
  var self = this
573
  specs.forEach(function (spec) {
574
    self.resolvePkg(spec, done())
575
  })
576
  done(cb)
577
}
578
579
Req.prototype.serveBootstrap = function () {
580
  var self = this
581
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
582
  var ssbNpmRegistryName = require('./package.json').name
583
  var ssbNpmRegistryVersion = require('./package.json').version
584
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
585
  var done = multicb({pluck: 1, spread: true})
586
  self.resolvePkg(ssbNpmRegistrySpec, done())
587
  self.resolvePkgs(pkgs, done())
588
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
589
    if (err) return self.respondErrorStr(500, err.stack || err)
590
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
591
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
592
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
593
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
594
    var globalPkgs = pkgsInfo.map(function (info) {
595
      return info.name + '@' + info.version
596
    }).join(' ')
597
    var npmCmd = 'npm install -g ' + globalPkgs
598
599
    var pkgTmpText = '/tmp/' + ssbNpmRegistryName + '.tar.gz'
600
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
601
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
602
    var wsLink = 'http://' + httpHost + ':' + self.server.wsPort + '/'
603
    var tarballLink = wsLink + 'blobs/get/' + ssbNpmRegistryBlobId
604
605
    var script =
606
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
607
      'echo ' + ssbNpmRegistryBlobHex + ' ' + pkgTmpText + ' | sha256sum -c &&\n' +
608
      'mkdir -p ~/.ssb/node_modules && cd ~/.ssb/node_modules &&\n' +
609
      'tar xzf ' + pkgTmpText + ' &&\n' +
610
      'mv package ' + ssbNpmRegistryName + ' &&\n' +
611
      ssbNpmRegistryName + '/bootstrap/bin.js --ws-url ' + wsLink + ' \\\n' +
612
        pkgMsgs.map(function (id) {
613
          return '  --branch ' + id + ' \\\n'
614
        }).join('') +
615
        '  -- ' + npmCmd + ' &&\n' +
616
      'sbot server'
617
618
    self.res.writeHead(200, {'Content-type': 'text/plain'})
619
    self.res.end(script)
620
  })
621
}
622
623
Req.prototype.serveRobots = function () {
624
  this.res.writeHead(200, {'Content-type': 'text/plain'})
625
  this.res.end('User-agent: *\nDisallow: /\n')
626
}
627
628
Req.prototype.serveWhoami = function () {
629
  var self = this
630
  self.server.sbot.whoami(function (err, feed) {
631
    if (err) return self.respondError(500, err.stack || err)
632
    self.respond(200, {username: feed.id})
633
  })
634
}
635
636
Req.prototype.serveUser1 = function () {
637
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
638
}
639
640
function decodeName(name) {
641
  var parts = String(name).replace(/\.tgz$/, '').split(':')
642
  return {
643
    name: parts[1],
644
    version: parts[2],
645
    distTag: parts[3],
646
  }
647
}
648
649
Req.prototype.getMsgMentions = function (name) {
650
  return pull(
651
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
652
    // decryption could be done here
653
    pull.map(function (msg) {
654
      var c = msg.value && msg.value.content
655
      if (!c.mentions || !Array.isArray(c.mentions)) return []
656
      return c.mentions.map(function (mention) {
657
        return {
658
          name: mention.name,
659
          size: mention.size,
660
          link: mention.link,
661
          author: msg.value.author,
662
          ts: msg.value.timestamp,
663
        }
664
      })
665
    }),
666
    pull.flatten(),
667
    pull.filter(typeof name === 'string' ? function (link) {
668
      return link.name === name
669
    } : name && name.$prefix ? function (link) {
670
      return link.name.substr(0, name.$prefix.length) === name.$prefix
671
    } : function () {
672
      throw new TypeError('unsupported name filter')
673
    })
674
  )
675
}
676
677
Req.prototype.getMentions = function (name) {
678
  var useMsgMentions = this.headMsgIds
679
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
680
  if (useServerMentions && !this.server.sbot.links2) {
681
    return this.headMsgPlus
682
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
683
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
684
  }
685
  return cat([
686
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
687
    useServerMentions ? this.server.getMentions(name) : pull.empty()
688
  ])
689
}
690
691
Req.prototype.getMentionLinks = function (blobId) {
692
  return pull(
693
    this.headMsgIds
694
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
695
      : this.server.sbot.links({
696
        dest: blobId,
697
        rel: 'mentions',
698
        values: true,
699
      }),
700
    // decryption could be done here
701
    pull.map(function (msg) {
702
      var c = msg.value && msg.value.content
703
      return c && Array.isArray(c.mentions) && c.mentions || []
704
    }),
705
    pull.flatten(),
706
    pull.filter(function (link) {
707
      return link && link.link === blobId
708
    })
709
  )
710
}
711
712
Req.prototype.servePkg = function (pathname) {
713
  var self = this
714
  var parts = pathname.split('/')
715
  var pkgName = parts.shift().replace(/%2f/i, '/')
716
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
717
  var spec = parts.shift()
718
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
719
  if (parts.length > 0) return this.respondError(404)
720
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
721
  var obj = {
722
    _id: pkgName,
723
    name: pkgName,
724
    'dist-tags': {},
725
    versions: {}
726
  }
727
  var distTags = {/* <tag>: {version, ts}*/}
728
  pull(
729
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
730
    pull.drain(function (mention) {
731
      var data = decodeName(mention.name)
732
      if (!data.version) return
733
      if (data.distTag) {
734
        var tag = distTags[data.distTag]
735
        if (!tag || mention.ts > tag.ts) {
736
          /* TODO: sort by causal order instead of only by timestamps */
737
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
738
        }
739
      }
740
      obj.versions[data.version] = {
741
        author: {
742
          url: mention.author
743
        },
744
        name: pkgName,
745
        version: data.version,
746
        dist: self.server.blobDist(mention.link)
747
      }
748
    }, function (err) {
749
      if (err) return self.respondError(500, err.stack || err)
750
      for (var tag in distTags) {
751
        obj['dist-tags'][tag] = distTags[tag].version
752
      }
753
      if (spec) resolveSpec()
754
      else if (self.fetchAll) resolveAll()
755
      else resolved()
756
    })
757
  )
758
  function resolveSpec() {
759
    var version = obj['dist-tags'][spec]
760
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
761
    obj = obj.versions[version]
762
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
763
    self.populatePackageJson(obj, function (err, pkg) {
764
      if (err) return resolved(err)
765
      obj = pkg || obj
766
      resolved()
767
    })
768
  }
769
  function resolveVersion(version, cb) {
770
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
771
      if (err) return cb(err)
772
      if (pkg) obj.versions[version] = pkg
773
      cb()
774
    })
775
  }
776
  function resolveAll() {
777
    var done = multicb()
778
    for (var version in obj.versions) {
779
      resolveVersion(version, done())
780
    }
781
    done(resolved)
782
  }
783
  function resolved(err) {
784
    if (err) return self.respondError(500, err.stack || err)
785
    self.respond(200, obj)
786
  }
787
}
788
789
Req.prototype.servePrebuild = function (name) {
790
  var self = this
791
  var getMention = self.getMentions('prebuild:' + name)
792
  var blobsByAuthor = {/* <author>: BlobId */}
793
  getMention(null, function next(err, link) {
794
    if (err === true) return done()
795
    if (err) return self.respondError(500, err.stack || err)
796
    blobsByAuthor[link.author] = link.link
797
    getMention(null, next)
798
  })
799
  function done() {
800
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
801
    var blobId
802
    for (var feed in blobsByAuthor) {
803
      var blob = blobId = blobsByAuthor[feed]
804
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
805
      feeds.push(feed)
806
    }
807
    switch (Object.keys(authorsByLink).length) {
808
      case 0:
809
        return self.respondError(404, 'Not Found')
810
      case 1:
811
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
812
        return self.res.end()
813
      default:
814
        return self.respond(300, {choices: authorsByLink})
815
    }
816
  }
817
}
818
819
var localhosts = {
820
  '::1': true,
821
  '127.0.0.1': true,
822
  '::ffff:127.0.0.1': true,
823
}
824
825
Req.prototype.publishPkg = function (pkgName) {
826
  var self = this
827
  var remoteAddress = self.req.socket.remoteAddress
828
  if (!(remoteAddress in localhosts)) {
829
    return self.respondError(403, 'You may not publish as this user.')
830
  }
831
832
  var chunks = []
833
  self.req.on('data', function (data) {
834
    chunks.push(data)
835
  })
836
  self.req.on('end', function () {
837
    var data
838
    try {
839
      data = JSON.parse(Buffer.concat(chunks))
840
    } catch(e) {
841
      return self.respondError(400, e.stack)
842
    }
843
    return self.publishPkg2(pkgName, data || {})
844
  })
845
}
846
847
Req.prototype.publishPkg2 = function (name, data) {
848
  var self = this
849
  if (data.users) console.trace('[npm-registry] users property is not supported')
850
  var attachments = data._attachments || {}
851
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
852
  var done = multicb()
853
  function addAttachmentAsBlob(filename, cb) {
854
    var data = attachments[filename].data
855
    var tarball = new Buffer(data, 'base64')
856
    var length = attachments[filename].length
857
    if (length && length !== tarball.length) return self.respondError(400,
858
      'Length mismatch for attachment \'' + filename + '\'')
859
    self.server.sbot.blobs.add(function (err, id) {
860
      if (err) return cb(err)
861
      self.blobsToPush.push(id)
862
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
863
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
864
      cb()
865
    })(pull.once(tarball))
866
  }
867
  for (var filename in attachments) {
868
    addAttachmentAsBlob(filename, done())
869
  }
870
  done(function (err) {
871
    if (err) return self.respondError(500, err.stack || err)
872
    try {
873
      self.publishPkg3(name, data, links)
874
    } catch(e) {
875
      self.respondError(500, e.stack || e)
876
    }
877
  })
878
}
879
880
Req.prototype.publishPkg3 = function (name, data, links) {
881
  var self = this
882
  var versions = data.versions || {}
883
  var linksByVersion = {/* <version>: link */}
884
885
  // associate tarball blobs with versions
886
  for (var version in versions) {
887
    var pkg = versions[version]
888
    if (!pkg) return self.respondError(400, 'Bad package object')
889
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
890
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
891
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
892
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
893
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
894
    var filename = m[1]
895
    var link = links[filename]
896
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
897
    // TODO?: try to find missing tarball mentioned in other messages
898
    if (pkg.version && pkg.version !== version)
899
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
900
    linksByVersion[version] = link
901
    link.version = version
902
    link.dependencies = pkg.dependencies || {}
903
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
904
  }
905
906
  // associate blobs with dist-tags
907
  var tags = data['dist-tags'] || {}
908
  for (var tag in tags) {
909
    var version = tags[tag]
910
    var link = linksByVersion[version]
911
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
912
    // TODO?: support setting dist-tag without version,
913
    // by looking up a tarball blob for the version
914
    link.tag = tag
915
  }
916
917
  // compute blob links to publish
918
  var mentions = []
919
  for (var filename in links) {
920
    var link = links[filename] || {}
921
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
922
    mentions.push({
923
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
924
      link: link.link,
925
      size: link.size,
926
      shasum: link.shasum,
927
      dependencies: link.dependencies,
928
      bundledDependencies: link.bundledDependencies,
929
    })
930
  }
931
  return self.publishPkgs(mentions)
932
}
933
934
Req.prototype.publishPkgs = function (mentions) {
935
  var self = this
936
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
937
    if (err) return self.respondError(500, err.stack || err)
938
    self.server.pushBlobs(self.blobsToPush, function (err) {
939
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
940
      self.respond(201)
941
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
942
    })
943
  })
944
}
945
946
Req.prototype.populatePackageJson = function (obj, cb) {
947
  var self = this
948
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
949
  var deps, bundledDeps, shasum
950
951
  // look for dependencies in links.
952
  // then fallback to getting it from the tarball blob
953
954
  pull(
955
    self.getMentionLinks(blobId),
956
    pull.drain(function (link) {
957
      if (link.dependencies) deps = link.dependencies
958
      if (link.shasum) shasum = link.shasum
959
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
960
      // how to handle multiple assignments of dependencies to a package?
961
    }, function (err) {
962
      if (err) return cb(new Error(err.stack || err))
963
      if (deps && (shasum || !self.needShasum)) {
964
        // assume that the dependencies in the links to the blob are
965
        // correct.
966
        obj.dependencies = deps
967
        obj.bundledDependencies = bundledDeps
968
        if (shasum) obj.dist.shasum = obj._shasum = shasum
969
        cb(null, obj)
970
      } else {
971
        // get dependencies from the tarball
972
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
973
          if (err) return cb(err)
974
          pkg.dist = obj.dist
975
          pkg.dist.shasum = pkg._shasum
976
          pkg.author = pkg.author || obj.author
977
          pkg.version = pkg.version || obj.version
978
          pkg.name = pkg.name || obj.name
979
          cb(null, pkg)
980
        })
981
      }
982
    })
983
  )
984
}
985
986
function getPackageJsonFromTarballBlob(sbot, id, cb) {
987
  var self = this
988
  getBlob(sbot, id, function (err, readBlob) {
989
    if (err) return cb(err)
990
    cb = once(cb)
991
    var extract = tar.extract()
992
    var pkg, shasum
993
    extract.on('entry', function (header, stream, next) {
994
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
995
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
996
          if (err) return cb(err)
997
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
998
          catch(e) { return cb(e) }
999
          next()
1000
        }))
1001
      } else {
1002
        stream.on('end', next)
1003
        stream.resume()
1004
      }
1005
    })
1006
    extract.on('finish', function () {
1007
      pkg._shasum = shasum
1008
      cb(null, pkg)
1009
    })
1010
    pull(
1011
      readBlob,
1012
      hash('sha1', 'hex', function (err, sum) {
1013
        if (err) return cb(err)
1014
        shasum = sum
1015
      }),
1016
      toPull(zlib.createGunzip()),
1017
      toPull(extract)
1018
    )
1019
  })
1020
}
1021