git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
77
}
78
79
exports.name = 'npm-registry'
80
exports.version = '1.0.0'
81
exports.manifest = {
82
  getAddress: 'async'
83
}
84
exports.init = function (sbot, config) {
85
  var conf = config.npm || {}
86
  var port = conf.port || 8043
87
  var host = conf.host || null
88
  var autoAuth = conf.autoAuth !== false
89
90
  var server = http.createServer(exports.respond(sbot, config))
91
  var getAddress = onceify(function (cb) {
92
    server.on('error', cb)
93
    server.listen(port, host, function () {
94
      server.removeListener('error', cb)
95
      var regHost = formatHost(host || 'localhost')
96
      var regPort = this.address().port
97
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
98
      if (autoAuth) npmLogin(regUrl, next)
99
      else next()
100
      function next(err) {
101
        cb(err, regUrl)
102
      }
103
    })
104
    sbot.on('close', function () {
105
      server.close()
106
    })
107
  })
108
109
  getAddress(function (err, addr) {
110
    if (err) return console.error(err)
111
    console.log('[npm-registry] Listening on ' + addr)
112
  })
113
114
  return {
115
    getAddress: getAddress
116
  }
117
}
118
119
exports.respond = function (sbot, config) {
120
  var reg = new SsbNpmRegistryServer(sbot, config)
121
  return function (req, res) {
122
    new Req(reg, req, res).serve()
123
  }
124
}
125
126
function publishMsg(sbot, value, cb) {
127
  var gotExpectedPrevious = false
128
  sbot.publish(value, function next(err, msg) {
129
    if (err && /^expected previous:/.test(err.message)) {
130
      // retry once on this error
131
      if (gotExpectedPrevious) return cb(err)
132
      gotExpectedPrevious = true
133
      return sbot.publish(value, next)
134
    }
135
    cb(err, msg)
136
  })
137
}
138
139
function getDependencyBranches(sbot, link, cb) {
140
  var id = link.link
141
142
  // get ids of heads of tree of dependencyBranch message links to include all
143
  // the dependencies of the given tarball id.
144
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
145
  var msgs = {}
146
  var branched = {}
147
  var blobs = {}
148
149
  function addPkgById(id, cb) {
150
    if (blobs[id]) return cb()
151
    blobs[id] = true
152
    getPackageJsonCached(id, function (err, pkg) {
153
      if (err) return cb(err)
154
      var done = multicb()
155
      for (var name in pkg.dependencies || {}) {
156
        addPkgBySpec(name, pkg.dependencies[name], done())
157
      }
158
      done(cb)
159
    })
160
  }
161
162
  function addPkgBySpec(name, spec, cb) {
163
    var versions = {}
164
    var distTags = {}
165
    pull(
166
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
167
      pull.filter(mentionMatches(null, name, spec)),
168
      pull.map(function (mention) {
169
        // query to get the messages since links2 does not include the msg id
170
        return sbot.links({
171
          source: mention.author,
172
          dest: mention.link,
173
          rel: 'mentions',
174
          values: true,
175
        })
176
      }),
177
      pull.flatten(),
178
      pull.drain(function (msg) {
179
        var c = msg && msg.value && msg.value.content
180
        if (!c || !Array.isArray(c.mentions)) return
181
        c.mentions.forEach(function (link) {
182
          var data = link && link.name && decodeName(link.name)
183
          if (!data || data.name !== name) return
184
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
185
          if (data.distTag) distTags[data.distTag] = data.version
186
        })
187
      }, function (err) {
188
        if (err) return cb(err)
189
        var version = distTags[spec]
190
          || semver.maxSatisfying(Object.keys(versions), spec)
191
        var item = versions[version]
192
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
193
        addDepBranchMsg(item.msg)
194
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
195
        addPkgById(item.mention.link, cb)
196
      })
197
    )
198
  }
199
200
  function addDepBranchMsg(msg) {
201
    var c = msg && msg.value && msg.value.content
202
    if (!c) return
203
    if (msgs[msg.key]) return
204
    msgs[msg.key] = msg.value
205
    if (Array.isArray(c.dependencyBranch)) {
206
      for (var k = 0; k < c.dependencyBranch.length; k++) {
207
        branched[c.dependencyBranch[k]] = true
208
      }
209
    }
210
    if (Array.isArray(c.versionBranch)) {
211
      for (var k = 0; k < c.versionBranch.length; k++) {
212
        branched[c.versionBranch[k]] = true
213
      }
214
    }
215
  }
216
217
  function addPkgsByName(name, cb) {
218
    var data = decodeName(name)
219
    pull(
220
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
221
      pull.map(function (mention) {
222
        return packageLinks(sbot, mention.author, mention.link, data.name)
223
      }),
224
      pull.flatten(),
225
      pull.drain(addDepBranchMsg, cb)
226
    )
227
  }
228
229
  var done = multicb()
230
  addPkgsByName(link.name, done())
231
  addPkgById(id, done())
232
  done(function (err) {
233
    if (err) return cb(err)
234
    var ids = []
235
    for (var key in msgs) {
236
      if (!branched[key]) ids.push(key)
237
    }
238
    cb(null, ids)
239
  })
240
}
241
242
function packageLinks(sbot, feed, id, name, spec) {
243
  var matches = mentionMatches(id, name, spec)
244
  return pull(
245
    sbot.links({
246
      source: feed,
247
      dest: id,
248
      rel: 'mentions',
249
      values: true,
250
    }),
251
    pull.filter(function (msg) {
252
      var c = msg && msg.value && msg.value.content
253
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
254
    })
255
  )
256
}
257
258
// For each dependency that is not a bundledDependency, get message ids for
259
// that dependency name + version.
260
function publishSingleMention(sbot, mention, cb) {
261
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
262
  // Calculate dependencyBranch message ids.
263
  var value = {
264
    type: 'npm-packages',
265
    mentions: [mention]
266
  }
267
  var done = multicb({pluck: 1, spread: true})
268
  getDependencyBranches(sbot, mention, function (err, branches) {
269
    if (err) return cb(err)
270
    value.dependencyBranch = branches || undefined
271
    publishMsg(sbot, value, cb)
272
  })
273
}
274
275
function publishMentions(sbot, mentions, cb) {
276
  // console.error("publishing %s mentions", mentions.length)
277
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
278
  // if it is just one mention, fetch and add useful metadata
279
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
280
  publishMsg(sbot, {
281
    type: 'npm-packages',
282
    mentions: mentions,
283
  }, cb)
284
}
285
286
exports.publishPkgMentions = function (sbot, mentions, cb) {
287
  // try to fit the mentions into as few messages as possible,
288
  // while fitting under the message size limit.
289
  var msgs = []
290
  ;(function next(i, chunks) {
291
    if (i >= mentions.length) return cb(null, msgs)
292
    var chunkLen = Math.ceil(mentions.length / chunks)
293
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
294
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
295
      if (err && msgs.length) return onPartialPublish(err)
296
      if (err) return cb(err)
297
      msgs.push(msg)
298
      next(i + chunkLen, chunks)
299
    })
300
  })(0, 1)
301
  function onPartialPublish(err) {
302
    var remaining = mentions.length - i
303
    return cb(new Error('Published messages ' +
304
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
305
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
306
  }
307
}
308
309
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
310
  cb = once(cb)
311
  var waiting = 0
312
  var expandedMentions = mentions.map(function (link) {
313
    var id = link && link.link
314
    if (!id) return link
315
    waiting++
316
    var newLink = {}
317
    for (var k in link) newLink[k] = link[k]
318
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
319
      if (err) return cb(err)
320
      for (var k in props) newLink[k] = pkg[k]
321
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
322
      if (!--waiting) next()
323
    })
324
    return newLink
325
  })
326
  if (!waiting) next()
327
  function next() {
328
    cb(null, expandedMentions)
329
  }
330
}
331
332
function SsbNpmRegistryServer(sbot, config) {
333
  this.sbot = sbot
334
  this.config = config
335
  this.npmConfig = config.npm || {}
336
  this.host = this.npmConfig.host || 'localhost'
337
  this.fetchAll = this.npmConfig.fetchAll
338
  this.needShasum = this.npmConfig.needShasum
339
  var wsPort = config.ws && Number(config.ws.port) || '8989'
340
  this.wsLink = this.npmConfig.wsLink
341
    || ('http://' + (config.host || 'localhost') + ':' + wsPort)
342
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
343
}
344
345
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
346
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
347
348
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
349
  var self = this
350
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
351
  ;(function next(i) {
352
    if (i >= ids.length) return cb()
353
    self.sbot.blobs.push(ids[i], function (err) {
354
      if (err) return cb(err)
355
      next(i+1)
356
    })
357
  })(0)
358
}
359
360
function getBlob(sbot, id, cb) {
361
  var blobs = sbot.blobs
362
  blobs.size(id, function (err, size) {
363
    if (typeof size === 'number') cb(null, blobs.get(id))
364
    else blobs.want(id, function (err, got) {
365
      if (err) cb(err)
366
      else if (!got) cb('missing blob ' + id)
367
      else cb(null, blobs.get(id))
368
    })
369
  })
370
}
371
372
SsbNpmRegistryServer.prototype.blobDist = function (id) {
373
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
374
  if (!m) throw new Error('bad blob id: ' + id)
375
  return {
376
    integrity: m[2] + '-' + m[1],
377
    tarball: this.wsLink + '/blobs/get/' + id
378
  }
379
}
380
381
function getMentions(links2, name) {
382
  return links2.read({
383
    query: [
384
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
385
      {$filter: {dest: {$prefix: '&'}}},
386
      {$map: {
387
        name: ['rel', 1],
388
        size: ['rel', 2],
389
        link: 'dest',
390
        author: 'source',
391
        ts: 'ts'
392
      }}
393
    ]
394
  })
395
}
396
397
function mentionMatches(id, name, spec) {
398
  return function (mention) {
399
    var data = mention
400
      && (!id || id === mention.link)
401
      && mention.name
402
      && decodeName(mention.name)
403
    return data
404
      && data.name === name
405
      && (spec ? semver.satisfies(data.version, spec) : true)
406
  }
407
}
408
409
SsbNpmRegistryServer.prototype.getMentions = function (name) {
410
  if (!this.sbot.links2) return pull.empty()
411
  return getMentions(this.sbot.links2, name)
412
}
413
414
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
415
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
416
  else this.sbot.get(id, function (err, value) {
417
    if (err) return cb(err)
418
    cb(null, {key: id, value: value})
419
  })
420
}
421
422
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
423
  var self = this
424
  var stack = heads.slice()
425
  var seen = {}
426
  return function (abort, cb) {
427
    if (abort) return cb(abort)
428
    for (var id; stack.length && seen[id = stack.pop()];);
429
    if (!id) return cb(true)
430
    seen[id] = true
431
    // TODO: use DFS
432
    self.getMsg(id, function (err, msg) {
433
      if (err) return cb(new Error(err.stack || err))
434
      var c = msg && msg.value && msg.value.content
435
      var links = c && c[prop]
436
      if (Array.isArray(links)) {
437
        stack.push.apply(stack, links)
438
      } else if (links) {
439
        stack.push(links)
440
      }
441
      cb(null, msg)
442
    })
443
  }
444
}
445
446
function Req(server, req, res) {
447
  this.server = server
448
  this.req = req
449
  this.res = res
450
  this.blobsToPush = []
451
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
452
  var ua = this.req.headers['user-agent']
453
  var m = /\bnpm\/([0-9]*)/.exec(ua)
454
  var npmVersion = m && m[1]
455
  this.needShasum = server.needShasum != null ? server.needShasum :
456
    (npmVersion && npmVersion < 5)
457
  this.wsLink = this.server.wsLink
458
}
459
460
Req.prototype.serve = function () {
461
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
462
  this.res.setTimeout(0)
463
  var pathname = this.req.url.replace(/\?.*/, '')
464
  var m
465
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
466
    try {
467
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
468
      this.headMsgPlus = !!m[1]
469
      // ^ means also include packages published after the head message id
470
    } catch(e) {
471
      return this.respondError(400, e.stack || e)
472
    }
473
    pathname = m[3]
474
  }
475
  if (pathname === '/') return this.serveHome()
476
  if (pathname === '/robots.txt') return this.serveRobots()
477
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
478
  if (pathname === '/-/whoami') return this.serveWhoami()
479
  if (pathname === '/-/ping') return this.respond(200, true)
480
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
481
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
482
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
483
  return this.respond(404)
484
}
485
486
Req.prototype.respond = function (status, message) {
487
  this.res.writeHead(status, {'content-type': 'application/json'})
488
  this.res.end(message && JSON.stringify(message, 0, 2))
489
}
490
491
Req.prototype.respondError = function (status, message) {
492
  this.respond(status, {error: message})
493
}
494
495
Req.prototype.respondErrorStr = function (status, err) {
496
  this.res.writeHead(status, {'content-type': 'text/plain'})
497
  this.res.end(err.stack || err)
498
}
499
500
Req.prototype.serveHome = function () {
501
  var self = this
502
  self.res.writeHead(200, {'content-type': 'text/html'})
503
  var port = 8044
504
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
505
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
506
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
507
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
508
    '</body></html>')
509
}
510
511
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
512
  var self = this
513
  pull(
514
    self.server.sbot.links({
515
      source: feedId,
516
      dest: blobId,
517
      rel: 'mentions',
518
      values: true,
519
    }),
520
    pull.filter(function (msg) {
521
      var c = msg && msg.value && msg.value.content
522
      return c.type === 'npm-packages'
523
    }),
524
    pull.collect(function (err, msgs) {
525
      if (err) return cb(err)
526
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
527
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
528
      // TODO: make a smarter decision about which message id to use
529
      cb(null, msgs.pop().key)
530
    })
531
  )
532
}
533
534
Req.prototype.resolvePkg = function (pkgSpec, cb) {
535
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
536
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
537
  var self = this
538
  var pkgName = m[1]
539
  var spec = m[2] || '*'
540
  var versions = {}
541
  var distTags = {}
542
  pull(
543
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
544
    pull.drain(function (mention) {
545
      var data = decodeName(mention.name)
546
      if (!data.version) return
547
      if (data.distTag) {
548
        distTags[data.distTag] = data.version
549
      }
550
      versions[data.version] = {
551
        author: mention.author,
552
        name: pkgName,
553
        version: data.version,
554
        blobId: mention.link
555
      }
556
    }, function (err) {
557
      if (err) return cb(err)
558
      var version = distTags[spec]
559
        || semver.maxSatisfying(Object.keys(versions), spec)
560
      var item = versions[version]
561
      if (!item) return cb(new Error('Version not found: ' + name + '@' + spec))
562
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
563
        if (err) return cb(err)
564
        item.msgId = id
565
        cb(null, item)
566
      })
567
    })
568
  )
569
}
570
571
Req.prototype.resolvePkgs = function (specs, cb) {
572
  var done = multicb({pluck: 1})
573
  var self = this
574
  specs.forEach(function (spec) {
575
    self.resolvePkg(spec, done())
576
  })
577
  done(cb)
578
}
579
580
Req.prototype.serveBootstrap = function () {
581
  var self = this
582
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
583
  var ssbNpmRegistryName = require('./package.json').name
584
  var ssbNpmRegistryVersion = require('./package.json').version
585
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
586
  var done = multicb({pluck: 1, spread: true})
587
  self.resolvePkg(ssbNpmRegistrySpec, done())
588
  self.resolvePkgs(pkgs, done())
589
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
590
    if (err) return self.respondErrorStr(500, err.stack || err)
591
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
592
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
593
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
594
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
595
    var globalPkgs = pkgsInfo.map(function (info) {
596
      return info.name + '@' + info.version
597
    }).join(' ')
598
    var npmCmd = 'npm install -g ' + globalPkgs
599
    var pkgTmpText = '/tmp/' + ssbNpmRegistryName + '.tar.gz'
600
    var tarballLink = self.wsLink + '/blobs/get/' + ssbNpmRegistryBlobId
601
602
    var script =
603
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
604
      'echo \'' + ssbNpmRegistryBlobHex + '  ' + pkgTmpText + '\' | sha256sum -c &&\n' +
605
      'mkdir -p ~/.ssb/node_modules && cd ~/.ssb/node_modules &&\n' +
606
      'tar xzf ' + pkgTmpText + ' &&\n' +
607
      'mv package ' + ssbNpmRegistryName + ' &&\n' +
608
      ssbNpmRegistryName + '/bootstrap/bin.js --ws-url ' + self.wsLink + ' \\\n' +
609
        pkgMsgs.map(function (id) {
610
          return '  --branch ' + id + ' \\\n'
611
        }).join('') +
612
        '  -- ' + npmCmd + ' &&\n' +
613
      'sbot server'
614
615
    self.res.writeHead(200, {'Content-type': 'text/plain'})
616
    self.res.end(script)
617
  })
618
}
619
620
Req.prototype.serveRobots = function () {
621
  this.res.writeHead(200, {'Content-type': 'text/plain'})
622
  this.res.end('User-agent: *\nDisallow: /\n')
623
}
624
625
Req.prototype.serveWhoami = function () {
626
  var self = this
627
  self.server.sbot.whoami(function (err, feed) {
628
    if (err) return self.respondError(500, err.stack || err)
629
    self.respond(200, {username: feed.id})
630
  })
631
}
632
633
Req.prototype.serveUser1 = function () {
634
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
635
}
636
637
function decodeName(name) {
638
  var parts = String(name).replace(/\.tgz$/, '').split(':')
639
  return {
640
    name: parts[1],
641
    version: parts[2],
642
    distTag: parts[3],
643
  }
644
}
645
646
Req.prototype.getMsgMentions = function (name) {
647
  return pull(
648
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
649
    // decryption could be done here
650
    pull.map(function (msg) {
651
      var c = msg.value && msg.value.content
652
      if (!c.mentions || !Array.isArray(c.mentions)) return []
653
      return c.mentions.map(function (mention) {
654
        return {
655
          name: mention.name,
656
          size: mention.size,
657
          link: mention.link,
658
          author: msg.value.author,
659
          ts: msg.value.timestamp,
660
        }
661
      })
662
    }),
663
    pull.flatten(),
664
    pull.filter(typeof name === 'string' ? function (link) {
665
      return link.name === name
666
    } : name && name.$prefix ? function (link) {
667
      return link.name.substr(0, name.$prefix.length) === name.$prefix
668
    } : function () {
669
      throw new TypeError('unsupported name filter')
670
    })
671
  )
672
}
673
674
Req.prototype.getMentions = function (name) {
675
  var useMsgMentions = this.headMsgIds
676
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
677
  if (useServerMentions && !this.server.sbot.links2) {
678
    return this.headMsgPlus
679
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
680
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
681
  }
682
  return cat([
683
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
684
    useServerMentions ? this.server.getMentions(name) : pull.empty()
685
  ])
686
}
687
688
Req.prototype.getMentionLinks = function (blobId) {
689
  return pull(
690
    this.headMsgIds
691
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
692
      : this.server.sbot.links({
693
        dest: blobId,
694
        rel: 'mentions',
695
        values: true,
696
      }),
697
    // decryption could be done here
698
    pull.map(function (msg) {
699
      var c = msg.value && msg.value.content
700
      return c && Array.isArray(c.mentions) && c.mentions || []
701
    }),
702
    pull.flatten(),
703
    pull.filter(function (link) {
704
      return link && link.link === blobId
705
    })
706
  )
707
}
708
709
Req.prototype.servePkg = function (pathname) {
710
  var self = this
711
  var parts = pathname.split('/')
712
  var pkgName = parts.shift().replace(/%2f/i, '/')
713
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
714
  var spec = parts.shift()
715
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
716
  if (parts.length > 0) return this.respondError(404)
717
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
718
  var obj = {
719
    _id: pkgName,
720
    name: pkgName,
721
    'dist-tags': {},
722
    versions: {}
723
  }
724
  var distTags = {/* <tag>: {version, ts}*/}
725
  pull(
726
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
727
    pull.drain(function (mention) {
728
      var data = decodeName(mention.name)
729
      if (!data.version) return
730
      if (data.distTag) {
731
        var tag = distTags[data.distTag]
732
        if (!tag || mention.ts > tag.ts) {
733
          /* TODO: sort by causal order instead of only by timestamps */
734
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
735
        }
736
      }
737
      obj.versions[data.version] = {
738
        author: {
739
          url: mention.author
740
        },
741
        name: pkgName,
742
        version: data.version,
743
        dist: self.server.blobDist(mention.link)
744
      }
745
    }, function (err) {
746
      if (err) return self.respondError(500, err.stack || err)
747
      for (var tag in distTags) {
748
        obj['dist-tags'][tag] = distTags[tag].version
749
      }
750
      if (spec) resolveSpec()
751
      else if (self.fetchAll) resolveAll()
752
      else resolved()
753
    })
754
  )
755
  function resolveSpec() {
756
    var version = obj['dist-tags'][spec]
757
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
758
    obj = obj.versions[version]
759
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
760
    self.populatePackageJson(obj, function (err, pkg) {
761
      if (err) return resolved(err)
762
      obj = pkg || obj
763
      resolved()
764
    })
765
  }
766
  function resolveVersion(version, cb) {
767
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
768
      if (err) return cb(err)
769
      if (pkg) obj.versions[version] = pkg
770
      cb()
771
    })
772
  }
773
  function resolveAll() {
774
    var done = multicb()
775
    for (var version in obj.versions) {
776
      resolveVersion(version, done())
777
    }
778
    done(resolved)
779
  }
780
  function resolved(err) {
781
    if (err) return self.respondError(500, err.stack || err)
782
    self.respond(200, obj)
783
  }
784
}
785
786
Req.prototype.servePrebuild = function (name) {
787
  var self = this
788
  var getMention = self.getMentions('prebuild:' + name)
789
  var blobsByAuthor = {/* <author>: BlobId */}
790
  getMention(null, function next(err, link) {
791
    if (err === true) return done()
792
    if (err) return self.respondError(500, err.stack || err)
793
    blobsByAuthor[link.author] = link.link
794
    getMention(null, next)
795
  })
796
  function done() {
797
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
798
    var blobId
799
    for (var feed in blobsByAuthor) {
800
      var blob = blobId = blobsByAuthor[feed]
801
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
802
      feeds.push(feed)
803
    }
804
    switch (Object.keys(authorsByLink).length) {
805
      case 0:
806
        return self.respondError(404, 'Not Found')
807
      case 1:
808
        self.res.writeHead(303, {Location: self.wsLink + '/blobs/get/' + blobId})
809
        return self.res.end()
810
      default:
811
        return self.respond(300, {choices: authorsByLink})
812
    }
813
  }
814
}
815
816
var localhosts = {
817
  '::1': true,
818
  '127.0.0.1': true,
819
  '::ffff:127.0.0.1': true,
820
}
821
822
Req.prototype.publishPkg = function (pkgName) {
823
  var self = this
824
  var remoteAddress = self.req.socket.remoteAddress
825
  if (!(remoteAddress in localhosts)) {
826
    return self.respondError(403, 'You may not publish as this user.')
827
  }
828
829
  var chunks = []
830
  self.req.on('data', function (data) {
831
    chunks.push(data)
832
  })
833
  self.req.on('end', function () {
834
    var data
835
    try {
836
      data = JSON.parse(Buffer.concat(chunks))
837
    } catch(e) {
838
      return self.respondError(400, e.stack)
839
    }
840
    return self.publishPkg2(pkgName, data || {})
841
  })
842
}
843
844
Req.prototype.publishPkg2 = function (name, data) {
845
  var self = this
846
  if (data.users) console.trace('[npm-registry] users property is not supported')
847
  var attachments = data._attachments || {}
848
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
849
  var done = multicb()
850
  function addAttachmentAsBlob(filename, cb) {
851
    var data = attachments[filename].data
852
    var tarball = new Buffer(data, 'base64')
853
    var length = attachments[filename].length
854
    if (length && length !== tarball.length) return self.respondError(400,
855
      'Length mismatch for attachment \'' + filename + '\'')
856
    self.server.sbot.blobs.add(function (err, id) {
857
      if (err) return cb(err)
858
      self.blobsToPush.push(id)
859
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
860
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
861
      cb()
862
    })(pull.once(tarball))
863
  }
864
  for (var filename in attachments) {
865
    addAttachmentAsBlob(filename, done())
866
  }
867
  done(function (err) {
868
    if (err) return self.respondError(500, err.stack || err)
869
    try {
870
      self.publishPkg3(name, data, links)
871
    } catch(e) {
872
      self.respondError(500, e.stack || e)
873
    }
874
  })
875
}
876
877
Req.prototype.publishPkg3 = function (name, data, links) {
878
  var self = this
879
  var versions = data.versions || {}
880
  var linksByVersion = {/* <version>: link */}
881
882
  // associate tarball blobs with versions
883
  for (var version in versions) {
884
    var pkg = versions[version]
885
    if (!pkg) return self.respondError(400, 'Bad package object')
886
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
887
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
888
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
889
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
890
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
891
    var filename = m[1]
892
    var link = links[filename]
893
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
894
    // TODO?: try to find missing tarball mentioned in other messages
895
    if (pkg.version && pkg.version !== version)
896
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
897
    linksByVersion[version] = link
898
    link.version = version
899
    link.dependencies = pkg.dependencies || {}
900
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
901
  }
902
903
  // associate blobs with dist-tags
904
  var tags = data['dist-tags'] || {}
905
  for (var tag in tags) {
906
    var version = tags[tag]
907
    var link = linksByVersion[version]
908
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
909
    // TODO?: support setting dist-tag without version,
910
    // by looking up a tarball blob for the version
911
    link.tag = tag
912
  }
913
914
  // compute blob links to publish
915
  var mentions = []
916
  for (var filename in links) {
917
    var link = links[filename] || {}
918
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
919
    mentions.push({
920
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
921
      link: link.link,
922
      size: link.size,
923
      shasum: link.shasum,
924
      dependencies: link.dependencies,
925
      bundledDependencies: link.bundledDependencies,
926
    })
927
  }
928
  return self.publishPkgs(mentions)
929
}
930
931
Req.prototype.publishPkgs = function (mentions) {
932
  var self = this
933
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
934
    if (err) return self.respondError(500, err.stack || err)
935
    self.server.pushBlobs(self.blobsToPush, function (err) {
936
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
937
      self.respond(201)
938
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
939
    })
940
  })
941
}
942
943
Req.prototype.populatePackageJson = function (obj, cb) {
944
  var self = this
945
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
946
  var deps, bundledDeps, shasum
947
948
  // look for dependencies in links.
949
  // then fallback to getting it from the tarball blob
950
951
  pull(
952
    self.getMentionLinks(blobId),
953
    pull.drain(function (link) {
954
      if (link.dependencies) deps = link.dependencies
955
      if (link.shasum) shasum = link.shasum
956
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
957
      // how to handle multiple assignments of dependencies to a package?
958
    }, function (err) {
959
      if (err) return cb(new Error(err.stack || err))
960
      if (deps && (shasum || !self.needShasum)) {
961
        // assume that the dependencies in the links to the blob are
962
        // correct.
963
        obj.dependencies = deps
964
        obj.bundledDependencies = bundledDeps
965
        if (shasum) obj.dist.shasum = obj._shasum = shasum
966
        cb(null, obj)
967
      } else {
968
        // get dependencies from the tarball
969
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
970
          if (err) return cb(err)
971
          pkg.dist = obj.dist
972
          pkg.dist.shasum = pkg._shasum
973
          pkg.author = pkg.author || obj.author
974
          pkg.version = pkg.version || obj.version
975
          pkg.name = pkg.name || obj.name
976
          cb(null, pkg)
977
        })
978
      }
979
    })
980
  )
981
}
982
983
function getPackageJsonFromTarballBlob(sbot, id, cb) {
984
  var self = this
985
  getBlob(sbot, id, function (err, readBlob) {
986
    if (err) return cb(err)
987
    cb = once(cb)
988
    var extract = tar.extract()
989
    var pkg, shasum
990
    extract.on('entry', function (header, stream, next) {
991
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
992
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
993
          if (err) return cb(err)
994
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
995
          catch(e) { return cb(e) }
996
          next()
997
        }))
998
      } else {
999
        stream.on('end', next)
1000
        stream.resume()
1001
      }
1002
    })
1003
    extract.on('finish', function () {
1004
      pkg._shasum = shasum
1005
      cb(null, pkg)
1006
    })
1007
    pull(
1008
      readBlob,
1009
      hash('sha1', 'hex', function (err, sum) {
1010
        if (err) return cb(err)
1011
        shasum = sum
1012
      }),
1013
      toPull(zlib.createGunzip()),
1014
      toPull(extract)
1015
    )
1016
  })
1017
}
1018