git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
77
}
78
79
exports.name = 'npm-registry'
80
exports.version = '1.0.0'
81
exports.manifest = {
82
  getAddress: 'async'
83
}
84
exports.init = function (sbot, config) {
85
  var conf = config.npm || {}
86
  var port = conf.port || 8043
87
  var host = conf.host || null
88
  var autoAuth = conf.autoAuth !== false
89
90
  var server = http.createServer(exports.respond(sbot, config))
91
  var getAddress = onceify(function (cb) {
92
    server.on('error', cb)
93
    server.listen(port, host, function () {
94
      server.removeListener('error', cb)
95
      var regHost = formatHost(host || 'localhost')
96
      var regPort = this.address().port
97
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
98
      if (autoAuth) npmLogin(regUrl, next)
99
      else next()
100
      function next(err) {
101
        cb(err, regUrl)
102
      }
103
    })
104
    sbot.on('close', function () {
105
      server.close()
106
    })
107
  })
108
109
  getAddress(function (err, addr) {
110
    if (err) return console.error(err)
111
    console.log('[npm-registry] Listening on ' + addr)
112
  })
113
114
  return {
115
    getAddress: getAddress
116
  }
117
}
118
119
exports.respond = function (sbot, config) {
120
  var reg = new SsbNpmRegistryServer(sbot, config)
121
  return function (req, res) {
122
    new Req(reg, req, res).serve()
123
  }
124
}
125
126
function publishMsg(sbot, value, cb) {
127
  var gotExpectedPrevious = false
128
  sbot.publish(value, function next(err, msg) {
129
    if (err && /^expected previous:/.test(err.message)) {
130
      // retry once on this error
131
      if (gotExpectedPrevious) return cb(err)
132
      gotExpectedPrevious = true
133
      return sbot.publish(value, next)
134
    }
135
    cb(err, msg)
136
  })
137
}
138
139
function getDependencyBranches(sbot, link, cb) {
140
  var id = link.link
141
142
  // get ids of heads of tree of dependencyBranch message links to include all
143
  // the dependencies of the given tarball id.
144
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
145
  var msgs = {}
146
  var branched = {}
147
  var blobs = {}
148
149
  function addPkgById(id, cb) {
150
    if (blobs[id]) return cb()
151
    blobs[id] = true
152
    getPackageJsonCached(id, function (err, pkg) {
153
      if (err) return cb(err)
154
      var done = multicb()
155
      for (var name in pkg.dependencies || {}) {
156
        addPkgBySpec(name, pkg.dependencies[name], done())
157
      }
158
      done(cb)
159
    })
160
  }
161
162
  function addPkgBySpec(name, spec, cb) {
163
    var versions = {}
164
    var distTags = {}
165
    pull(
166
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
167
      pull.filter(mentionMatches(null, name, spec)),
168
      pull.map(function (mention) {
169
        // query to get the messages since links2 does not include the msg id
170
        return sbot.links({
171
          source: mention.author,
172
          dest: mention.link,
173
          rel: 'mentions',
174
          values: true,
175
        })
176
      }),
177
      pull.flatten(),
178
      pull.drain(function (msg) {
179
        var c = msg && msg.value && msg.value.content
180
        if (!c || !Array.isArray(c.mentions)) return
181
        c.mentions.forEach(function (link) {
182
          var data = link && link.name && decodeName(link.name)
183
          if (!data || data.name !== name) return
184
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
185
          if (data.distTag) distTags[data.distTag] = data.version
186
        })
187
      }, function (err) {
188
        if (err) return cb(err)
189
        var version = distTags[spec]
190
          || semver.maxSatisfying(Object.keys(versions), spec)
191
        var item = versions[version]
192
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
193
        addDepBranchMsg(item.msg)
194
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
195
        addPkgById(item.mention.link, cb)
196
      })
197
    )
198
  }
199
200
  function addDepBranchMsg(msg) {
201
    var c = msg && msg.value && msg.value.content
202
    if (!c) return
203
    if (msgs[msg.key]) return
204
    msgs[msg.key] = msg.value
205
    if (Array.isArray(c.dependencyBranch)) {
206
      for (var k = 0; k < c.dependencyBranch.length; k++) {
207
        branched[c.dependencyBranch[k]] = true
208
      }
209
    }
210
    if (Array.isArray(c.versionBranch)) {
211
      for (var k = 0; k < c.versionBranch.length; k++) {
212
        branched[c.versionBranch[k]] = true
213
      }
214
    }
215
  }
216
217
  function addPkgsByName(name, cb) {
218
    var data = decodeName(name)
219
    pull(
220
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
221
      pull.map(function (mention) {
222
        return packageLinks(sbot, mention.author, mention.link, data.name)
223
      }),
224
      pull.flatten(),
225
      pull.drain(addDepBranchMsg, cb)
226
    )
227
  }
228
229
  var done = multicb()
230
  addPkgsByName(link.name, done())
231
  addPkgById(id, done())
232
  done(function (err) {
233
    if (err) return cb(err)
234
    var ids = []
235
    for (var key in msgs) {
236
      if (!branched[key]) ids.push(key)
237
    }
238
    cb(null, ids)
239
  })
240
}
241
242
function packageLinks(sbot, feed, id, name, spec) {
243
  var matches = mentionMatches(id, name, spec)
244
  return pull(
245
    sbot.links({
246
      source: feed,
247
      dest: id,
248
      rel: 'mentions',
249
      values: true,
250
    }),
251
    pull.filter(function (msg) {
252
      var c = msg && msg.value && msg.value.content
253
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
254
    })
255
  )
256
}
257
258
// For each dependency that is not a bundledDependency, get message ids for
259
// that dependency name + version.
260
function publishSingleMention(sbot, mention, cb) {
261
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
262
  // Calculate dependencyBranch message ids.
263
  var value = {
264
    type: 'npm-packages',
265
    mentions: [mention]
266
  }
267
  var done = multicb({pluck: 1, spread: true})
268
  getDependencyBranches(sbot, mention, function (err, branches) {
269
    if (err) return cb(err)
270
    value.dependencyBranch = branches || undefined
271
    publishMsg(sbot, value, cb)
272
  })
273
}
274
275
function publishMentions(sbot, mentions, cb) {
276
  // console.error("publishing %s mentions", mentions.length)
277
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
278
  // if it is just one mention, fetch and add useful metadata
279
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
280
  publishMsg(sbot, {
281
    type: 'npm-packages',
282
    mentions: mentions,
283
  }, cb)
284
}
285
286
exports.publishPkgMentions = function (sbot, mentions, cb) {
287
  // try to fit the mentions into as few messages as possible,
288
  // while fitting under the message size limit.
289
  var msgs = []
290
  ;(function next(i, chunks) {
291
    if (i >= mentions.length) return cb(null, msgs)
292
    var chunkLen = Math.ceil(mentions.length / chunks)
293
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
294
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
295
      if (err && msgs.length) return onPartialPublish(err)
296
      if (err) return cb(err)
297
      msgs.push(msg)
298
      next(i + chunkLen, chunks)
299
    })
300
  })(0, 1)
301
  function onPartialPublish(err) {
302
    var remaining = mentions.length - i
303
    return cb(new Error('Published messages ' +
304
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
305
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
306
  }
307
}
308
309
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
310
  cb = once(cb)
311
  var waiting = 0
312
  var expandedMentions = mentions.map(function (link) {
313
    var id = link && link.link
314
    if (!id) return link
315
    waiting++
316
    var newLink = {}
317
    for (var k in link) newLink[k] = link[k]
318
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
319
      if (err) return cb(err)
320
      for (var k in props) newLink[k] = pkg[k]
321
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
322
      if (!--waiting) next()
323
    })
324
    return newLink
325
  })
326
  if (!waiting) next()
327
  function next() {
328
    cb(null, expandedMentions)
329
  }
330
}
331
332
function SsbNpmRegistryServer(sbot, config) {
333
  this.sbot = sbot
334
  this.config = config
335
  this.npmConfig = config.npm || {}
336
  this.host = this.npmConfig.host || 'localhost'
337
  this.fetchAll = this.npmConfig.fetchAll
338
  this.needShasum = this.npmConfig.needShasum
339
  var wsPort = config.ws && Number(config.ws.port) || '8989'
340
  this.wsLink = 'http://' + (config.host || 'localhost') + ':' + wsPort
341
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
342
}
343
344
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
345
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
346
347
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
348
  var self = this
349
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
350
  ;(function next(i) {
351
    if (i >= ids.length) return cb()
352
    self.sbot.blobs.push(ids[i], function (err) {
353
      if (err) return cb(err)
354
      next(i+1)
355
    })
356
  })(0)
357
}
358
359
function getBlob(sbot, id, cb) {
360
  var blobs = sbot.blobs
361
  blobs.size(id, function (err, size) {
362
    if (typeof size === 'number') cb(null, blobs.get(id))
363
    else blobs.want(id, function (err, got) {
364
      if (err) cb(err)
365
      else if (!got) cb('missing blob ' + id)
366
      else cb(null, blobs.get(id))
367
    })
368
  })
369
}
370
371
SsbNpmRegistryServer.prototype.blobDist = function (id) {
372
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
373
  if (!m) throw new Error('bad blob id: ' + id)
374
  return {
375
    integrity: m[2] + '-' + m[1],
376
    tarball: this.wsLink + '/blobs/get/' + id
377
  }
378
}
379
380
function getMentions(links2, name) {
381
  return links2.read({
382
    query: [
383
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
384
      {$filter: {dest: {$prefix: '&'}}},
385
      {$map: {
386
        name: ['rel', 1],
387
        size: ['rel', 2],
388
        link: 'dest',
389
        author: 'source',
390
        ts: 'ts'
391
      }}
392
    ]
393
  })
394
}
395
396
function mentionMatches(id, name, spec) {
397
  return function (mention) {
398
    var data = mention
399
      && (!id || id === mention.link)
400
      && mention.name
401
      && decodeName(mention.name)
402
    return data
403
      && data.name === name
404
      && (spec ? semver.satisfies(data.version, spec) : true)
405
  }
406
}
407
408
SsbNpmRegistryServer.prototype.getMentions = function (name) {
409
  if (!this.sbot.links2) return pull.empty()
410
  return getMentions(this.sbot.links2, name)
411
}
412
413
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
414
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
415
  else this.sbot.get(id, function (err, value) {
416
    if (err) return cb(err)
417
    cb(null, {key: id, value: value})
418
  })
419
}
420
421
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
422
  var self = this
423
  var stack = heads.slice()
424
  var seen = {}
425
  return function (abort, cb) {
426
    if (abort) return cb(abort)
427
    for (var id; stack.length && seen[id = stack.pop()];);
428
    if (!id) return cb(true)
429
    seen[id] = true
430
    // TODO: use DFS
431
    self.getMsg(id, function (err, msg) {
432
      if (err) return cb(new Error(err.stack || err))
433
      var c = msg && msg.value && msg.value.content
434
      var links = c && c[prop]
435
      if (Array.isArray(links)) {
436
        stack.push.apply(stack, links)
437
      } else if (links) {
438
        stack.push(links)
439
      }
440
      cb(null, msg)
441
    })
442
  }
443
}
444
445
function Req(server, req, res) {
446
  this.server = server
447
  this.req = req
448
  this.res = res
449
  this.blobsToPush = []
450
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
451
  var ua = this.req.headers['user-agent']
452
  var m = /\bnpm\/([0-9]*)/.exec(ua)
453
  var npmVersion = m && m[1]
454
  this.needShasum = server.needShasum != null ? server.needShasum :
455
    (npmVersion && npmVersion < 5)
456
  this.wsLink = this.server.wsLink
457
}
458
459
Req.prototype.serve = function () {
460
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
461
  this.res.setTimeout(0)
462
  var pathname = this.req.url.replace(/\?.*/, '')
463
  var m
464
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
465
    try {
466
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
467
      this.headMsgPlus = !!m[1]
468
      // ^ means also include packages published after the head message id
469
    } catch(e) {
470
      return this.respondError(400, e.stack || e)
471
    }
472
    pathname = m[3]
473
  }
474
  if (pathname === '/') return this.serveHome()
475
  if (pathname === '/bootstrap') return this.serveBootstrap()
476
  if (pathname === '/robots.txt') return this.serveRobots()
477
  if (pathname === '/-/whoami') return this.serveWhoami()
478
  if (pathname === '/-/ping') return this.respond(200, true)
479
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
480
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
481
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
482
  return this.respond(404)
483
}
484
485
Req.prototype.respond = function (status, message) {
486
  this.res.writeHead(status, {'content-type': 'application/json'})
487
  this.res.end(message && JSON.stringify(message, 0, 2))
488
}
489
490
Req.prototype.respondError = function (status, message) {
491
  this.respond(status, {error: message})
492
}
493
494
Req.prototype.respondErrorStr = function (status, err) {
495
  this.res.writeHead(status, {'content-type': 'text/plain'})
496
  this.res.end(err.stack || err)
497
}
498
499
Req.prototype.serveHome = function () {
500
  var self = this
501
  self.res.writeHead(200, {'content-type': 'text/html'})
502
  var port = 8044
503
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
504
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
505
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
506
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
507
    '</body></html>')
508
}
509
510
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
511
  var self = this
512
  pull(
513
    self.server.sbot.links({
514
      source: feedId,
515
      dest: blobId,
516
      rel: 'mentions',
517
      values: true,
518
    }),
519
    pull.filter(function (msg) {
520
      var c = msg && msg.value && msg.value.content
521
      return c.type === 'npm-packages'
522
    }),
523
    pull.collect(function (err, msgs) {
524
      if (err) return cb(err)
525
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
526
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
527
      // TODO: make a smarter decision about which message id to use
528
      cb(null, msgs.pop().key)
529
    })
530
  )
531
}
532
533
Req.prototype.resolvePkg = function (pkgSpec, cb) {
534
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
535
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
536
  var self = this
537
  var pkgName = m[1]
538
  var spec = m[2] || '*'
539
  var versions = {}
540
  var distTags = {}
541
  pull(
542
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
543
    pull.drain(function (mention) {
544
      var data = decodeName(mention.name)
545
      if (!data.version) return
546
      if (data.distTag) {
547
        distTags[data.distTag] = data.version
548
      }
549
      versions[data.version] = {
550
        author: mention.author,
551
        name: pkgName,
552
        version: data.version,
553
        blobId: mention.link
554
      }
555
    }, function (err) {
556
      if (err) return cb(err)
557
      var version = distTags[spec]
558
        || semver.maxSatisfying(Object.keys(versions), spec)
559
      var item = versions[version]
560
      if (!item) return cb(new Error('Version not found: ' + name + '@' + spec))
561
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
562
        if (err) return cb(err)
563
        item.msgId = id
564
        cb(null, item)
565
      })
566
    })
567
  )
568
}
569
570
Req.prototype.resolvePkgs = function (specs, cb) {
571
  var done = multicb({pluck: 1})
572
  var self = this
573
  specs.forEach(function (spec) {
574
    self.resolvePkg(spec, done())
575
  })
576
  done(cb)
577
}
578
579
Req.prototype.serveBootstrap = function () {
580
  var self = this
581
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
582
  var ssbNpmRegistryName = require('./package.json').name
583
  var ssbNpmRegistryVersion = require('./package.json').version
584
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
585
  var done = multicb({pluck: 1, spread: true})
586
  self.resolvePkg(ssbNpmRegistrySpec, done())
587
  self.resolvePkgs(pkgs, done())
588
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
589
    if (err) return self.respondErrorStr(500, err.stack || err)
590
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
591
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
592
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
593
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
594
    var globalPkgs = pkgsInfo.map(function (info) {
595
      return info.name + '@' + info.version
596
    }).join(' ')
597
    var npmCmd = 'npm install -g ' + globalPkgs
598
    var pkgTmpText = '/tmp/' + ssbNpmRegistryName + '.tar.gz'
599
    var tarballLink = self.wsLink + '/blobs/get/' + ssbNpmRegistryBlobId
600
601
    var script =
602
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
603
      'echo \'' + ssbNpmRegistryBlobHex + '  ' + pkgTmpText + '\' | sha256sum -c &&\n' +
604
      'mkdir -p ~/.ssb/node_modules && cd ~/.ssb/node_modules &&\n' +
605
      'tar xzf ' + pkgTmpText + ' &&\n' +
606
      'mv package ' + ssbNpmRegistryName + ' &&\n' +
607
      ssbNpmRegistryName + '/bootstrap/bin.js --ws-url ' + self.wsLink + ' \\\n' +
608
        pkgMsgs.map(function (id) {
609
          return '  --branch ' + id + ' \\\n'
610
        }).join('') +
611
        '  -- ' + npmCmd + ' &&\n' +
612
      'sbot server'
613
614
    self.res.writeHead(200, {'Content-type': 'text/plain'})
615
    self.res.end(script)
616
  })
617
}
618
619
Req.prototype.serveRobots = function () {
620
  this.res.writeHead(200, {'Content-type': 'text/plain'})
621
  this.res.end('User-agent: *\nDisallow: /\n')
622
}
623
624
Req.prototype.serveWhoami = function () {
625
  var self = this
626
  self.server.sbot.whoami(function (err, feed) {
627
    if (err) return self.respondError(500, err.stack || err)
628
    self.respond(200, {username: feed.id})
629
  })
630
}
631
632
Req.prototype.serveUser1 = function () {
633
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
634
}
635
636
function decodeName(name) {
637
  var parts = String(name).replace(/\.tgz$/, '').split(':')
638
  return {
639
    name: parts[1],
640
    version: parts[2],
641
    distTag: parts[3],
642
  }
643
}
644
645
Req.prototype.getMsgMentions = function (name) {
646
  return pull(
647
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
648
    // decryption could be done here
649
    pull.map(function (msg) {
650
      var c = msg.value && msg.value.content
651
      if (!c.mentions || !Array.isArray(c.mentions)) return []
652
      return c.mentions.map(function (mention) {
653
        return {
654
          name: mention.name,
655
          size: mention.size,
656
          link: mention.link,
657
          author: msg.value.author,
658
          ts: msg.value.timestamp,
659
        }
660
      })
661
    }),
662
    pull.flatten(),
663
    pull.filter(typeof name === 'string' ? function (link) {
664
      return link.name === name
665
    } : name && name.$prefix ? function (link) {
666
      return link.name.substr(0, name.$prefix.length) === name.$prefix
667
    } : function () {
668
      throw new TypeError('unsupported name filter')
669
    })
670
  )
671
}
672
673
Req.prototype.getMentions = function (name) {
674
  var useMsgMentions = this.headMsgIds
675
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
676
  if (useServerMentions && !this.server.sbot.links2) {
677
    return this.headMsgPlus
678
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
679
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
680
  }
681
  return cat([
682
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
683
    useServerMentions ? this.server.getMentions(name) : pull.empty()
684
  ])
685
}
686
687
Req.prototype.getMentionLinks = function (blobId) {
688
  return pull(
689
    this.headMsgIds
690
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
691
      : this.server.sbot.links({
692
        dest: blobId,
693
        rel: 'mentions',
694
        values: true,
695
      }),
696
    // decryption could be done here
697
    pull.map(function (msg) {
698
      var c = msg.value && msg.value.content
699
      return c && Array.isArray(c.mentions) && c.mentions || []
700
    }),
701
    pull.flatten(),
702
    pull.filter(function (link) {
703
      return link && link.link === blobId
704
    })
705
  )
706
}
707
708
Req.prototype.servePkg = function (pathname) {
709
  var self = this
710
  var parts = pathname.split('/')
711
  var pkgName = parts.shift().replace(/%2f/i, '/')
712
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
713
  var spec = parts.shift()
714
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
715
  if (parts.length > 0) return this.respondError(404)
716
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
717
  var obj = {
718
    _id: pkgName,
719
    name: pkgName,
720
    'dist-tags': {},
721
    versions: {}
722
  }
723
  var distTags = {/* <tag>: {version, ts}*/}
724
  pull(
725
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
726
    pull.drain(function (mention) {
727
      var data = decodeName(mention.name)
728
      if (!data.version) return
729
      if (data.distTag) {
730
        var tag = distTags[data.distTag]
731
        if (!tag || mention.ts > tag.ts) {
732
          /* TODO: sort by causal order instead of only by timestamps */
733
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
734
        }
735
      }
736
      obj.versions[data.version] = {
737
        author: {
738
          url: mention.author
739
        },
740
        name: pkgName,
741
        version: data.version,
742
        dist: self.server.blobDist(mention.link)
743
      }
744
    }, function (err) {
745
      if (err) return self.respondError(500, err.stack || err)
746
      for (var tag in distTags) {
747
        obj['dist-tags'][tag] = distTags[tag].version
748
      }
749
      if (spec) resolveSpec()
750
      else if (self.fetchAll) resolveAll()
751
      else resolved()
752
    })
753
  )
754
  function resolveSpec() {
755
    var version = obj['dist-tags'][spec]
756
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
757
    obj = obj.versions[version]
758
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
759
    self.populatePackageJson(obj, function (err, pkg) {
760
      if (err) return resolved(err)
761
      obj = pkg || obj
762
      resolved()
763
    })
764
  }
765
  function resolveVersion(version, cb) {
766
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
767
      if (err) return cb(err)
768
      if (pkg) obj.versions[version] = pkg
769
      cb()
770
    })
771
  }
772
  function resolveAll() {
773
    var done = multicb()
774
    for (var version in obj.versions) {
775
      resolveVersion(version, done())
776
    }
777
    done(resolved)
778
  }
779
  function resolved(err) {
780
    if (err) return self.respondError(500, err.stack || err)
781
    self.respond(200, obj)
782
  }
783
}
784
785
Req.prototype.servePrebuild = function (name) {
786
  var self = this
787
  var getMention = self.getMentions('prebuild:' + name)
788
  var blobsByAuthor = {/* <author>: BlobId */}
789
  getMention(null, function next(err, link) {
790
    if (err === true) return done()
791
    if (err) return self.respondError(500, err.stack || err)
792
    blobsByAuthor[link.author] = link.link
793
    getMention(null, next)
794
  })
795
  function done() {
796
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
797
    var blobId
798
    for (var feed in blobsByAuthor) {
799
      var blob = blobId = blobsByAuthor[feed]
800
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
801
      feeds.push(feed)
802
    }
803
    switch (Object.keys(authorsByLink).length) {
804
      case 0:
805
        return self.respondError(404, 'Not Found')
806
      case 1:
807
        self.res.writeHead(303, {Location: self.wsLink + '/blobs/get/' + blobId})
808
        return self.res.end()
809
      default:
810
        return self.respond(300, {choices: authorsByLink})
811
    }
812
  }
813
}
814
815
var localhosts = {
816
  '::1': true,
817
  '127.0.0.1': true,
818
  '::ffff:127.0.0.1': true,
819
}
820
821
Req.prototype.publishPkg = function (pkgName) {
822
  var self = this
823
  var remoteAddress = self.req.socket.remoteAddress
824
  if (!(remoteAddress in localhosts)) {
825
    return self.respondError(403, 'You may not publish as this user.')
826
  }
827
828
  var chunks = []
829
  self.req.on('data', function (data) {
830
    chunks.push(data)
831
  })
832
  self.req.on('end', function () {
833
    var data
834
    try {
835
      data = JSON.parse(Buffer.concat(chunks))
836
    } catch(e) {
837
      return self.respondError(400, e.stack)
838
    }
839
    return self.publishPkg2(pkgName, data || {})
840
  })
841
}
842
843
Req.prototype.publishPkg2 = function (name, data) {
844
  var self = this
845
  if (data.users) console.trace('[npm-registry] users property is not supported')
846
  var attachments = data._attachments || {}
847
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
848
  var done = multicb()
849
  function addAttachmentAsBlob(filename, cb) {
850
    var data = attachments[filename].data
851
    var tarball = new Buffer(data, 'base64')
852
    var length = attachments[filename].length
853
    if (length && length !== tarball.length) return self.respondError(400,
854
      'Length mismatch for attachment \'' + filename + '\'')
855
    self.server.sbot.blobs.add(function (err, id) {
856
      if (err) return cb(err)
857
      self.blobsToPush.push(id)
858
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
859
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
860
      cb()
861
    })(pull.once(tarball))
862
  }
863
  for (var filename in attachments) {
864
    addAttachmentAsBlob(filename, done())
865
  }
866
  done(function (err) {
867
    if (err) return self.respondError(500, err.stack || err)
868
    try {
869
      self.publishPkg3(name, data, links)
870
    } catch(e) {
871
      self.respondError(500, e.stack || e)
872
    }
873
  })
874
}
875
876
Req.prototype.publishPkg3 = function (name, data, links) {
877
  var self = this
878
  var versions = data.versions || {}
879
  var linksByVersion = {/* <version>: link */}
880
881
  // associate tarball blobs with versions
882
  for (var version in versions) {
883
    var pkg = versions[version]
884
    if (!pkg) return self.respondError(400, 'Bad package object')
885
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
886
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
887
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
888
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
889
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
890
    var filename = m[1]
891
    var link = links[filename]
892
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
893
    // TODO?: try to find missing tarball mentioned in other messages
894
    if (pkg.version && pkg.version !== version)
895
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
896
    linksByVersion[version] = link
897
    link.version = version
898
    link.dependencies = pkg.dependencies || {}
899
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
900
  }
901
902
  // associate blobs with dist-tags
903
  var tags = data['dist-tags'] || {}
904
  for (var tag in tags) {
905
    var version = tags[tag]
906
    var link = linksByVersion[version]
907
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
908
    // TODO?: support setting dist-tag without version,
909
    // by looking up a tarball blob for the version
910
    link.tag = tag
911
  }
912
913
  // compute blob links to publish
914
  var mentions = []
915
  for (var filename in links) {
916
    var link = links[filename] || {}
917
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
918
    mentions.push({
919
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
920
      link: link.link,
921
      size: link.size,
922
      shasum: link.shasum,
923
      dependencies: link.dependencies,
924
      bundledDependencies: link.bundledDependencies,
925
    })
926
  }
927
  return self.publishPkgs(mentions)
928
}
929
930
Req.prototype.publishPkgs = function (mentions) {
931
  var self = this
932
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
933
    if (err) return self.respondError(500, err.stack || err)
934
    self.server.pushBlobs(self.blobsToPush, function (err) {
935
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
936
      self.respond(201)
937
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
938
    })
939
  })
940
}
941
942
Req.prototype.populatePackageJson = function (obj, cb) {
943
  var self = this
944
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
945
  var deps, bundledDeps, shasum
946
947
  // look for dependencies in links.
948
  // then fallback to getting it from the tarball blob
949
950
  pull(
951
    self.getMentionLinks(blobId),
952
    pull.drain(function (link) {
953
      if (link.dependencies) deps = link.dependencies
954
      if (link.shasum) shasum = link.shasum
955
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
956
      // how to handle multiple assignments of dependencies to a package?
957
    }, function (err) {
958
      if (err) return cb(new Error(err.stack || err))
959
      if (deps && (shasum || !self.needShasum)) {
960
        // assume that the dependencies in the links to the blob are
961
        // correct.
962
        obj.dependencies = deps
963
        obj.bundledDependencies = bundledDeps
964
        if (shasum) obj.dist.shasum = obj._shasum = shasum
965
        cb(null, obj)
966
      } else {
967
        // get dependencies from the tarball
968
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
969
          if (err) return cb(err)
970
          pkg.dist = obj.dist
971
          pkg.dist.shasum = pkg._shasum
972
          pkg.author = pkg.author || obj.author
973
          pkg.version = pkg.version || obj.version
974
          pkg.name = pkg.name || obj.name
975
          cb(null, pkg)
976
        })
977
      }
978
    })
979
  )
980
}
981
982
function getPackageJsonFromTarballBlob(sbot, id, cb) {
983
  var self = this
984
  getBlob(sbot, id, function (err, readBlob) {
985
    if (err) return cb(err)
986
    cb = once(cb)
987
    var extract = tar.extract()
988
    var pkg, shasum
989
    extract.on('entry', function (header, stream, next) {
990
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
991
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
992
          if (err) return cb(err)
993
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
994
          catch(e) { return cb(e) }
995
          next()
996
        }))
997
      } else {
998
        stream.on('end', next)
999
        stream.resume()
1000
      }
1001
    })
1002
    extract.on('finish', function () {
1003
      pkg._shasum = shasum
1004
      cb(null, pkg)
1005
    })
1006
    pull(
1007
      readBlob,
1008
      hash('sha1', 'hex', function (err, sum) {
1009
        if (err) return cb(err)
1010
        shasum = sum
1011
      }),
1012
      toPull(zlib.createGunzip()),
1013
      toPull(extract)
1014
    )
1015
  })
1016
}
1017