git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
77
}
78
79
exports.name = 'npm-registry'
80
exports.version = '1.0.0'
81
exports.manifest = {
82
  getAddress: 'async'
83
}
84
exports.init = function (sbot, config) {
85
  var conf = config.npm || {}
86
  var port = conf.port || 8043
87
  var host = conf.host || null
88
  var autoAuth = conf.autoAuth !== false
89
90
  var server = http.createServer(exports.respond(sbot, config))
91
  var getAddress = onceify(function (cb) {
92
    server.on('error', cb)
93
    server.listen(port, host, function () {
94
      server.removeListener('error', cb)
95
      var regHost = formatHost(host || 'localhost')
96
      var regPort = this.address().port
97
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
98
      if (autoAuth) npmLogin(regUrl, next)
99
      else next()
100
      function next(err) {
101
        cb(err, regUrl)
102
      }
103
    })
104
    sbot.on('close', function () {
105
      server.close()
106
    })
107
  })
108
109
  getAddress(function (err, addr) {
110
    if (err) return console.error('[npm-registry]', err.stack || err)
111
    console.log('[npm-registry] Listening on ' + addr)
112
  })
113
114
  return {
115
    getAddress: getAddress
116
  }
117
}
118
119
exports.respond = function (sbot, config) {
120
  var reg = new SsbNpmRegistryServer(sbot, config)
121
  return function (req, res) {
122
    new Req(reg, req, res).serve()
123
  }
124
}
125
126
function publishMsg(sbot, value, cb) {
127
  var gotExpectedPrevious = false
128
  sbot.publish(value, function next(err, msg) {
129
    if (err && /^expected previous:/.test(err.message)) {
130
      // retry once on this error
131
      if (gotExpectedPrevious) return cb(err)
132
      gotExpectedPrevious = true
133
      return sbot.publish(value, next)
134
    }
135
    cb(err, msg)
136
  })
137
}
138
139
function getDependencyBranches(sbot, link, cb) {
140
  var id = link.link
141
142
  // get ids of heads of tree of dependencyBranch message links to include all
143
  // the dependencies of the given tarball id.
144
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
145
  var msgs = {}
146
  var branched = {}
147
  var blobs = {}
148
149
  function addPkgById(id, cb) {
150
    if (blobs[id]) return cb()
151
    blobs[id] = true
152
    getPackageJsonCached(id, function (err, pkg) {
153
      if (err) return cb(err)
154
      var done = multicb()
155
      for (var name in pkg.dependencies || {}) {
156
        addPkgBySpec(name, pkg.dependencies[name], done())
157
      }
158
      done(cb)
159
    })
160
  }
161
162
  function addPkgBySpec(name, spec, cb) {
163
    var versions = {}
164
    var distTags = {}
165
    pull(
166
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
167
      pull.filter(mentionMatches(null, name, spec)),
168
      pull.map(function (mention) {
169
        // query to get the messages since links2 does not include the msg id
170
        return sbot.links({
171
          source: mention.author,
172
          dest: mention.link,
173
          rel: 'mentions',
174
          values: true,
175
        })
176
      }),
177
      pull.flatten(),
178
      pull.drain(function (msg) {
179
        var c = msg && msg.value && msg.value.content
180
        if (!c || !Array.isArray(c.mentions)) return
181
        c.mentions.forEach(function (link) {
182
          var data = link && link.name && decodeName(link.name)
183
          if (!data || data.name !== name) return
184
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
185
          if (data.distTag) distTags[data.distTag] = data.version
186
        })
187
      }, function (err) {
188
        if (err) return cb(err)
189
        var version = distTags[spec]
190
          || semver.maxSatisfying(Object.keys(versions), spec)
191
        var item = versions[version]
192
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
193
        addDepBranchMsg(item.msg)
194
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
195
        addPkgById(item.mention.link, cb)
196
      })
197
    )
198
  }
199
200
  function addDepBranchMsg(msg) {
201
    var c = msg && msg.value && msg.value.content
202
    if (!c) return
203
    if (msgs[msg.key]) return
204
    msgs[msg.key] = msg.value
205
    if (Array.isArray(c.dependencyBranch)) {
206
      for (var k = 0; k < c.dependencyBranch.length; k++) {
207
        branched[c.dependencyBranch[k]] = true
208
      }
209
    }
210
    if (Array.isArray(c.versionBranch)) {
211
      for (var k = 0; k < c.versionBranch.length; k++) {
212
        branched[c.versionBranch[k]] = true
213
      }
214
    }
215
  }
216
217
  function addPkgsByName(name, cb) {
218
    var data = decodeName(name)
219
    pull(
220
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
221
      pull.map(function (mention) {
222
        return packageLinks(sbot, mention.author, mention.link, data.name)
223
      }),
224
      pull.flatten(),
225
      pull.drain(addDepBranchMsg, cb)
226
    )
227
  }
228
229
  var done = multicb()
230
  addPkgsByName(link.name, done())
231
  addPkgById(id, done())
232
  done(function (err) {
233
    if (err) return cb(err)
234
    var ids = []
235
    for (var key in msgs) {
236
      if (!branched[key]) ids.push(key)
237
    }
238
    cb(null, ids)
239
  })
240
}
241
242
function packageLinks(sbot, feed, id, name, spec) {
243
  var matches = mentionMatches(id, name, spec)
244
  return pull(
245
    sbot.links({
246
      source: feed,
247
      dest: id,
248
      rel: 'mentions',
249
      values: true,
250
    }),
251
    pull.filter(function (msg) {
252
      var c = msg && msg.value && msg.value.content
253
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
254
    })
255
  )
256
}
257
258
// For each dependency that is not a bundledDependency, get message ids for
259
// that dependency name + version.
260
function publishSingleMention(sbot, mention, cb) {
261
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
262
  // Calculate dependencyBranch message ids.
263
  var value = {
264
    type: 'npm-packages',
265
    mentions: [mention]
266
  }
267
  var done = multicb({pluck: 1, spread: true})
268
  getDependencyBranches(sbot, mention, function (err, branches) {
269
    if (err) return cb(err)
270
    value.dependencyBranch = branches || undefined
271
    publishMsg(sbot, value, cb)
272
  })
273
}
274
275
function publishMentions(sbot, mentions, cb) {
276
  // console.error("publishing %s mentions", mentions.length)
277
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
278
  // if it is just one mention, fetch and add useful metadata
279
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
280
  publishMsg(sbot, {
281
    type: 'npm-packages',
282
    mentions: mentions,
283
  }, cb)
284
}
285
286
exports.publishPkgMentions = function (sbot, mentions, cb) {
287
  // try to fit the mentions into as few messages as possible,
288
  // while fitting under the message size limit.
289
  var msgs = []
290
  ;(function next(i, chunks) {
291
    if (i >= mentions.length) return cb(null, msgs)
292
    var chunkLen = Math.ceil(mentions.length / chunks)
293
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
294
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
295
      if (err && msgs.length) return onPartialPublish(err)
296
      if (err) return cb(err)
297
      msgs.push(msg)
298
      next(i + chunkLen, chunks)
299
    })
300
  })(0, 1)
301
  function onPartialPublish(err) {
302
    var remaining = mentions.length - i
303
    return cb(new Error('Published messages ' +
304
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
305
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
306
  }
307
}
308
309
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
310
  cb = once(cb)
311
  var waiting = 0
312
  var expandedMentions = mentions.map(function (link) {
313
    var id = link && link.link
314
    if (!id) return link
315
    waiting++
316
    var newLink = {}
317
    for (var k in link) newLink[k] = link[k]
318
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
319
      if (err) return cb(err)
320
      for (var k in props) newLink[k] = pkg[k]
321
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
322
      if (!--waiting) next()
323
    })
324
    return newLink
325
  })
326
  if (!waiting) next()
327
  function next() {
328
    cb(null, expandedMentions)
329
  }
330
}
331
332
function SsbNpmRegistryServer(sbot, config) {
333
  this.sbot = sbot
334
  this.config = config
335
  this.npmConfig = config.npm || {}
336
  this.host = this.npmConfig.host || 'localhost'
337
  this.fetchAll = this.npmConfig.fetchAll
338
  this.needShasum = this.npmConfig.needShasum
339
  var wsPort = config.ws && Number(config.ws.port) || '8989'
340
  this.wsLink = this.npmConfig.wsLink
341
    || ('http://' + (config.host || 'localhost') + ':' + wsPort)
342
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
343
}
344
345
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
346
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
347
348
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
349
  var self = this
350
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
351
  ;(function next(i) {
352
    if (i >= ids.length) return cb()
353
    self.sbot.blobs.push(ids[i], function (err) {
354
      if (err) return cb(err)
355
      next(i+1)
356
    })
357
  })(0)
358
}
359
360
function getBlob(sbot, id, cb) {
361
  var blobs = sbot.blobs
362
  blobs.size(id, function (err, size) {
363
    if (err) return cb(err)
364
    if (typeof size === 'number') cb(null, blobs.get(id), size)
365
    else blobs.want(id, function (err, got) {
366
      if (err) cb(err)
367
      else if (!got) cb('missing blob ' + id)
368
      else blobs.size(id, function (err, size) {
369
        if (err) return cb(err)
370
        cb(null, blobs.get(id), size)
371
      })
372
    })
373
  })
374
}
375
376
SsbNpmRegistryServer.prototype.blobDist = function (id) {
377
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
378
  if (!m) throw new Error('bad blob id: ' + id)
379
  return {
380
    integrity: m[2] + '-' + m[1],
381
    tarball: this.wsLink + '/blobs/get/' + id
382
  }
383
}
384
385
function getMentions(links2, name) {
386
  return links2.read({
387
    query: [
388
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
389
      {$filter: {dest: {$prefix: '&'}}},
390
      {$map: {
391
        name: ['rel', 1],
392
        size: ['rel', 2],
393
        link: 'dest',
394
        author: 'source',
395
        ts: 'ts'
396
      }}
397
    ]
398
  })
399
}
400
401
function mentionMatches(id, name, spec) {
402
  return function (mention) {
403
    var data = mention
404
      && (!id || id === mention.link)
405
      && mention.name
406
      && decodeName(mention.name)
407
    return data
408
      && data.name === name
409
      && (spec ? semver.satisfies(data.version, spec) : true)
410
  }
411
}
412
413
SsbNpmRegistryServer.prototype.getMentions = function (name) {
414
  if (!this.sbot.links2) return pull.empty()
415
  return getMentions(this.sbot.links2, name)
416
}
417
418
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
419
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
420
  else this.sbot.get(id, function (err, value) {
421
    if (err) return cb(err)
422
    cb(null, {key: id, value: value})
423
  })
424
}
425
426
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
427
  var self = this
428
  var stack = heads.slice()
429
  var seen = {}
430
  return function (abort, cb) {
431
    if (abort) return cb(abort)
432
    for (var id; stack.length && seen[id = stack.pop()];);
433
    if (!id) return cb(true)
434
    seen[id] = true
435
    // TODO: use DFS
436
    self.getMsg(id, function (err, msg) {
437
      if (err) return cb(new Error(err.stack || err))
438
      var c = msg && msg.value && msg.value.content
439
      var links = c && c[prop]
440
      if (Array.isArray(links)) {
441
        stack.push.apply(stack, links)
442
      } else if (links) {
443
        stack.push(links)
444
      }
445
      cb(null, msg)
446
    })
447
  }
448
}
449
450
function Req(server, req, res) {
451
  this.server = server
452
  this.req = req
453
  this.res = res
454
  this.blobsToPush = []
455
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
456
  var ua = this.req.headers['user-agent']
457
  var m = /\bnpm\/([0-9]*)/.exec(ua)
458
  var npmVersion = m && m[1]
459
  this.needShasum = server.needShasum != null ? server.needShasum :
460
    (npmVersion && npmVersion < 5)
461
  this.wsLink = this.server.wsLink
462
}
463
464
Req.prototype.serve = function () {
465
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
466
  this.res.setTimeout(0)
467
  var pathname = this.req.url.replace(/\?.*/, '')
468
  var m
469
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
470
    try {
471
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
472
      this.headMsgPlus = !!m[1]
473
      // ^ means also include packages published after the head message id
474
    } catch(e) {
475
      return this.respondError(400, e.stack || e)
476
    }
477
    pathname = m[3]
478
  }
479
  if (pathname === '/') return this.serveHome()
480
  if (pathname === '/robots.txt') return this.serveRobots()
481
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
482
  if (pathname === '/-/whoami') return this.serveWhoami()
483
  if (pathname === '/-/ping') return this.respond(200, true)
484
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
485
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
486
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
487
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
488
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
489
  return this.respond(404, {error: 'Not found'})
490
}
491
492
Req.prototype.respond = function (status, message) {
493
  this.res.writeHead(status, {'content-type': 'application/json'})
494
  this.res.end(message && JSON.stringify(message, 0, 2))
495
}
496
497
Req.prototype.respondError = function (status, message) {
498
  this.respond(status, {error: message})
499
}
500
501
Req.prototype.respondErrorStr = function (status, err) {
502
  this.res.writeHead(status, {'content-type': 'text/plain'})
503
  this.res.end(err.stack || err)
504
}
505
506
Req.prototype.respondRaw = function (status, body) {
507
  this.res.writeHead(status)
508
  this.res.end(body)
509
}
510
511
Req.prototype.serveHome = function () {
512
  var self = this
513
  self.res.writeHead(200, {'content-type': 'text/html'})
514
  var port = 8044
515
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
516
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
517
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
518
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
519
    '</body></html>')
520
}
521
522
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
523
  var self = this
524
  pull(
525
    self.server.sbot.links({
526
      source: feedId,
527
      dest: blobId,
528
      rel: 'mentions',
529
      values: true,
530
    }),
531
    pull.filter(function (msg) {
532
      var c = msg && msg.value && msg.value.content
533
      return c.type === 'npm-packages'
534
    }),
535
    pull.collect(function (err, msgs) {
536
      if (err) return cb(err)
537
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
538
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
539
      // TODO: make a smarter decision about which message id to use
540
      cb(null, msgs.pop().key)
541
    })
542
  )
543
}
544
545
Req.prototype.resolvePkg = function (pkgSpec, cb) {
546
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
547
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
548
  var self = this
549
  var pkgName = m[1]
550
  var spec = m[2] || '*'
551
  var versions = {}
552
  var distTags = {}
553
  pull(
554
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
555
    pull.drain(function (mention) {
556
      var data = decodeName(mention.name)
557
      if (!data.version) return
558
      if (data.distTag) {
559
        distTags[data.distTag] = data.version
560
      }
561
      versions[data.version] = {
562
        author: mention.author,
563
        name: pkgName,
564
        version: data.version,
565
        blobId: mention.link
566
      }
567
    }, function (err) {
568
      if (err) return cb(err)
569
      var version = distTags[spec]
570
        || semver.maxSatisfying(Object.keys(versions), spec)
571
      var item = versions[version]
572
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
573
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
574
        if (err) return cb(err)
575
        item.msgId = id
576
        cb(null, item)
577
      })
578
    })
579
  )
580
}
581
582
Req.prototype.resolvePkgs = function (specs, cb) {
583
  var done = multicb({pluck: 1})
584
  var self = this
585
  specs.forEach(function (spec) {
586
    self.resolvePkg(spec, done())
587
  })
588
  done(cb)
589
}
590
591
Req.prototype.serveBootstrap = function () {
592
  var self = this
593
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
594
  var ssbNpmRegistryName = require('./package.json').name
595
  var ssbNpmRegistryVersion = require('./package.json').version
596
  var ssbNpmRegistryNameVersion = ssbNpmRegistryName + '-' + ssbNpmRegistryVersion
597
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
598
  var done = multicb({pluck: 1, spread: true})
599
  self.resolvePkg(ssbNpmRegistrySpec, done())
600
  self.resolvePkgs(pkgs, done())
601
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
602
    if (err) return self.respondErrorStr(500, err.stack || err)
603
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
604
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
605
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
606
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
607
    var globalPkgs = pkgsInfo.map(function (info) {
608
      return info.name + '@' + info.version
609
    }).join(' ')
610
    var npmCmd = 'install -g ' + globalPkgs
611
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
612
    var tarballLink = self.wsLink + '/blobs/get/' + ssbNpmRegistryBlobId
613
614
    var script =
615
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
616
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
617
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
618
      'tar xzf package.tgz &&\n' +
619
      './*/bootstrap/bin.js --ws-url ' + self.wsLink + ' \\\n' +
620
        pkgMsgs.map(function (id) {
621
          return '  --branch ' + id + ' \\\n'
622
        }).join('') +
623
        '  -- ' + npmCmd + ' &&\n' +
624
      'sbot server'
625
626
    self.res.writeHead(200, {'Content-type': 'text/plain'})
627
    self.res.end(script)
628
  })
629
}
630
631
Req.prototype.serveRobots = function () {
632
  this.res.writeHead(200, {'Content-type': 'text/plain'})
633
  this.res.end('User-agent: *\nDisallow: /\n')
634
}
635
636
Req.prototype.serveWhoami = function () {
637
  var self = this
638
  self.server.sbot.whoami(function (err, feed) {
639
    if (err) return self.respondError(500, err.stack || err)
640
    self.respond(200, {username: feed.id})
641
  })
642
}
643
644
Req.prototype.serveUser1 = function () {
645
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
646
}
647
648
Req.prototype.serveBlob = function (id) {
649
  var self = this
650
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
651
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
652
    if (err) {
653
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
654
      else return self.respondErrorStr(500, err.message || err)
655
    }
656
    self.res.writeHead(200, {
657
      'Cache-Control': 'public, max-age=315360000',
658
      'Content-Length': size,
659
      'etag': id
660
    })
661
    pull(
662
      readBlob,
663
      toPull(self.res, function (err) {
664
        if (err) console.error('[npm-registry]', err)
665
      })
666
    )
667
  })
668
}
669
670
Req.prototype.serveMsg = function (id) {
671
  var self = this
672
  try { id = decodeURIComponent(id) }
673
  catch (e) {}
674
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
675
  self.server.sbot.get(id, function (err, value) {
676
    if (err) return self.respondError(500, err.message || err)
677
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
678
    self.res.writeHead(200, {
679
      'Content-Type': 'application/json',
680
      'Cache-Control': 'public, max-age=315360000',
681
      'Content-Length': out.length,
682
      'etag': id
683
    })
684
    self.res.end(out)
685
  })
686
}
687
688
function decodeName(name) {
689
  var parts = String(name).replace(/\.tgz$/, '').split(':')
690
  return {
691
    name: parts[1],
692
    version: parts[2],
693
    distTag: parts[3],
694
  }
695
}
696
697
Req.prototype.getMsgMentions = function (name) {
698
  return pull(
699
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
700
    // decryption could be done here
701
    pull.map(function (msg) {
702
      var c = msg.value && msg.value.content
703
      if (!c.mentions || !Array.isArray(c.mentions)) return []
704
      return c.mentions.map(function (mention) {
705
        return {
706
          name: mention.name,
707
          size: mention.size,
708
          link: mention.link,
709
          author: msg.value.author,
710
          ts: msg.value.timestamp,
711
        }
712
      })
713
    }),
714
    pull.flatten(),
715
    pull.filter(typeof name === 'string' ? function (link) {
716
      return link.name === name
717
    } : name && name.$prefix ? function (link) {
718
      return link.name.substr(0, name.$prefix.length) === name.$prefix
719
    } : function () {
720
      throw new TypeError('unsupported name filter')
721
    })
722
  )
723
}
724
725
Req.prototype.getMentions = function (name) {
726
  var useMsgMentions = this.headMsgIds
727
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
728
  if (useServerMentions && !this.server.sbot.links2) {
729
    return this.headMsgPlus
730
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
731
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
732
  }
733
  return cat([
734
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
735
    useServerMentions ? this.server.getMentions(name) : pull.empty()
736
  ])
737
}
738
739
Req.prototype.getMentionLinks = function (blobId) {
740
  return pull(
741
    this.headMsgIds
742
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
743
      : this.server.sbot.links({
744
        dest: blobId,
745
        rel: 'mentions',
746
        values: true,
747
      }),
748
    // decryption could be done here
749
    pull.map(function (msg) {
750
      var c = msg.value && msg.value.content
751
      return c && Array.isArray(c.mentions) && c.mentions || []
752
    }),
753
    pull.flatten(),
754
    pull.filter(function (link) {
755
      return link && link.link === blobId
756
    })
757
  )
758
}
759
760
Req.prototype.servePkg = function (pathname) {
761
  var self = this
762
  var parts = pathname.split('/')
763
  var pkgName = parts.shift().replace(/%2f/i, '/')
764
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
765
  var spec = parts.shift()
766
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
767
  if (parts.length > 0) return this.respondError(404)
768
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
769
  var obj = {
770
    _id: pkgName,
771
    name: pkgName,
772
    'dist-tags': {},
773
    versions: {}
774
  }
775
  var distTags = {/* <tag>: {version, ts}*/}
776
  pull(
777
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
778
    pull.drain(function (mention) {
779
      var data = decodeName(mention.name)
780
      if (!data.version) return
781
      if (data.distTag) {
782
        var tag = distTags[data.distTag]
783
        if (!tag || mention.ts > tag.ts) {
784
          /* TODO: sort by causal order instead of only by timestamps */
785
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
786
        }
787
      }
788
      obj.versions[data.version] = {
789
        author: {
790
          url: mention.author
791
        },
792
        name: pkgName,
793
        version: data.version,
794
        dist: self.server.blobDist(mention.link)
795
      }
796
    }, function (err) {
797
      if (err) return self.respondError(500, err.stack || err)
798
      for (var tag in distTags) {
799
        obj['dist-tags'][tag] = distTags[tag].version
800
      }
801
      if (spec) resolveSpec()
802
      else if (self.fetchAll) resolveAll()
803
      else resolved()
804
    })
805
  )
806
  function resolveSpec() {
807
    var version = obj['dist-tags'][spec]
808
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
809
    obj = obj.versions[version]
810
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
811
    self.populatePackageJson(obj, function (err, pkg) {
812
      if (err) return resolved(err)
813
      obj = pkg || obj
814
      resolved()
815
    })
816
  }
817
  function resolveVersion(version, cb) {
818
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
819
      if (err) return cb(err)
820
      if (pkg) obj.versions[version] = pkg
821
      cb()
822
    })
823
  }
824
  function resolveAll() {
825
    var done = multicb()
826
    for (var version in obj.versions) {
827
      resolveVersion(version, done())
828
    }
829
    done(resolved)
830
  }
831
  function resolved(err) {
832
    if (err) return self.respondError(500, err.stack || err)
833
    self.respond(200, obj)
834
  }
835
}
836
837
Req.prototype.servePrebuild = function (name) {
838
  var self = this
839
  var getMention = self.getMentions('prebuild:' + name)
840
  var blobsByAuthor = {/* <author>: BlobId */}
841
  pull(getMention, pull.drain(function (link) {
842
    blobsByAuthor[link.author] = link.link
843
  }, function (err) {
844
    if (err) return self.respondError(500, err.stack || err)
845
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
846
    var blobId
847
    for (var feed in blobsByAuthor) {
848
      var blob = blobId = blobsByAuthor[feed]
849
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
850
      feeds.push(feed)
851
    }
852
    switch (Object.keys(authorsByLink).length) {
853
      case 0:
854
        return self.respondError(404, 'Not Found')
855
      case 1:
856
        self.res.writeHead(303, {Location: self.wsLink + '/blobs/get/' + blobId})
857
        return self.res.end()
858
      default:
859
        return self.respond(300, {choices: authorsByLink})
860
    }
861
  }))
862
}
863
864
var localhosts = {
865
  '::1': true,
866
  '127.0.0.1': true,
867
  '::ffff:127.0.0.1': true,
868
}
869
870
Req.prototype.publishPkg = function (pkgName) {
871
  var self = this
872
  var remoteAddress = self.req.socket.remoteAddress
873
  if (!(remoteAddress in localhosts)) {
874
    return self.respondError(403, 'You may not publish as this user.')
875
  }
876
877
  var chunks = []
878
  self.req.on('data', function (data) {
879
    chunks.push(data)
880
  })
881
  self.req.on('end', function () {
882
    var data
883
    try {
884
      data = JSON.parse(Buffer.concat(chunks))
885
    } catch(e) {
886
      return self.respondError(400, e.stack)
887
    }
888
    return self.publishPkg2(pkgName, data || {})
889
  })
890
}
891
892
Req.prototype.publishPkg2 = function (name, data) {
893
  var self = this
894
  if (data.users) console.trace('[npm-registry] users property is not supported')
895
  var attachments = data._attachments || {}
896
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
897
  var done = multicb()
898
  function addAttachmentAsBlob(filename, cb) {
899
    var data = attachments[filename].data
900
    var tarball = new Buffer(data, 'base64')
901
    var length = attachments[filename].length
902
    if (length && length !== tarball.length) return self.respondError(400,
903
      'Length mismatch for attachment \'' + filename + '\'')
904
    self.server.sbot.blobs.add(function (err, id) {
905
      if (err) return cb(err)
906
      self.blobsToPush.push(id)
907
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
908
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
909
      cb()
910
    })(pull.once(tarball))
911
  }
912
  for (var filename in attachments) {
913
    addAttachmentAsBlob(filename, done())
914
  }
915
  done(function (err) {
916
    if (err) return self.respondError(500, err.stack || err)
917
    try {
918
      self.publishPkg3(name, data, links)
919
    } catch(e) {
920
      self.respondError(500, e.stack || e)
921
    }
922
  })
923
}
924
925
Req.prototype.publishPkg3 = function (name, data, links) {
926
  var self = this
927
  var versions = data.versions || {}
928
  var linksByVersion = {/* <version>: link */}
929
930
  // associate tarball blobs with versions
931
  for (var version in versions) {
932
    var pkg = versions[version]
933
    if (!pkg) return self.respondError(400, 'Bad package object')
934
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
935
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
936
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
937
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
938
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
939
    var filename = m[1]
940
    var link = links[filename]
941
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
942
    // TODO?: try to find missing tarball mentioned in other messages
943
    if (pkg.version && pkg.version !== version)
944
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
945
    linksByVersion[version] = link
946
    link.version = version
947
    link.dependencies = pkg.dependencies || {}
948
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
949
  }
950
951
  // associate blobs with dist-tags
952
  var tags = data['dist-tags'] || {}
953
  for (var tag in tags) {
954
    var version = tags[tag]
955
    var link = linksByVersion[version]
956
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
957
    // TODO?: support setting dist-tag without version,
958
    // by looking up a tarball blob for the version
959
    link.tag = tag
960
  }
961
962
  // compute blob links to publish
963
  var mentions = []
964
  for (var filename in links) {
965
    var link = links[filename] || {}
966
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
967
    mentions.push({
968
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
969
      link: link.link,
970
      size: link.size,
971
      shasum: link.shasum,
972
      dependencies: link.dependencies,
973
      bundledDependencies: link.bundledDependencies,
974
    })
975
  }
976
  return self.publishPkgs(mentions)
977
}
978
979
Req.prototype.publishPkgs = function (mentions) {
980
  var self = this
981
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
982
    if (err) return self.respondError(500, err.stack || err)
983
    self.server.pushBlobs(self.blobsToPush, function (err) {
984
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
985
      self.respond(201)
986
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
987
    })
988
  })
989
}
990
991
Req.prototype.populatePackageJson = function (obj, cb) {
992
  var self = this
993
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
994
  var deps, bundledDeps, shasum
995
996
  // look for dependencies in links.
997
  // then fallback to getting it from the tarball blob
998
999
  pull(
1000
    self.getMentionLinks(blobId),
1001
    pull.drain(function (link) {
1002
      if (link.dependencies) deps = link.dependencies
1003
      if (link.shasum) shasum = link.shasum
1004
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1005
      // how to handle multiple assignments of dependencies to a package?
1006
    }, function (err) {
1007
      if (err) return cb(new Error(err.stack || err))
1008
      if (deps && (shasum || !self.needShasum)) {
1009
        // assume that the dependencies in the links to the blob are
1010
        // correct.
1011
        obj.dependencies = deps
1012
        obj.bundledDependencies = bundledDeps
1013
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1014
        cb(null, obj)
1015
      } else {
1016
        // get dependencies from the tarball
1017
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1018
          if (err) return cb(err)
1019
          pkg.dist = obj.dist
1020
          pkg.dist.shasum = pkg._shasum
1021
          pkg.author = pkg.author || obj.author
1022
          pkg.version = pkg.version || obj.version
1023
          pkg.name = pkg.name || obj.name
1024
          cb(null, pkg)
1025
        })
1026
      }
1027
    })
1028
  )
1029
}
1030
1031
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1032
  var self = this
1033
  getBlob(sbot, id, function (err, readBlob) {
1034
    if (err) return cb(err)
1035
    cb = once(cb)
1036
    var extract = tar.extract()
1037
    var pkg, shasum
1038
    extract.on('entry', function (header, stream, next) {
1039
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1040
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1041
          if (err) return cb(err)
1042
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1043
          catch(e) { return cb(e) }
1044
          next()
1045
        }))
1046
      } else {
1047
        stream.on('end', next)
1048
        stream.resume()
1049
      }
1050
    })
1051
    extract.on('finish', function () {
1052
      pkg._shasum = shasum
1053
      cb(null, pkg)
1054
    })
1055
    pull(
1056
      readBlob,
1057
      hash('sha1', 'hex', function (err, sum) {
1058
        if (err) return cb(err)
1059
        shasum = sum
1060
      }),
1061
      toPull(zlib.createGunzip()),
1062
      toPull(extract)
1063
    )
1064
  })
1065
}
1066