git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || null
90
  var autoAuth = conf.autoAuth !== false
91
92
  var server = http.createServer(exports.respond(sbot, config))
93
  var getAddress = onceify(function (cb) {
94
    server.on('error', cb)
95
    server.listen(port, host, function () {
96
      server.removeListener('error', cb)
97
      var regHost = formatHost(host || 'localhost')
98
      var regPort = this.address().port
99
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
100
      if (autoAuth) npmLogin(regUrl, next)
101
      else next()
102
      function next(err) {
103
        cb(err, regUrl)
104
      }
105
    })
106
    sbot.on('close', function () {
107
      server.close()
108
    })
109
  })
110
111
  getAddress(function (err, addr) {
112
    if (err) return console.error('[npm-registry]', err.stack || err)
113
    console.log('[npm-registry] Listening on ' + addr)
114
  })
115
116
  return {
117
    getAddress: getAddress
118
  }
119
}
120
121
exports.respond = function (sbot, config) {
122
  var reg = new SsbNpmRegistryServer(sbot, config)
123
  return function (req, res) {
124
    new Req(reg, req, res).serve()
125
  }
126
}
127
128
function publishMsg(sbot, value, cb) {
129
  var gotExpectedPrevious = false
130
  sbot.publish(value, function next(err, msg) {
131
    if (err && /^expected previous:/.test(err.message)) {
132
      // retry once on this error
133
      if (gotExpectedPrevious) return cb(err)
134
      gotExpectedPrevious = true
135
      return sbot.publish(value, next)
136
    }
137
    cb(err, msg)
138
  })
139
}
140
141
function getDependencyBranches(sbot, link, cb) {
142
  var id = link.link
143
144
  // get ids of heads of tree of dependencyBranch message links to include all
145
  // the dependencies of the given tarball id.
146
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
147
  var msgs = {}
148
  var branched = {}
149
  var blobs = {}
150
151
  function addPkgById(id, cb) {
152
    if (blobs[id]) return cb()
153
    blobs[id] = true
154
    getPackageJsonCached(id, function (err, pkg) {
155
      if (err) return cb(err)
156
      var done = multicb()
157
      for (var name in pkg.dependencies || {}) {
158
        addPkgBySpec(name, pkg.dependencies[name], done())
159
      }
160
      done(cb)
161
    })
162
  }
163
164
  function addPkgBySpec(name, spec, cb) {
165
    var versions = {}
166
    var distTags = {}
167
    pull(
168
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
169
      pull.filter(mentionMatches(null, name, spec)),
170
      pull.map(function (mention) {
171
        // query to get the messages since links2 does not include the msg id
172
        return sbot.links({
173
          source: mention.author,
174
          dest: mention.link,
175
          rel: 'mentions',
176
          values: true,
177
        })
178
      }),
179
      pull.flatten(),
180
      pull.drain(function (msg) {
181
        var c = msg && msg.value && msg.value.content
182
        if (!c || !Array.isArray(c.mentions)) return
183
        c.mentions.forEach(function (link) {
184
          var data = link && link.name && decodeName(link.name)
185
          if (!data || data.name !== name) return
186
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
187
          if (data.distTag) distTags[data.distTag] = data.version
188
        })
189
      }, function (err) {
190
        if (err) return cb(err)
191
        var version = distTags[spec]
192
          || semver.maxSatisfying(Object.keys(versions), spec)
193
        var item = versions[version]
194
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
195
        addDepBranchMsg(item.msg)
196
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
197
        addPkgById(item.mention.link, cb)
198
      })
199
    )
200
  }
201
202
  function addDepBranchMsg(msg) {
203
    var c = msg && msg.value && msg.value.content
204
    if (!c) return
205
    if (msgs[msg.key]) return
206
    msgs[msg.key] = msg.value
207
    if (Array.isArray(c.dependencyBranch)) {
208
      for (var k = 0; k < c.dependencyBranch.length; k++) {
209
        branched[c.dependencyBranch[k]] = true
210
      }
211
    }
212
    if (Array.isArray(c.versionBranch)) {
213
      for (var k = 0; k < c.versionBranch.length; k++) {
214
        branched[c.versionBranch[k]] = true
215
      }
216
    }
217
  }
218
219
  function addPkgsByName(name, cb) {
220
    var data = decodeName(name)
221
    pull(
222
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
223
      pull.map(function (mention) {
224
        return packageLinks(sbot, mention.author, mention.link, data.name)
225
      }),
226
      pull.flatten(),
227
      pull.drain(addDepBranchMsg, cb)
228
    )
229
  }
230
231
  var done = multicb()
232
  addPkgsByName(link.name, done())
233
  addPkgById(id, done())
234
  done(function (err) {
235
    if (err) return cb(err)
236
    var ids = []
237
    for (var key in msgs) {
238
      if (!branched[key]) ids.push(key)
239
    }
240
    cb(null, ids)
241
  })
242
}
243
244
function packageLinks(sbot, feed, id, name, spec) {
245
  var matches = mentionMatches(id, name, spec)
246
  return pull(
247
    sbot.links({
248
      source: feed,
249
      dest: id,
250
      rel: 'mentions',
251
      values: true,
252
    }),
253
    pull.filter(function (msg) {
254
      var c = msg && msg.value && msg.value.content
255
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
256
    })
257
  )
258
}
259
260
// For each dependency that is not a bundledDependency, get message ids for
261
// that dependency name + version.
262
function publishSingleMention(sbot, mention, cb) {
263
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
264
  // Calculate dependencyBranch message ids.
265
  var value = {
266
    type: 'npm-packages',
267
    mentions: [mention]
268
  }
269
  var done = multicb({pluck: 1, spread: true})
270
  getDependencyBranches(sbot, mention, function (err, branches) {
271
    if (err) return cb(err)
272
    value.dependencyBranch = branches || undefined
273
    publishMsg(sbot, value, cb)
274
  })
275
}
276
277
function publishMentions(sbot, mentions, cb) {
278
  // console.error("publishing %s mentions", mentions.length)
279
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
280
  // if it is just one mention, fetch and add useful metadata
281
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
282
  publishMsg(sbot, {
283
    type: 'npm-packages',
284
    mentions: mentions,
285
  }, cb)
286
}
287
288
exports.publishPkgMentions = function (sbot, mentions, cb) {
289
  // try to fit the mentions into as few messages as possible,
290
  // while fitting under the message size limit.
291
  var msgs = []
292
  ;(function next(i, chunks) {
293
    if (i >= mentions.length) return cb(null, msgs)
294
    var chunkLen = Math.ceil(mentions.length / chunks)
295
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
296
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
297
      if (err && msgs.length) return onPartialPublish(err)
298
      if (err) return cb(err)
299
      msgs.push(msg)
300
      next(i + chunkLen, chunks)
301
    })
302
  })(0, 1)
303
  function onPartialPublish(err) {
304
    var remaining = mentions.length - i
305
    return cb(new Error('Published messages ' +
306
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
307
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
308
  }
309
}
310
311
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
312
  cb = once(cb)
313
  var waiting = 0
314
  var expandedMentions = mentions.map(function (link) {
315
    var id = link && link.link
316
    if (!id) return link
317
    waiting++
318
    var newLink = {}
319
    for (var k in link) newLink[k] = link[k]
320
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
321
      if (err) return cb(err)
322
      for (var k in props) newLink[k] = pkg[k]
323
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
324
      if (!--waiting) next()
325
    })
326
    return newLink
327
  })
328
  if (!waiting) next()
329
  function next() {
330
    cb(null, expandedMentions)
331
  }
332
}
333
334
function SsbNpmRegistryServer(sbot, config) {
335
  this.sbot = sbot
336
  this.config = config
337
  this.npmConfig = config.npm || {}
338
  this.host = this.npmConfig.host || 'localhost'
339
  this.fetchAll = this.npmConfig.fetchAll
340
  this.needShasum = this.npmConfig.needShasum
341
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
342
}
343
344
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
345
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
346
347
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
348
  var self = this
349
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
350
  ;(function next(i) {
351
    if (i >= ids.length) return cb()
352
    self.sbot.blobs.push(ids[i], function (err) {
353
      if (err) return cb(err)
354
      next(i+1)
355
    })
356
  })(0)
357
}
358
359
function getBlob(sbot, id, cb) {
360
  var blobs = sbot.blobs
361
  blobs.size(id, function (err, size) {
362
    if (err) return cb(err)
363
    if (typeof size === 'number') cb(null, blobs.get(id), size)
364
    else blobs.want(id, function (err, got) {
365
      if (err) cb(err)
366
      else if (!got) cb('missing blob ' + id)
367
      else blobs.size(id, function (err, size) {
368
        if (err) return cb(err)
369
        cb(null, blobs.get(id), size)
370
      })
371
    })
372
  })
373
}
374
375
function getMentions(links2, name) {
376
  return links2.read({
377
    query: [
378
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
379
      {$filter: {dest: {$prefix: '&'}}},
380
      {$map: {
381
        name: ['rel', 1],
382
        size: ['rel', 2],
383
        link: 'dest',
384
        author: 'source',
385
        ts: 'ts'
386
      }}
387
    ]
388
  })
389
}
390
391
function mentionMatches(id, name, spec) {
392
  return function (mention) {
393
    var data = mention
394
      && (!id || id === mention.link)
395
      && mention.name
396
      && decodeName(mention.name)
397
    return data
398
      && data.name === name
399
      && (spec ? semver.satisfies(data.version, spec) : true)
400
  }
401
}
402
403
SsbNpmRegistryServer.prototype.getMentions = function (name) {
404
  if (!this.sbot.links2) return pull.empty()
405
  return getMentions(this.sbot.links2, name)
406
}
407
408
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
409
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
410
  else this.sbot.get(id, function (err, value) {
411
    if (err) return cb(err)
412
    cb(null, {key: id, value: value})
413
  })
414
}
415
416
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
417
  var self = this
418
  var stack = heads.slice()
419
  var seen = {}
420
  return function (abort, cb) {
421
    if (abort) return cb(abort)
422
    for (var id; stack.length && seen[id = stack.pop()];);
423
    if (!id) return cb(true)
424
    seen[id] = true
425
    // TODO: use DFS
426
    self.getMsg(id, function (err, msg) {
427
      if (err) return cb(new Error(err.stack || err))
428
      var c = msg && msg.value && msg.value.content
429
      var links = c && c[prop]
430
      if (Array.isArray(links)) {
431
        stack.push.apply(stack, links)
432
      } else if (links) {
433
        stack.push(links)
434
      }
435
      cb(null, msg)
436
    })
437
  }
438
}
439
440
function Req(server, req, res) {
441
  this.server = server
442
  this.req = req
443
  this.res = res
444
  this.blobsToPush = []
445
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
446
  var ua = this.req.headers['user-agent']
447
  var m = /\bnpm\/([0-9]*)/.exec(ua)
448
  var npmVersion = m && m[1]
449
  this.needShasum = server.needShasum != null ? server.needShasum :
450
    (npmVersion && npmVersion < 5)
451
  this.baseUrl = this.server.npmConfig.baseUrl
452
  if (this.baseUrl) {
453
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
454
  } else {
455
    var hostname = req.headers.host
456
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
457
    this.baseUrl = 'http://' + hostname
458
  }
459
}
460
461
Req.prototype.serve = function () {
462
  console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
463
  this.res.setTimeout(0)
464
  var pathname = this.req.url.replace(/\?.*/, '')
465
  var m
466
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
467
    try {
468
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
469
      this.headMsgPlus = !!m[1]
470
      // ^ means also include packages published after the head message id
471
    } catch(e) {
472
      return this.respondError(400, e.stack || e)
473
    }
474
    pathname = m[3]
475
  }
476
  if (pathname === '/') return this.serveHome()
477
  if (pathname === '/robots.txt') return this.serveRobots()
478
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
479
  if (pathname === '/-/whoami') return this.serveWhoami()
480
  if (pathname === '/-/ping') return this.respond(200, true)
481
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
482
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
483
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
484
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
485
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
486
  return this.respond(404, {error: 'Not found'})
487
}
488
489
Req.prototype.respond = function (status, message) {
490
  this.res.writeHead(status, {'content-type': 'application/json'})
491
  this.res.end(message && JSON.stringify(message, 0, 2))
492
}
493
494
Req.prototype.respondError = function (status, message) {
495
  this.respond(status, {error: message})
496
}
497
498
Req.prototype.respondErrorStr = function (status, err) {
499
  this.res.writeHead(status, {'content-type': 'text/plain'})
500
  this.res.end(err.stack || err)
501
}
502
503
Req.prototype.respondRaw = function (status, body) {
504
  this.res.writeHead(status)
505
  this.res.end(body)
506
}
507
508
Req.prototype.serveHome = function () {
509
  var self = this
510
  self.res.writeHead(200, {'content-type': 'text/html'})
511
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
512
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
513
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
514
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
515
    '</body></html>')
516
}
517
518
Req.prototype.blobDist = function (id) {
519
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
520
  if (!m) throw new Error('bad blob id: ' + id)
521
  return {
522
    integrity: m[2] + '-' + m[1],
523
    tarball: this.baseUrl + '/-/blobs/get/' + id
524
  }
525
}
526
527
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
528
  var self = this
529
  pull(
530
    self.server.sbot.links({
531
      source: feedId,
532
      dest: blobId,
533
      rel: 'mentions',
534
      values: true,
535
    }),
536
    pull.filter(function (msg) {
537
      var c = msg && msg.value && msg.value.content
538
      return c.type === 'npm-packages'
539
    }),
540
    pull.collect(function (err, msgs) {
541
      if (err) return cb(err)
542
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
543
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
544
      // TODO: make a smarter decision about which message id to use
545
      cb(null, msgs.pop().key)
546
    })
547
  )
548
}
549
550
Req.prototype.resolvePkg = function (pkgSpec, cb) {
551
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
552
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
553
  var self = this
554
  var pkgName = m[1]
555
  var spec = m[2] || '*'
556
  var versions = {}
557
  var distTags = {}
558
  pull(
559
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
560
    pull.drain(function (mention) {
561
      var data = decodeName(mention.name)
562
      if (!data.version) return
563
      if (data.distTag) {
564
        distTags[data.distTag] = data.version
565
      }
566
      versions[data.version] = {
567
        author: mention.author,
568
        name: pkgName,
569
        version: data.version,
570
        blobId: mention.link
571
      }
572
    }, function (err) {
573
      if (err) return cb(err)
574
      var version = distTags[spec]
575
        || semver.maxSatisfying(Object.keys(versions), spec)
576
      var item = versions[version]
577
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
578
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
579
        if (err) return cb(err)
580
        item.msgId = id
581
        cb(null, item)
582
      })
583
    })
584
  )
585
}
586
587
Req.prototype.resolvePkgs = function (specs, cb) {
588
  var done = multicb({pluck: 1})
589
  var self = this
590
  specs.forEach(function (spec) {
591
    self.resolvePkg(spec, done())
592
  })
593
  done(cb)
594
}
595
596
Req.prototype.serveBootstrap = function () {
597
  var self = this
598
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
599
  var postInstallCmd = self.server.npmConfig.postInstallCmd
600
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
601
  var ssbNpmRegistryName = require('./package.json').name
602
  var ssbNpmRegistryVersion = require('./package.json').version
603
  var ssbNpmRegistryNameVersion = ssbNpmRegistryName + '-' + ssbNpmRegistryVersion
604
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
605
  var done = multicb({pluck: 1, spread: true})
606
  self.resolvePkg(ssbNpmRegistrySpec, done())
607
  self.resolvePkgs(pkgs, done())
608
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
609
    if (err) return self.respondErrorStr(500, err.stack || err)
610
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
611
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
612
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
613
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
614
    var globalPkgs = pkgsInfo.map(function (info) {
615
      return info.name + '@' + info.version
616
    }).join(' ')
617
    var npmCmd = 'install -g ' + globalPkgs
618
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
619
    var wsUrl = self.baseUrl + '/-'
620
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
621
622
    var script =
623
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
624
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
625
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
626
      'tar xzf package.tgz &&\n' +
627
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
628
        pkgMsgs.map(function (id) {
629
          return '  --branch ' + id + ' \\\n'
630
        }).join('') +
631
        '  -- ' + npmCmd +
632
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
633
634
    self.res.writeHead(200, {'Content-type': 'text/plain'})
635
    self.res.end(script)
636
  })
637
}
638
639
Req.prototype.serveRobots = function () {
640
  this.res.writeHead(200, {'Content-type': 'text/plain'})
641
  this.res.end('User-agent: *\nDisallow: /\n')
642
}
643
644
Req.prototype.serveWhoami = function () {
645
  var self = this
646
  self.server.sbot.whoami(function (err, feed) {
647
    if (err) return self.respondError(500, err.stack || err)
648
    self.respond(200, {username: feed.id})
649
  })
650
}
651
652
Req.prototype.serveUser1 = function () {
653
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
654
}
655
656
Req.prototype.serveBlob = function (id) {
657
  var self = this
658
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
659
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
660
    if (err) {
661
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
662
      else return self.respondErrorStr(500, err.message || err)
663
    }
664
    self.res.writeHead(200, {
665
      'Cache-Control': 'public, max-age=315360000',
666
      'Content-Length': size,
667
      'etag': id
668
    })
669
    pull(
670
      readBlob,
671
      toPull(self.res, function (err) {
672
        if (err) console.error('[npm-registry]', err)
673
      })
674
    )
675
  })
676
}
677
678
Req.prototype.serveMsg = function (id) {
679
  var self = this
680
  try { id = decodeURIComponent(id) }
681
  catch (e) {}
682
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
683
  self.server.sbot.get(id, function (err, value) {
684
    if (err) return self.respondError(500, err.message || err)
685
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
686
    self.res.writeHead(200, {
687
      'Content-Type': 'application/json',
688
      'Cache-Control': 'public, max-age=315360000',
689
      'Content-Length': out.length,
690
      'etag': id
691
    })
692
    self.res.end(out)
693
  })
694
}
695
696
function decodeName(name) {
697
  var parts = String(name).replace(/\.tgz$/, '').split(':')
698
  return {
699
    name: parts[1],
700
    version: parts[2],
701
    distTag: parts[3],
702
  }
703
}
704
705
Req.prototype.getMsgMentions = function (name) {
706
  return pull(
707
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
708
    // decryption could be done here
709
    pull.map(function (msg) {
710
      var c = msg.value && msg.value.content
711
      if (!c.mentions || !Array.isArray(c.mentions)) return []
712
      return c.mentions.map(function (mention) {
713
        return {
714
          name: mention.name,
715
          size: mention.size,
716
          link: mention.link,
717
          author: msg.value.author,
718
          ts: msg.value.timestamp,
719
        }
720
      })
721
    }),
722
    pull.flatten(),
723
    pull.filter(typeof name === 'string' ? function (link) {
724
      return link.name === name
725
    } : name && name.$prefix ? function (link) {
726
      return link.name.substr(0, name.$prefix.length) === name.$prefix
727
    } : function () {
728
      throw new TypeError('unsupported name filter')
729
    })
730
  )
731
}
732
733
Req.prototype.getMentions = function (name) {
734
  var useMsgMentions = this.headMsgIds
735
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
736
  if (useServerMentions && !this.server.sbot.links2) {
737
    return this.headMsgPlus
738
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
739
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
740
  }
741
  return cat([
742
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
743
    useServerMentions ? this.server.getMentions(name) : pull.empty()
744
  ])
745
}
746
747
Req.prototype.getMentionLinks = function (blobId) {
748
  return pull(
749
    this.headMsgIds
750
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
751
      : this.server.sbot.links({
752
        dest: blobId,
753
        rel: 'mentions',
754
        values: true,
755
      }),
756
    // decryption could be done here
757
    pull.map(function (msg) {
758
      var c = msg.value && msg.value.content
759
      return c && Array.isArray(c.mentions) && c.mentions || []
760
    }),
761
    pull.flatten(),
762
    pull.filter(function (link) {
763
      return link && link.link === blobId
764
    })
765
  )
766
}
767
768
Req.prototype.servePkg = function (pathname) {
769
  var self = this
770
  var parts = pathname.split('/')
771
  var pkgName = parts.shift().replace(/%2f/i, '/')
772
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
773
  var spec = parts.shift()
774
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
775
  if (parts.length > 0) return this.respondError(404)
776
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
777
  var obj = {
778
    _id: pkgName,
779
    name: pkgName,
780
    'dist-tags': {},
781
    versions: {},
782
    time: {}
783
  }
784
  var distTags = {/* <tag>: {version, ts}*/}
785
  pull(
786
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
787
    pull.drain(function (mention) {
788
      var data = decodeName(mention.name)
789
      if (!data.version) return
790
      if (data.distTag) {
791
        var tag = distTags[data.distTag]
792
        if (!tag || mention.ts > tag.ts) {
793
          /* TODO: sort by causal order instead of only by timestamps */
794
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
795
        }
796
      }
797
      obj.versions[data.version] = {
798
        author: {
799
          url: mention.author
800
        },
801
        name: pkgName,
802
        version: data.version,
803
        dist: self.blobDist(mention.link)
804
      }
805
      var ts = new Date(mention.ts)
806
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
807
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
808
      obj.time[data.version] = ts.toISOString()
809
    }, function (err) {
810
      if (err) return self.respondError(500, err.stack || err)
811
      for (var tag in distTags) {
812
        obj['dist-tags'][tag] = distTags[tag].version
813
      }
814
      if (spec) resolveSpec()
815
      else if (self.fetchAll) resolveAll()
816
      else resolved()
817
    })
818
  )
819
  function resolveSpec() {
820
    var version = obj['dist-tags'][spec]
821
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
822
    obj = obj.versions[version]
823
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
824
    self.populatePackageJson(obj, function (err, pkg) {
825
      if (err) return resolved(err)
826
      obj = pkg || obj
827
      resolved()
828
    })
829
  }
830
  function resolveVersion(version, cb) {
831
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
832
      if (err) return cb(err)
833
      if (pkg) obj.versions[version] = pkg
834
      cb()
835
    })
836
  }
837
  function resolveAll() {
838
    var done = multicb()
839
    for (var version in obj.versions) {
840
      resolveVersion(version, done())
841
    }
842
    done(resolved)
843
  }
844
  function resolved(err) {
845
    if (err) return self.respondError(500, err.stack || err)
846
    self.respond(200, obj)
847
  }
848
}
849
850
Req.prototype.servePrebuild = function (name) {
851
  var self = this
852
  var getMention = self.getMentions('prebuild:' + name)
853
  var blobsByAuthor = {/* <author>: BlobId */}
854
  pull(getMention, pull.drain(function (link) {
855
    blobsByAuthor[link.author] = link.link
856
  }, function (err) {
857
    if (err) return self.respondError(500, err.stack || err)
858
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
859
    var blobId
860
    for (var feed in blobsByAuthor) {
861
      var blob = blobId = blobsByAuthor[feed]
862
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
863
      feeds.push(feed)
864
    }
865
    switch (Object.keys(authorsByLink).length) {
866
      case 0:
867
        return self.respondError(404, 'Not Found')
868
      case 1:
869
        return self.serveBlob(blobId)
870
      default:
871
        return self.respond(300, {choices: authorsByLink})
872
    }
873
  }))
874
}
875
876
var localhosts = {
877
  '::1': true,
878
  '127.0.0.1': true,
879
  '::ffff:127.0.0.1': true,
880
}
881
882
Req.prototype.publishPkg = function (pkgName) {
883
  var self = this
884
  var remoteAddress = self.req.socket.remoteAddress
885
  if (!(remoteAddress in localhosts)) {
886
    return self.respondError(403, 'You may not publish as this user.')
887
  }
888
889
  var chunks = []
890
  self.req.on('data', function (data) {
891
    chunks.push(data)
892
  })
893
  self.req.on('end', function () {
894
    var data
895
    try {
896
      data = JSON.parse(Buffer.concat(chunks))
897
    } catch(e) {
898
      return self.respondError(400, e.stack)
899
    }
900
    return self.publishPkg2(pkgName, data || {})
901
  })
902
}
903
904
Req.prototype.publishPkg2 = function (name, data) {
905
  var self = this
906
  if (data.users) console.trace('[npm-registry] users property is not supported')
907
  var attachments = data._attachments || {}
908
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
909
  var done = multicb()
910
  function addAttachmentAsBlob(filename, cb) {
911
    var data = attachments[filename].data
912
    var tarball = new Buffer(data, 'base64')
913
    var length = attachments[filename].length
914
    if (length && length !== tarball.length) return self.respondError(400,
915
      'Length mismatch for attachment \'' + filename + '\'')
916
    self.server.sbot.blobs.add(function (err, id) {
917
      if (err) return cb(err)
918
      self.blobsToPush.push(id)
919
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
920
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
921
      cb()
922
    })(pull.once(tarball))
923
  }
924
  for (var filename in attachments) {
925
    addAttachmentAsBlob(filename, done())
926
  }
927
  done(function (err) {
928
    if (err) return self.respondError(500, err.stack || err)
929
    try {
930
      self.publishPkg3(name, data, links)
931
    } catch(e) {
932
      self.respondError(500, e.stack || e)
933
    }
934
  })
935
}
936
937
Req.prototype.publishPkg3 = function (name, data, links) {
938
  var self = this
939
  var versions = data.versions || {}
940
  var linksByVersion = {/* <version>: link */}
941
942
  // associate tarball blobs with versions
943
  for (var version in versions) {
944
    var pkg = versions[version]
945
    if (!pkg) return self.respondError(400, 'Bad package object')
946
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
947
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
948
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
949
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
950
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
951
    var filename = m[1]
952
    var link = links[filename]
953
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
954
    // TODO?: try to find missing tarball mentioned in other messages
955
    if (pkg.version && pkg.version !== version)
956
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
957
    linksByVersion[version] = link
958
    link.version = version
959
    link.dependencies = pkg.dependencies || {}
960
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
961
  }
962
963
  // associate blobs with dist-tags
964
  var tags = data['dist-tags'] || {}
965
  for (var tag in tags) {
966
    var version = tags[tag]
967
    var link = linksByVersion[version]
968
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
969
    // TODO?: support setting dist-tag without version,
970
    // by looking up a tarball blob for the version
971
    link.tag = tag
972
  }
973
974
  // compute blob links to publish
975
  var mentions = []
976
  for (var filename in links) {
977
    var link = links[filename] || {}
978
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
979
    mentions.push({
980
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
981
      link: link.link,
982
      size: link.size,
983
      shasum: link.shasum,
984
      dependencies: link.dependencies,
985
      bundledDependencies: link.bundledDependencies,
986
    })
987
  }
988
  return self.publishPkgs(mentions)
989
}
990
991
Req.prototype.publishPkgs = function (mentions) {
992
  var self = this
993
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
994
    if (err) return self.respondError(500, err.stack || err)
995
    self.server.pushBlobs(self.blobsToPush, function (err) {
996
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
997
      self.respond(201)
998
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
999
    })
1000
  })
1001
}
1002
1003
Req.prototype.populatePackageJson = function (obj, cb) {
1004
  var self = this
1005
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1006
  var deps, bundledDeps, shasum
1007
1008
  // look for dependencies in links.
1009
  // then fallback to getting it from the tarball blob
1010
1011
  pull(
1012
    self.getMentionLinks(blobId),
1013
    pull.drain(function (link) {
1014
      if (link.dependencies) deps = link.dependencies
1015
      if (link.shasum) shasum = link.shasum
1016
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1017
      // how to handle multiple assignments of dependencies to a package?
1018
    }, function (err) {
1019
      if (err) return cb(new Error(err.stack || err))
1020
      if (deps && (shasum || !self.needShasum)) {
1021
        // assume that the dependencies in the links to the blob are
1022
        // correct.
1023
        obj.dependencies = deps
1024
        obj.bundledDependencies = bundledDeps
1025
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1026
        cb(null, obj)
1027
      } else {
1028
        // get dependencies from the tarball
1029
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1030
          if (err) return cb(err)
1031
          pkg.dist = obj.dist
1032
          pkg.dist.shasum = pkg._shasum
1033
          pkg.author = pkg.author || obj.author
1034
          pkg.version = pkg.version || obj.version
1035
          pkg.name = pkg.name || obj.name
1036
          cb(null, pkg)
1037
        })
1038
      }
1039
    })
1040
  )
1041
}
1042
1043
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1044
  var self = this
1045
  getBlob(sbot, id, function (err, readBlob) {
1046
    if (err) return cb(err)
1047
    cb = once(cb)
1048
    var extract = tar.extract()
1049
    var pkg, shasum
1050
    extract.on('entry', function (header, stream, next) {
1051
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1052
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1053
          if (err) return cb(err)
1054
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1055
          catch(e) { return cb(e) }
1056
          next()
1057
        }))
1058
      } else {
1059
        stream.on('end', next)
1060
        stream.resume()
1061
      }
1062
    })
1063
    extract.on('finish', function () {
1064
      pkg._shasum = shasum
1065
      cb(null, pkg)
1066
    })
1067
    pull(
1068
      readBlob,
1069
      hash('sha1', 'hex', function (err, sum) {
1070
        if (err) return cb(err)
1071
        shasum = sum
1072
      }),
1073
      toPull(zlib.createGunzip()),
1074
      toPull(extract)
1075
    )
1076
  })
1077
}
1078