git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || null
90
  var autoAuth = conf.autoAuth !== false
91
92
  var server = http.createServer(exports.respond(sbot, config))
93
  var getAddress = onceify(function (cb) {
94
    server.on('error', cb)
95
    server.listen(port, host, function () {
96
      server.removeListener('error', cb)
97
      var regHost = formatHost(host || 'localhost')
98
      var regPort = this.address().port
99
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
100
      if (autoAuth) npmLogin(regUrl, next)
101
      else next()
102
      function next(err) {
103
        cb(err, regUrl)
104
      }
105
    })
106
    sbot.on('close', function () {
107
      server.close()
108
    })
109
  })
110
111
  getAddress(function (err, addr) {
112
    if (err) return console.error('[npm-registry]', err.stack || err)
113
    console.log('[npm-registry] Listening on ' + addr)
114
  })
115
116
  return {
117
    getAddress: getAddress
118
  }
119
}
120
121
exports.respond = function (sbot, config) {
122
  var reg = new SsbNpmRegistryServer(sbot, config)
123
  return function (req, res) {
124
    new Req(reg, req, res).serve()
125
  }
126
}
127
128
function publishMsg(sbot, value, cb) {
129
  var gotExpectedPrevious = false
130
  sbot.publish(value, function next(err, msg) {
131
    if (err && /^expected previous:/.test(err.message)) {
132
      // retry once on this error
133
      if (gotExpectedPrevious) return cb(err)
134
      gotExpectedPrevious = true
135
      return sbot.publish(value, next)
136
    }
137
    cb(err, msg)
138
  })
139
}
140
141
function getDependencyBranches(sbot, link, cb) {
142
  var id = link.link
143
144
  // get ids of heads of tree of dependencyBranch message links to include all
145
  // the dependencies of the given tarball id.
146
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
147
  var msgs = {}
148
  var branched = {}
149
  var blobs = {}
150
151
  function addPkgById(id, cb) {
152
    if (blobs[id]) return cb()
153
    blobs[id] = true
154
    getPackageJsonCached(id, function (err, pkg) {
155
      if (err) return cb(err)
156
      var done = multicb()
157
      for (var name in pkg.dependencies || {}) {
158
        addPkgBySpec(name, pkg.dependencies[name], done())
159
      }
160
      done(cb)
161
    })
162
  }
163
164
  function addPkgBySpec(name, spec, cb) {
165
    var versions = {}
166
    var distTags = {}
167
    pull(
168
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
169
      pull.filter(mentionMatches(null, name, spec)),
170
      pull.map(function (mention) {
171
        // query to get the messages since links2 does not include the msg id
172
        return sbot.links({
173
          source: mention.author,
174
          dest: mention.link,
175
          rel: 'mentions',
176
          values: true,
177
        })
178
      }),
179
      pull.flatten(),
180
      pull.drain(function (msg) {
181
        var c = msg && msg.value && msg.value.content
182
        if (!c || !Array.isArray(c.mentions)) return
183
        c.mentions.forEach(function (link) {
184
          var data = link && link.name && decodeName(link.name)
185
          if (!data || data.name !== name) return
186
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
187
          if (data.distTag) distTags[data.distTag] = data.version
188
        })
189
      }, function (err) {
190
        if (err) return cb(err)
191
        var version = distTags[spec]
192
          || semver.maxSatisfying(Object.keys(versions), spec)
193
        var item = versions[version]
194
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
195
        addDepBranchMsg(item.msg)
196
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
197
        addPkgById(item.mention.link, cb)
198
      })
199
    )
200
  }
201
202
  function addDepBranchMsg(msg) {
203
    var c = msg && msg.value && msg.value.content
204
    if (!c) return
205
    if (msgs[msg.key]) return
206
    msgs[msg.key] = msg.value
207
    if (Array.isArray(c.dependencyBranch)) {
208
      for (var k = 0; k < c.dependencyBranch.length; k++) {
209
        branched[c.dependencyBranch[k]] = true
210
      }
211
    }
212
    if (Array.isArray(c.versionBranch)) {
213
      for (var k = 0; k < c.versionBranch.length; k++) {
214
        branched[c.versionBranch[k]] = true
215
      }
216
    }
217
  }
218
219
  function addPkgsByName(name, cb) {
220
    var data = decodeName(name)
221
    pull(
222
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
223
      pull.map(function (mention) {
224
        return packageLinks(sbot, mention.author, mention.link, data.name)
225
      }),
226
      pull.flatten(),
227
      pull.drain(addDepBranchMsg, cb)
228
    )
229
  }
230
231
  var done = multicb()
232
  addPkgsByName(link.name, done())
233
  addPkgById(id, done())
234
  done(function (err) {
235
    if (err) return cb(err)
236
    var ids = []
237
    for (var key in msgs) {
238
      if (!branched[key]) ids.push(key)
239
    }
240
    cb(null, ids)
241
  })
242
}
243
244
function packageLinks(sbot, feed, id, name, spec) {
245
  var matches = mentionMatches(id, name, spec)
246
  return pull(
247
    sbot.links({
248
      source: feed,
249
      dest: id,
250
      rel: 'mentions',
251
      values: true,
252
    }),
253
    pull.filter(function (msg) {
254
      var c = msg && msg.value && msg.value.content
255
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
256
    })
257
  )
258
}
259
260
// For each dependency that is not a bundledDependency, get message ids for
261
// that dependency name + version.
262
function publishSingleMention(sbot, mention, cb) {
263
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
264
  // Calculate dependencyBranch message ids.
265
  var value = {
266
    type: 'npm-packages',
267
    mentions: [mention]
268
  }
269
  var done = multicb({pluck: 1, spread: true})
270
  getDependencyBranches(sbot, mention, function (err, branches) {
271
    if (err) return cb(err)
272
    value.dependencyBranch = branches || undefined
273
    publishMsg(sbot, value, cb)
274
  })
275
}
276
277
function publishMentions(sbot, mentions, cb) {
278
  // console.error("publishing %s mentions", mentions.length)
279
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
280
  // if it is just one mention, fetch and add useful metadata
281
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
282
  publishMsg(sbot, {
283
    type: 'npm-packages',
284
    mentions: mentions,
285
  }, cb)
286
}
287
288
exports.publishPkgMentions = function (sbot, mentions, cb) {
289
  // try to fit the mentions into as few messages as possible,
290
  // while fitting under the message size limit.
291
  var msgs = []
292
  ;(function next(i, chunks) {
293
    if (i >= mentions.length) return cb(null, msgs)
294
    var chunkLen = Math.ceil(mentions.length / chunks)
295
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
296
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
297
      if (err && msgs.length) return onPartialPublish(err)
298
      if (err) return cb(err)
299
      msgs.push(msg)
300
      next(i + chunkLen, chunks)
301
    })
302
  })(0, 1)
303
  function onPartialPublish(err) {
304
    var remaining = mentions.length - i
305
    return cb(new Error('Published messages ' +
306
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
307
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
308
  }
309
}
310
311
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
312
  cb = once(cb)
313
  var waiting = 0
314
  var expandedMentions = mentions.map(function (link) {
315
    var id = link && link.link
316
    if (!id) return link
317
    waiting++
318
    var newLink = {}
319
    for (var k in link) newLink[k] = link[k]
320
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
321
      if (err) return cb(err)
322
      for (var k in props) newLink[k] = pkg[k]
323
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
324
      if (!--waiting) next()
325
    })
326
    return newLink
327
  })
328
  if (!waiting) next()
329
  function next() {
330
    cb(null, expandedMentions)
331
  }
332
}
333
334
function SsbNpmRegistryServer(sbot, config) {
335
  this.sbot = sbot
336
  this.config = config
337
  this.npmConfig = config.npm || {}
338
  this.host = this.npmConfig.host || 'localhost'
339
  this.fetchAll = this.npmConfig.fetchAll
340
  this.needShasum = this.npmConfig.needShasum
341
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
342
}
343
344
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
345
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
346
347
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
348
  var self = this
349
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
350
  ;(function next(i) {
351
    if (i >= ids.length) return cb()
352
    self.sbot.blobs.push(ids[i], function (err) {
353
      if (err) return cb(err)
354
      next(i+1)
355
    })
356
  })(0)
357
}
358
359
function getBlob(sbot, id, cb) {
360
  var blobs = sbot.blobs
361
  blobs.size(id, function (err, size) {
362
    if (err) return cb(err)
363
    if (typeof size === 'number') cb(null, blobs.get(id), size)
364
    else blobs.want(id, function (err, got) {
365
      if (err) cb(err)
366
      else if (!got) cb('missing blob ' + id)
367
      else blobs.size(id, function (err, size) {
368
        if (err) return cb(err)
369
        cb(null, blobs.get(id), size)
370
      })
371
    })
372
  })
373
}
374
375
function getMentions(links2, name) {
376
  return links2.read({
377
    query: [
378
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
379
      {$filter: {dest: {$prefix: '&'}}},
380
      {$map: {
381
        name: ['rel', 1],
382
        size: ['rel', 2],
383
        link: 'dest',
384
        author: 'source',
385
        ts: 'ts'
386
      }}
387
    ]
388
  })
389
}
390
391
function mentionMatches(id, name, spec) {
392
  return function (mention) {
393
    var data = mention
394
      && (!id || id === mention.link)
395
      && mention.name
396
      && decodeName(mention.name)
397
    return data
398
      && data.name === name
399
      && (spec ? semver.satisfies(data.version, spec) : true)
400
  }
401
}
402
403
SsbNpmRegistryServer.prototype.getMentions = function (name) {
404
  if (!this.sbot.links2) return pull.empty()
405
  return getMentions(this.sbot.links2, name)
406
}
407
408
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
409
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
410
  else this.sbot.get(id, function (err, value) {
411
    if (err) return cb(err)
412
    cb(null, {key: id, value: value})
413
  })
414
}
415
416
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
417
  var self = this
418
  var stack = heads.slice()
419
  var seen = {}
420
  return function (abort, cb) {
421
    if (abort) return cb(abort)
422
    for (var id; stack.length && seen[id = stack.pop()];);
423
    if (!id) return cb(true)
424
    seen[id] = true
425
    // TODO: use DFS
426
    self.getMsg(id, function (err, msg) {
427
      if (err) return cb(new Error(err.stack || err))
428
      var c = msg && msg.value && msg.value.content
429
      var links = c && c[prop]
430
      if (Array.isArray(links)) {
431
        stack.push.apply(stack, links)
432
      } else if (links) {
433
        stack.push(links)
434
      }
435
      cb(null, msg)
436
    })
437
  }
438
}
439
440
function Req(server, req, res) {
441
  this.server = server
442
  this.req = req
443
  this.res = res
444
  this.blobsToPush = []
445
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
446
  var ua = this.req.headers['user-agent']
447
  var m = /\bnpm\/([0-9]*)/.exec(ua)
448
  var npmVersion = m && m[1]
449
  this.needShasum = server.needShasum != null ? server.needShasum :
450
    (npmVersion && npmVersion < 5)
451
  this.baseUrl = this.server.npmConfig.baseUrl
452
  if (this.baseUrl) {
453
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
454
  } else {
455
    var hostname = req.headers.host
456
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
457
    this.baseUrl = 'http://' + hostname
458
  }
459
}
460
461
Req.prototype.serve = function () {
462
  console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
463
  this.res.setTimeout(0)
464
  var pathname = this.req.url.replace(/\?.*/, '')
465
  var m
466
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
467
    try {
468
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
469
      this.headMsgPlus = !!m[1]
470
      // ^ means also include packages published after the head message id
471
    } catch(e) {
472
      return this.respondError(400, e.stack || e)
473
    }
474
    pathname = m[3]
475
  }
476
  if (pathname === '/') return this.serveHome()
477
  if (pathname === '/robots.txt') return this.serveRobots()
478
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
479
  if (pathname === '/-/whoami') return this.serveWhoami()
480
  if (pathname === '/-/ping') return this.respond(200, true)
481
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
482
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
483
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
484
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
485
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
486
  return this.respond(404, {error: 'Not found'})
487
}
488
489
Req.prototype.respond = function (status, message) {
490
  this.res.writeHead(status, {'content-type': 'application/json'})
491
  this.res.end(message && JSON.stringify(message, 0, 2))
492
}
493
494
Req.prototype.respondError = function (status, message) {
495
  this.respond(status, {error: message})
496
}
497
498
Req.prototype.respondErrorStr = function (status, err) {
499
  this.res.writeHead(status, {'content-type': 'text/plain'})
500
  this.res.end(err.stack || err)
501
}
502
503
Req.prototype.respondRaw = function (status, body) {
504
  this.res.writeHead(status)
505
  this.res.end(body)
506
}
507
508
Req.prototype.serveHome = function () {
509
  var self = this
510
  self.res.writeHead(200, {'content-type': 'text/html'})
511
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
512
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
513
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
514
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
515
    '</body></html>')
516
}
517
518
Req.prototype.blobDist = function (id) {
519
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
520
  if (!m) throw new Error('bad blob id: ' + id)
521
  return {
522
    integrity: m[2] + '-' + m[1],
523
    tarball: this.baseUrl + '/-/blobs/get/' + id
524
  }
525
}
526
527
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
528
  var self = this
529
  pull(
530
    self.server.sbot.links({
531
      source: feedId,
532
      dest: blobId,
533
      rel: 'mentions',
534
      values: true,
535
    }),
536
    pull.filter(function (msg) {
537
      var c = msg && msg.value && msg.value.content
538
      return c.type === 'npm-packages'
539
    }),
540
    pull.collect(function (err, msgs) {
541
      if (err) return cb(err)
542
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
543
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
544
      // TODO: make a smarter decision about which message id to use
545
      cb(null, msgs.pop().key)
546
    })
547
  )
548
}
549
550
Req.prototype.resolvePkg = function (pkgSpec, cb) {
551
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
552
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
553
  var self = this
554
  var pkgName = m[1]
555
  var spec = m[2] || '*'
556
  var versions = {}
557
  var distTags = {}
558
  pull(
559
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
560
    pull.drain(function (mention) {
561
      var data = decodeName(mention.name)
562
      if (!data.version) return
563
      if (data.distTag) {
564
        distTags[data.distTag] = data.version
565
      }
566
      versions[data.version] = {
567
        author: mention.author,
568
        name: pkgName,
569
        version: data.version,
570
        blobId: mention.link
571
      }
572
    }, function (err) {
573
      if (err) return cb(err)
574
      var version = distTags[spec]
575
        || semver.maxSatisfying(Object.keys(versions), spec)
576
      var item = versions[version]
577
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
578
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
579
        if (err) return cb(err)
580
        item.msgId = id
581
        cb(null, item)
582
      })
583
    })
584
  )
585
}
586
587
Req.prototype.resolvePkgs = function (specs, cb) {
588
  var done = multicb({pluck: 1})
589
  var self = this
590
  specs.forEach(function (spec) {
591
    self.resolvePkg(spec, done())
592
  })
593
  done(cb)
594
}
595
596
Req.prototype.serveBootstrap = function () {
597
  var self = this
598
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
599
  var ssbNpmRegistryName = require('./package.json').name
600
  var ssbNpmRegistryVersion = require('./package.json').version
601
  var ssbNpmRegistryNameVersion = ssbNpmRegistryName + '-' + ssbNpmRegistryVersion
602
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
603
  var done = multicb({pluck: 1, spread: true})
604
  self.resolvePkg(ssbNpmRegistrySpec, done())
605
  self.resolvePkgs(pkgs, done())
606
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
607
    if (err) return self.respondErrorStr(500, err.stack || err)
608
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
609
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
610
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
611
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
612
    var globalPkgs = pkgsInfo.map(function (info) {
613
      return info.name + '@' + info.version
614
    }).join(' ')
615
    var npmCmd = 'install -g ' + globalPkgs
616
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
617
    var wsUrl = self.baseUrl + '/-'
618
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
619
620
    var script =
621
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
622
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
623
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
624
      'tar xzf package.tgz &&\n' +
625
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
626
        pkgMsgs.map(function (id) {
627
          return '  --branch ' + id + ' \\\n'
628
        }).join('') +
629
        '  -- ' + npmCmd + ' &&\n' +
630
      'sbot server'
631
632
    self.res.writeHead(200, {'Content-type': 'text/plain'})
633
    self.res.end(script)
634
  })
635
}
636
637
Req.prototype.serveRobots = function () {
638
  this.res.writeHead(200, {'Content-type': 'text/plain'})
639
  this.res.end('User-agent: *\nDisallow: /\n')
640
}
641
642
Req.prototype.serveWhoami = function () {
643
  var self = this
644
  self.server.sbot.whoami(function (err, feed) {
645
    if (err) return self.respondError(500, err.stack || err)
646
    self.respond(200, {username: feed.id})
647
  })
648
}
649
650
Req.prototype.serveUser1 = function () {
651
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
652
}
653
654
Req.prototype.serveBlob = function (id) {
655
  var self = this
656
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
657
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
658
    if (err) {
659
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
660
      else return self.respondErrorStr(500, err.message || err)
661
    }
662
    self.res.writeHead(200, {
663
      'Cache-Control': 'public, max-age=315360000',
664
      'Content-Length': size,
665
      'etag': id
666
    })
667
    pull(
668
      readBlob,
669
      toPull(self.res, function (err) {
670
        if (err) console.error('[npm-registry]', err)
671
      })
672
    )
673
  })
674
}
675
676
Req.prototype.serveMsg = function (id) {
677
  var self = this
678
  try { id = decodeURIComponent(id) }
679
  catch (e) {}
680
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
681
  self.server.sbot.get(id, function (err, value) {
682
    if (err) return self.respondError(500, err.message || err)
683
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
684
    self.res.writeHead(200, {
685
      'Content-Type': 'application/json',
686
      'Cache-Control': 'public, max-age=315360000',
687
      'Content-Length': out.length,
688
      'etag': id
689
    })
690
    self.res.end(out)
691
  })
692
}
693
694
function decodeName(name) {
695
  var parts = String(name).replace(/\.tgz$/, '').split(':')
696
  return {
697
    name: parts[1],
698
    version: parts[2],
699
    distTag: parts[3],
700
  }
701
}
702
703
Req.prototype.getMsgMentions = function (name) {
704
  return pull(
705
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
706
    // decryption could be done here
707
    pull.map(function (msg) {
708
      var c = msg.value && msg.value.content
709
      if (!c.mentions || !Array.isArray(c.mentions)) return []
710
      return c.mentions.map(function (mention) {
711
        return {
712
          name: mention.name,
713
          size: mention.size,
714
          link: mention.link,
715
          author: msg.value.author,
716
          ts: msg.value.timestamp,
717
        }
718
      })
719
    }),
720
    pull.flatten(),
721
    pull.filter(typeof name === 'string' ? function (link) {
722
      return link.name === name
723
    } : name && name.$prefix ? function (link) {
724
      return link.name.substr(0, name.$prefix.length) === name.$prefix
725
    } : function () {
726
      throw new TypeError('unsupported name filter')
727
    })
728
  )
729
}
730
731
Req.prototype.getMentions = function (name) {
732
  var useMsgMentions = this.headMsgIds
733
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
734
  if (useServerMentions && !this.server.sbot.links2) {
735
    return this.headMsgPlus
736
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
737
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
738
  }
739
  return cat([
740
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
741
    useServerMentions ? this.server.getMentions(name) : pull.empty()
742
  ])
743
}
744
745
Req.prototype.getMentionLinks = function (blobId) {
746
  return pull(
747
    this.headMsgIds
748
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
749
      : this.server.sbot.links({
750
        dest: blobId,
751
        rel: 'mentions',
752
        values: true,
753
      }),
754
    // decryption could be done here
755
    pull.map(function (msg) {
756
      var c = msg.value && msg.value.content
757
      return c && Array.isArray(c.mentions) && c.mentions || []
758
    }),
759
    pull.flatten(),
760
    pull.filter(function (link) {
761
      return link && link.link === blobId
762
    })
763
  )
764
}
765
766
Req.prototype.servePkg = function (pathname) {
767
  var self = this
768
  var parts = pathname.split('/')
769
  var pkgName = parts.shift().replace(/%2f/i, '/')
770
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
771
  var spec = parts.shift()
772
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
773
  if (parts.length > 0) return this.respondError(404)
774
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
775
  var obj = {
776
    _id: pkgName,
777
    name: pkgName,
778
    'dist-tags': {},
779
    versions: {}
780
  }
781
  var distTags = {/* <tag>: {version, ts}*/}
782
  pull(
783
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
784
    pull.drain(function (mention) {
785
      var data = decodeName(mention.name)
786
      if (!data.version) return
787
      if (data.distTag) {
788
        var tag = distTags[data.distTag]
789
        if (!tag || mention.ts > tag.ts) {
790
          /* TODO: sort by causal order instead of only by timestamps */
791
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
792
        }
793
      }
794
      obj.versions[data.version] = {
795
        author: {
796
          url: mention.author
797
        },
798
        name: pkgName,
799
        version: data.version,
800
        dist: self.blobDist(mention.link)
801
      }
802
    }, function (err) {
803
      if (err) return self.respondError(500, err.stack || err)
804
      for (var tag in distTags) {
805
        obj['dist-tags'][tag] = distTags[tag].version
806
      }
807
      if (spec) resolveSpec()
808
      else if (self.fetchAll) resolveAll()
809
      else resolved()
810
    })
811
  )
812
  function resolveSpec() {
813
    var version = obj['dist-tags'][spec]
814
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
815
    obj = obj.versions[version]
816
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
817
    self.populatePackageJson(obj, function (err, pkg) {
818
      if (err) return resolved(err)
819
      obj = pkg || obj
820
      resolved()
821
    })
822
  }
823
  function resolveVersion(version, cb) {
824
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
825
      if (err) return cb(err)
826
      if (pkg) obj.versions[version] = pkg
827
      cb()
828
    })
829
  }
830
  function resolveAll() {
831
    var done = multicb()
832
    for (var version in obj.versions) {
833
      resolveVersion(version, done())
834
    }
835
    done(resolved)
836
  }
837
  function resolved(err) {
838
    if (err) return self.respondError(500, err.stack || err)
839
    self.respond(200, obj)
840
  }
841
}
842
843
Req.prototype.servePrebuild = function (name) {
844
  var self = this
845
  var getMention = self.getMentions('prebuild:' + name)
846
  var blobsByAuthor = {/* <author>: BlobId */}
847
  pull(getMention, pull.drain(function (link) {
848
    blobsByAuthor[link.author] = link.link
849
  }, function (err) {
850
    if (err) return self.respondError(500, err.stack || err)
851
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
852
    var blobId
853
    for (var feed in blobsByAuthor) {
854
      var blob = blobId = blobsByAuthor[feed]
855
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
856
      feeds.push(feed)
857
    }
858
    switch (Object.keys(authorsByLink).length) {
859
      case 0:
860
        return self.respondError(404, 'Not Found')
861
      case 1:
862
        return self.serveBlob(blobId)
863
      default:
864
        return self.respond(300, {choices: authorsByLink})
865
    }
866
  }))
867
}
868
869
var localhosts = {
870
  '::1': true,
871
  '127.0.0.1': true,
872
  '::ffff:127.0.0.1': true,
873
}
874
875
Req.prototype.publishPkg = function (pkgName) {
876
  var self = this
877
  var remoteAddress = self.req.socket.remoteAddress
878
  if (!(remoteAddress in localhosts)) {
879
    return self.respondError(403, 'You may not publish as this user.')
880
  }
881
882
  var chunks = []
883
  self.req.on('data', function (data) {
884
    chunks.push(data)
885
  })
886
  self.req.on('end', function () {
887
    var data
888
    try {
889
      data = JSON.parse(Buffer.concat(chunks))
890
    } catch(e) {
891
      return self.respondError(400, e.stack)
892
    }
893
    return self.publishPkg2(pkgName, data || {})
894
  })
895
}
896
897
Req.prototype.publishPkg2 = function (name, data) {
898
  var self = this
899
  if (data.users) console.trace('[npm-registry] users property is not supported')
900
  var attachments = data._attachments || {}
901
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
902
  var done = multicb()
903
  function addAttachmentAsBlob(filename, cb) {
904
    var data = attachments[filename].data
905
    var tarball = new Buffer(data, 'base64')
906
    var length = attachments[filename].length
907
    if (length && length !== tarball.length) return self.respondError(400,
908
      'Length mismatch for attachment \'' + filename + '\'')
909
    self.server.sbot.blobs.add(function (err, id) {
910
      if (err) return cb(err)
911
      self.blobsToPush.push(id)
912
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
913
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
914
      cb()
915
    })(pull.once(tarball))
916
  }
917
  for (var filename in attachments) {
918
    addAttachmentAsBlob(filename, done())
919
  }
920
  done(function (err) {
921
    if (err) return self.respondError(500, err.stack || err)
922
    try {
923
      self.publishPkg3(name, data, links)
924
    } catch(e) {
925
      self.respondError(500, e.stack || e)
926
    }
927
  })
928
}
929
930
Req.prototype.publishPkg3 = function (name, data, links) {
931
  var self = this
932
  var versions = data.versions || {}
933
  var linksByVersion = {/* <version>: link */}
934
935
  // associate tarball blobs with versions
936
  for (var version in versions) {
937
    var pkg = versions[version]
938
    if (!pkg) return self.respondError(400, 'Bad package object')
939
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
940
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
941
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
942
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
943
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
944
    var filename = m[1]
945
    var link = links[filename]
946
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
947
    // TODO?: try to find missing tarball mentioned in other messages
948
    if (pkg.version && pkg.version !== version)
949
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
950
    linksByVersion[version] = link
951
    link.version = version
952
    link.dependencies = pkg.dependencies || {}
953
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
954
  }
955
956
  // associate blobs with dist-tags
957
  var tags = data['dist-tags'] || {}
958
  for (var tag in tags) {
959
    var version = tags[tag]
960
    var link = linksByVersion[version]
961
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
962
    // TODO?: support setting dist-tag without version,
963
    // by looking up a tarball blob for the version
964
    link.tag = tag
965
  }
966
967
  // compute blob links to publish
968
  var mentions = []
969
  for (var filename in links) {
970
    var link = links[filename] || {}
971
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
972
    mentions.push({
973
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
974
      link: link.link,
975
      size: link.size,
976
      shasum: link.shasum,
977
      dependencies: link.dependencies,
978
      bundledDependencies: link.bundledDependencies,
979
    })
980
  }
981
  return self.publishPkgs(mentions)
982
}
983
984
Req.prototype.publishPkgs = function (mentions) {
985
  var self = this
986
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
987
    if (err) return self.respondError(500, err.stack || err)
988
    self.server.pushBlobs(self.blobsToPush, function (err) {
989
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
990
      self.respond(201)
991
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
992
    })
993
  })
994
}
995
996
Req.prototype.populatePackageJson = function (obj, cb) {
997
  var self = this
998
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
999
  var deps, bundledDeps, shasum
1000
1001
  // look for dependencies in links.
1002
  // then fallback to getting it from the tarball blob
1003
1004
  pull(
1005
    self.getMentionLinks(blobId),
1006
    pull.drain(function (link) {
1007
      if (link.dependencies) deps = link.dependencies
1008
      if (link.shasum) shasum = link.shasum
1009
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1010
      // how to handle multiple assignments of dependencies to a package?
1011
    }, function (err) {
1012
      if (err) return cb(new Error(err.stack || err))
1013
      if (deps && (shasum || !self.needShasum)) {
1014
        // assume that the dependencies in the links to the blob are
1015
        // correct.
1016
        obj.dependencies = deps
1017
        obj.bundledDependencies = bundledDeps
1018
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1019
        cb(null, obj)
1020
      } else {
1021
        // get dependencies from the tarball
1022
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1023
          if (err) return cb(err)
1024
          pkg.dist = obj.dist
1025
          pkg.dist.shasum = pkg._shasum
1026
          pkg.author = pkg.author || obj.author
1027
          pkg.version = pkg.version || obj.version
1028
          pkg.name = pkg.name || obj.name
1029
          cb(null, pkg)
1030
        })
1031
      }
1032
    })
1033
  )
1034
}
1035
1036
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1037
  var self = this
1038
  getBlob(sbot, id, function (err, readBlob) {
1039
    if (err) return cb(err)
1040
    cb = once(cb)
1041
    var extract = tar.extract()
1042
    var pkg, shasum
1043
    extract.on('entry', function (header, stream, next) {
1044
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1045
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1046
          if (err) return cb(err)
1047
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1048
          catch(e) { return cb(e) }
1049
          next()
1050
        }))
1051
      } else {
1052
        stream.on('end', next)
1053
        stream.resume()
1054
      }
1055
    })
1056
    extract.on('finish', function () {
1057
      pkg._shasum = shasum
1058
      cb(null, pkg)
1059
    })
1060
    pull(
1061
      readBlob,
1062
      hash('sha1', 'hex', function (err, sum) {
1063
        if (err) return cb(err)
1064
        shasum = sum
1065
      }),
1066
      toPull(zlib.createGunzip()),
1067
      toPull(extract)
1068
    )
1069
  })
1070
}
1071