git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
77
}
78
79
exports.name = 'npm-registry'
80
exports.version = '1.0.0'
81
exports.manifest = {
82
  getAddress: 'async'
83
}
84
exports.init = function (sbot, config) {
85
  var conf = config.npm || {}
86
  var port = conf.port || 8043
87
  var host = conf.host || null
88
  var autoAuth = conf.autoAuth !== false
89
90
  var server = http.createServer(exports.respond(sbot, config))
91
  var getAddress = onceify(function (cb) {
92
    server.on('error', cb)
93
    server.listen(port, host, function () {
94
      server.removeListener('error', cb)
95
      var regHost = formatHost(host || 'localhost')
96
      var regPort = this.address().port
97
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
98
      if (autoAuth) npmLogin(regUrl, next)
99
      else next()
100
      function next(err) {
101
        cb(err, regUrl)
102
      }
103
    })
104
    sbot.on('close', function () {
105
      server.close()
106
    })
107
  })
108
109
  getAddress(function (err, addr) {
110
    if (err) return console.error(err)
111
    console.log('[npm-registry] Listening on ' + addr)
112
  })
113
114
  return {
115
    getAddress: getAddress
116
  }
117
}
118
119
exports.respond = function (sbot, config) {
120
  var reg = new SsbNpmRegistryServer(sbot, config)
121
  return function (req, res) {
122
    new Req(reg, req, res).serve()
123
  }
124
}
125
126
function publishMsg(sbot, value, cb) {
127
  var gotExpectedPrevious = false
128
  sbot.publish(value, function next(err, msg) {
129
    if (err && /^expected previous:/.test(err.message)) {
130
      // retry once on this error
131
      if (gotExpectedPrevious) return cb(err)
132
      gotExpectedPrevious = true
133
      return sbot.publish(value, next)
134
    }
135
    cb(err, msg)
136
  })
137
}
138
139
function getDependencyBranches(sbot, link, cb) {
140
  var id = link.link
141
142
  // get ids of heads of tree of dependencyBranch message links to include all
143
  // the dependencies of the given tarball id.
144
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
145
  var msgs = {}
146
  var branched = {}
147
  var blobs = {}
148
149
  function addPkgById(id, cb) {
150
    if (blobs[id]) return cb()
151
    blobs[id] = true
152
    getPackageJsonCached(id, function (err, pkg) {
153
      if (err) return cb(err)
154
      var done = multicb()
155
      for (var name in pkg.dependencies || {}) {
156
        addPkgBySpec(name, pkg.dependencies[name], done())
157
      }
158
      done(cb)
159
    })
160
  }
161
162
  function addPkgBySpec(name, spec, cb) {
163
    var versions = {}
164
    var distTags = {}
165
    pull(
166
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
167
      pull.filter(mentionMatches(null, name, spec)),
168
      pull.map(function (mention) {
169
        // query to get the messages since links2 does not include the msg id
170
        return sbot.links({
171
          source: mention.author,
172
          dest: mention.link,
173
          rel: 'mentions',
174
          values: true,
175
        })
176
      }),
177
      pull.flatten(),
178
      pull.drain(function (msg) {
179
        var c = msg && msg.value && msg.value.content
180
        if (!c || !Array.isArray(c.mentions)) return
181
        c.mentions.forEach(function (link) {
182
          var data = link && link.name && decodeName(link.name)
183
          if (!data || data.name !== name) return
184
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
185
          if (data.distTag) distTags[data.distTag] = data.version
186
        })
187
      }, function (err) {
188
        if (err) return cb(err)
189
        var version = distTags[spec]
190
          || semver.maxSatisfying(Object.keys(versions), spec)
191
        var item = versions[version]
192
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
193
        addDepBranchMsg(item.msg)
194
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
195
        addPkgById(item.mention.link, cb)
196
      })
197
    )
198
  }
199
200
  function addDepBranchMsg(msg) {
201
    var c = msg && msg.value && msg.value.content
202
    if (!c) return
203
    if (msgs[msg.key]) return
204
    msgs[msg.key] = msg.value
205
    if (Array.isArray(c.dependencyBranch)) {
206
      for (var k = 0; k < c.dependencyBranch.length; k++) {
207
        branched[c.dependencyBranch[k]] = true
208
      }
209
    }
210
    if (Array.isArray(c.versionBranch)) {
211
      for (var k = 0; k < c.versionBranch.length; k++) {
212
        branched[c.versionBranch[k]] = true
213
      }
214
    }
215
  }
216
217
  function addPkgsByName(name, cb) {
218
    var data = decodeName(name)
219
    pull(
220
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
221
      pull.map(function (mention) {
222
        return packageLinks(sbot, mention.author, mention.link, data.name)
223
      }),
224
      pull.flatten(),
225
      pull.drain(addDepBranchMsg, cb)
226
    )
227
  }
228
229
  var done = multicb()
230
  addPkgsByName(link.name, done())
231
  addPkgById(id, done())
232
  done(function (err) {
233
    if (err) return cb(err)
234
    var ids = []
235
    for (var key in msgs) {
236
      if (!branched[key]) ids.push(key)
237
    }
238
    cb(null, ids)
239
  })
240
}
241
242
function packageLinks(sbot, feed, id, name, spec) {
243
  var matches = mentionMatches(id, name, spec)
244
  return pull(
245
    sbot.links({
246
      source: feed,
247
      dest: id,
248
      rel: 'mentions',
249
      values: true,
250
    }),
251
    pull.filter(function (msg) {
252
      var c = msg && msg.value && msg.value.content
253
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
254
    })
255
  )
256
}
257
258
function getVersionBranches(sbot, link, cb) {
259
}
260
261
// For each dependency that is not a bundledDependency, get message ids for
262
// that dependency name + version.
263
function publishSingleMention(sbot, mention, cb) {
264
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
265
  // Calculate dependencyBranch message ids.
266
  var value = {
267
    type: 'npm-packages',
268
    mentions: [mention]
269
  }
270
  var done = multicb({pluck: 1, spread: true})
271
  getDependencyBranches(sbot, mention, function (err, branches) {
272
    if (err) return cb(err)
273
    value.dependencyBranch = branches || undefined
274
    publishMsg(sbot, value, cb)
275
  })
276
}
277
278
function publishMentions(sbot, mentions, cb) {
279
  // console.error("publishing %s mentions", mentions.length)
280
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
281
  // if it is just one mention, fetch and add useful metadata
282
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
283
  publishMsg(sbot, {
284
    type: 'npm-packages',
285
    mentions: mentions,
286
  }, cb)
287
}
288
289
exports.publishPkgMentions = function (sbot, mentions, cb) {
290
  // try to fit the mentions into as few messages as possible,
291
  // while fitting under the message size limit.
292
  var msgs = []
293
  ;(function next(i, chunks) {
294
    if (i >= mentions.length) return cb(null, msgs)
295
    var chunkLen = Math.ceil(mentions.length / chunks)
296
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
297
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
298
      if (err && msgs.length) return onPartialPublish(err)
299
      if (err) return cb(err)
300
      msgs.push(msg)
301
      next(i + chunkLen, chunks)
302
    })
303
  })(0, 1)
304
  function onPartialPublish(err) {
305
    var remaining = mentions.length - i
306
    return cb(new Error('Published messages ' +
307
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
308
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
309
  }
310
}
311
312
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
313
  cb = once(cb)
314
  var waiting = 0
315
  var expandedMentions = mentions.map(function (link) {
316
    var id = link && link.link
317
    if (!id) return link
318
    waiting++
319
    var newLink = {}
320
    for (var k in link) newLink[k] = link[k]
321
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
322
      if (err) return cb(err)
323
      for (var k in props) newLink[k] = pkg[k]
324
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
325
      if (!--waiting) next()
326
    })
327
    return newLink
328
  })
329
  if (!waiting) next()
330
  function next() {
331
    cb(null, expandedMentions)
332
  }
333
}
334
335
function SsbNpmRegistryServer(sbot, config) {
336
  this.sbot = sbot
337
  this.config = config
338
  this.npmConfig = config.npm || {}
339
  this.host = this.npmConfig.host || 'localhost'
340
  this.fetchAll = this.npmConfig.fetchAll
341
  this.needShasum = this.npmConfig.needShasum
342
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
343
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
344
    + this.wsPort + '/blobs/get/'
345
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
346
}
347
348
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
349
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
350
351
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
352
  var self = this
353
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
354
  ;(function next(i) {
355
    if (i >= ids.length) return cb()
356
    self.sbot.blobs.push(ids[i], function (err) {
357
      if (err) return cb(err)
358
      next(i+1)
359
    })
360
  })(0)
361
}
362
363
function getBlob(sbot, id, cb) {
364
  var blobs = sbot.blobs
365
  blobs.size(id, function (err, size) {
366
    if (typeof size === 'number') cb(null, blobs.get(id))
367
    else blobs.want(id, function (err, got) {
368
      if (err) cb(err)
369
      else if (!got) cb('missing blob ' + id)
370
      else cb(null, blobs.get(id))
371
    })
372
  })
373
}
374
375
SsbNpmRegistryServer.prototype.blobDist = function (id) {
376
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
377
  if (!m) throw new Error('bad blob id: ' + id)
378
  return {
379
    integrity: m[2] + '-' + m[1],
380
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
381
  }
382
}
383
384
function getMentions(links2, name) {
385
  return links2.read({
386
    query: [
387
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
388
      {$filter: {dest: {$prefix: '&'}}},
389
      {$map: {
390
        name: ['rel', 1],
391
        size: ['rel', 2],
392
        link: 'dest',
393
        author: 'source',
394
        ts: 'ts'
395
      }}
396
    ]
397
  })
398
}
399
400
function mentionMatches(id, name, spec) {
401
  return function (mention) {
402
    var data = mention
403
      && (!id || id === mention.link)
404
      && mention.name
405
      && decodeName(mention.name)
406
    return data
407
      && data.name === name
408
      && (spec ? semver.satisfies(data.version, spec) : true)
409
  }
410
}
411
412
SsbNpmRegistryServer.prototype.getMentions = function (name) {
413
  if (!this.sbot.links2) return pull.empty()
414
  return getMentions(this.sbot.links2, name)
415
}
416
417
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
418
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
419
  else this.sbot.get(id, function (err, value) {
420
    if (err) return cb(err)
421
    cb(null, {key: id, value: value})
422
  })
423
}
424
425
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
426
  var self = this
427
  var stack = heads.slice()
428
  var seen = {}
429
  return function (abort, cb) {
430
    if (abort) return cb(abort)
431
    for (var id; stack.length && seen[id = stack.pop()];);
432
    if (!id) return cb(true)
433
    seen[id] = true
434
    // TODO: use DFS
435
    self.getMsg(id, function (err, msg) {
436
      if (err) return cb(new Error(err.stack || err))
437
      var c = msg && msg.value && msg.value.content
438
      var links = c && c[prop]
439
      if (Array.isArray(links)) {
440
        stack.push.apply(stack, links)
441
      } else if (links) {
442
        stack.push(links)
443
      }
444
      cb(null, msg)
445
    })
446
  }
447
}
448
449
function Req(server, req, res) {
450
  this.server = server
451
  this.req = req
452
  this.res = res
453
  this.blobsToPush = []
454
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
455
  var ua = this.req.headers['user-agent']
456
  var m = /\bnpm\/([0-9]*)/.exec(ua)
457
  var npmVersion = m && m[1]
458
  this.needShasum = server.needShasum != null ? server.needShasum :
459
    (npmVersion && npmVersion < 5)
460
}
461
462
Req.prototype.serve = function () {
463
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
464
  this.res.setTimeout(0)
465
  var pathname = this.req.url.replace(/\?.*/, '')
466
  var m
467
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
468
    try {
469
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
470
      this.headMsgPlus = !!m[1]
471
      // ^ means also include packages published after the head message id
472
    } catch(e) {
473
      return this.respondError(400, e.stack || e)
474
    }
475
    pathname = m[3]
476
  }
477
  if (pathname === '/') return this.serveHome()
478
  if (pathname === '/bootstrap') return this.serveBootstrap()
479
  if (pathname === '/robots.txt') return this.serveRobots()
480
  if (pathname === '/-/whoami') return this.serveWhoami()
481
  if (pathname === '/-/ping') return this.respond(200, true)
482
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
483
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
484
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
485
  return this.respond(404)
486
}
487
488
Req.prototype.respond = function (status, message) {
489
  this.res.writeHead(status, {'content-type': 'application/json'})
490
  this.res.end(message && JSON.stringify(message, 0, 2))
491
}
492
493
Req.prototype.respondError = function (status, message) {
494
  this.respond(status, {error: message})
495
}
496
497
Req.prototype.respondErrorStr = function (status, err) {
498
  this.res.writeHead(status, {'content-type': 'text/plain'})
499
  this.res.end(err.stack || err)
500
}
501
502
Req.prototype.serveHome = function () {
503
  var self = this
504
  self.res.writeHead(200, {'content-type': 'text/html'})
505
  var port = 8044
506
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
507
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
508
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
509
    '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
510
    '</body></html>')
511
}
512
513
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
514
  var self = this
515
  pull(
516
    self.server.sbot.links({
517
      source: feedId,
518
      dest: blobId,
519
      rel: 'mentions',
520
      values: true,
521
    }),
522
    pull.filter(function (msg) {
523
      var c = msg && msg.value && msg.value.content
524
      return c.type === 'npm-packages'
525
    }),
526
    pull.collect(function (err, msgs) {
527
      if (err) return cb(err)
528
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
529
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
530
      // TODO: make a smarter decision about which message id to use
531
      cb(null, msgs.pop().key)
532
    })
533
  )
534
}
535
536
Req.prototype.resolvePkg = function (pkgSpec, cb) {
537
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
538
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
539
  var self = this
540
  var pkgName = m[1]
541
  var spec = m[2] || '*'
542
  var versions = {}
543
  var distTags = {}
544
  pull(
545
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
546
    pull.drain(function (mention) {
547
      var data = decodeName(mention.name)
548
      if (!data.version) return
549
      if (data.distTag) {
550
        distTags[data.distTag] = data.version
551
      }
552
      versions[data.version] = {
553
        author: mention.author,
554
        name: pkgName,
555
        version: data.version,
556
        blobId: mention.link
557
      }
558
    }, function (err) {
559
      if (err) return cb(err)
560
      var version = distTags[spec]
561
        || semver.maxSatisfying(Object.keys(versions), spec)
562
      var item = versions[version]
563
      if (!item) return cb(new Error('Version not found: ' + name + '@' + spec))
564
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
565
        if (err) return cb(err)
566
        item.msgId = id
567
        cb(null, item)
568
      })
569
    })
570
  )
571
}
572
573
Req.prototype.resolvePkgs = function (specs, cb) {
574
  var done = multicb({pluck: 1})
575
  var self = this
576
  specs.forEach(function (spec) {
577
    self.resolvePkg(spec, done())
578
  })
579
  done(cb)
580
}
581
582
Req.prototype.serveBootstrap = function () {
583
  var self = this
584
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
585
  var ssbNpmRegistryName = require('./package.json').name
586
  var ssbNpmRegistryVersion = require('./package.json').version
587
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
588
  var done = multicb({pluck: 1, spread: true})
589
  self.resolvePkg(ssbNpmRegistrySpec, done())
590
  self.resolvePkgs(pkgs, done())
591
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
592
    if (err) return self.respondErrorStr(500, err.stack || err)
593
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
594
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
595
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
596
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
597
    var globalPkgs = pkgsInfo.map(function (info) {
598
      return info.name + '@' + info.version
599
    }).join(' ')
600
    var npmCmd = 'npm install -g ' + globalPkgs
601
602
    var pkgTmpText = '/tmp/' + ssbNpmRegistryName + '.tar.gz'
603
    var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
604
    var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
605
    var wsLink = 'http://' + httpHost + ':' + self.server.wsPort + '/'
606
    var tarballLink = wsLink + 'blobs/get/' + ssbNpmRegistryBlobId
607
608
    var script =
609
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
610
      'echo ' + ssbNpmRegistryBlobHex + ' ' + pkgTmpText + ' | sha256sum -c &&\n' +
611
      'mkdir -p ~/.ssb/node_modules && cd ~/.ssb/node_modules &&\n' +
612
      'tar xzf ' + pkgTmpText + ' &&\n' +
613
      'mv package ' + ssbNpmRegistryName + ' &&\n' +
614
      ssbNpmRegistryName + '/bootstrap/bin.js --ws-url ' + wsLink + ' \\\n' +
615
        pkgMsgs.map(function (id) {
616
          return '  --branch ' + id + ' \\\n'
617
        }).join('') +
618
        '  -- ' + npmCmd + ' &&\n' +
619
      'sbot server'
620
621
    self.res.writeHead(200, {'Content-type': 'text/plain'})
622
    self.res.end(script)
623
  })
624
}
625
626
Req.prototype.serveRobots = function () {
627
  this.res.writeHead(200, {'Content-type': 'text/plain'})
628
  this.res.end('User-agent: *\nDisallow: /\n')
629
}
630
631
Req.prototype.serveWhoami = function () {
632
  var self = this
633
  self.server.sbot.whoami(function (err, feed) {
634
    if (err) return self.respondError(500, err.stack || err)
635
    self.respond(200, {username: feed.id})
636
  })
637
}
638
639
Req.prototype.serveUser1 = function () {
640
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
641
}
642
643
function decodeName(name) {
644
  var parts = String(name).replace(/\.tgz$/, '').split(':')
645
  return {
646
    name: parts[1],
647
    version: parts[2],
648
    distTag: parts[3],
649
  }
650
}
651
652
Req.prototype.getMsgMentions = function (name) {
653
  return pull(
654
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
655
    // decryption could be done here
656
    pull.map(function (msg) {
657
      var c = msg.value && msg.value.content
658
      if (!c.mentions || !Array.isArray(c.mentions)) return []
659
      return c.mentions.map(function (mention) {
660
        return {
661
          name: mention.name,
662
          size: mention.size,
663
          link: mention.link,
664
          author: msg.value.author,
665
          ts: msg.value.timestamp,
666
        }
667
      })
668
    }),
669
    pull.flatten(),
670
    pull.filter(typeof name === 'string' ? function (link) {
671
      return link.name === name
672
    } : name && name.$prefix ? function (link) {
673
      return link.name.substr(0, name.$prefix.length) === name.$prefix
674
    } : function () {
675
      throw new TypeError('unsupported name filter')
676
    })
677
  )
678
}
679
680
Req.prototype.getMentions = function (name) {
681
  var useMsgMentions = this.headMsgIds
682
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
683
  if (useServerMentions && !this.server.sbot.links2) {
684
    return this.headMsgPlus
685
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
686
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
687
  }
688
  return cat([
689
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
690
    useServerMentions ? this.server.getMentions(name) : pull.empty()
691
  ])
692
}
693
694
Req.prototype.getMentionLinks = function (blobId) {
695
  return pull(
696
    this.headMsgIds
697
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
698
      : this.server.sbot.links({
699
        dest: blobId,
700
        rel: 'mentions',
701
        values: true,
702
      }),
703
    // decryption could be done here
704
    pull.map(function (msg) {
705
      var c = msg.value && msg.value.content
706
      return c && Array.isArray(c.mentions) && c.mentions || []
707
    }),
708
    pull.flatten(),
709
    pull.filter(function (link) {
710
      return link && link.link === blobId
711
    })
712
  )
713
}
714
715
Req.prototype.servePkg = function (pathname) {
716
  var self = this
717
  var parts = pathname.split('/')
718
  var pkgName = parts.shift().replace(/%2f/i, '/')
719
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
720
  var spec = parts.shift()
721
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
722
  if (parts.length > 0) return this.respondError(404)
723
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
724
  var obj = {
725
    _id: pkgName,
726
    name: pkgName,
727
    'dist-tags': {},
728
    versions: {}
729
  }
730
  var distTags = {/* <tag>: {version, ts}*/}
731
  pull(
732
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
733
    pull.drain(function (mention) {
734
      var data = decodeName(mention.name)
735
      if (!data.version) return
736
      if (data.distTag) {
737
        var tag = distTags[data.distTag]
738
        if (!tag || mention.ts > tag.ts) {
739
          /* TODO: sort by causal order instead of only by timestamps */
740
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
741
        }
742
      }
743
      obj.versions[data.version] = {
744
        author: {
745
          url: mention.author
746
        },
747
        name: pkgName,
748
        version: data.version,
749
        dist: self.server.blobDist(mention.link)
750
      }
751
    }, function (err) {
752
      if (err) return self.respondError(500, err.stack || err)
753
      for (var tag in distTags) {
754
        obj['dist-tags'][tag] = distTags[tag].version
755
      }
756
      if (spec) resolveSpec()
757
      else if (self.fetchAll) resolveAll()
758
      else resolved()
759
    })
760
  )
761
  function resolveSpec() {
762
    var version = obj['dist-tags'][spec]
763
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
764
    obj = obj.versions[version]
765
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
766
    self.populatePackageJson(obj, function (err, pkg) {
767
      if (err) return resolved(err)
768
      obj = pkg || obj
769
      resolved()
770
    })
771
  }
772
  function resolveVersion(version, cb) {
773
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
774
      if (err) return cb(err)
775
      if (pkg) obj.versions[version] = pkg
776
      cb()
777
    })
778
  }
779
  function resolveAll() {
780
    var done = multicb()
781
    for (var version in obj.versions) {
782
      resolveVersion(version, done())
783
    }
784
    done(resolved)
785
  }
786
  function resolved(err) {
787
    if (err) return self.respondError(500, err.stack || err)
788
    self.respond(200, obj)
789
  }
790
}
791
792
Req.prototype.servePrebuild = function (name) {
793
  var self = this
794
  var getMention = self.getMentions('prebuild:' + name)
795
  var blobsByAuthor = {/* <author>: BlobId */}
796
  getMention(null, function next(err, link) {
797
    if (err === true) return done()
798
    if (err) return self.respondError(500, err.stack || err)
799
    blobsByAuthor[link.author] = link.link
800
    getMention(null, next)
801
  })
802
  function done() {
803
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
804
    var blobId
805
    for (var feed in blobsByAuthor) {
806
      var blob = blobId = blobsByAuthor[feed]
807
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
808
      feeds.push(feed)
809
    }
810
    switch (Object.keys(authorsByLink).length) {
811
      case 0:
812
        return self.respondError(404, 'Not Found')
813
      case 1:
814
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
815
        return self.res.end()
816
      default:
817
        return self.respond(300, {choices: authorsByLink})
818
    }
819
  }
820
}
821
822
var localhosts = {
823
  '::1': true,
824
  '127.0.0.1': true,
825
  '::ffff:127.0.0.1': true,
826
}
827
828
Req.prototype.publishPkg = function (pkgName) {
829
  var self = this
830
  var remoteAddress = self.req.socket.remoteAddress
831
  if (!(remoteAddress in localhosts)) {
832
    return self.respondError(403, 'You may not publish as this user.')
833
  }
834
835
  var chunks = []
836
  self.req.on('data', function (data) {
837
    chunks.push(data)
838
  })
839
  self.req.on('end', function () {
840
    var data
841
    try {
842
      data = JSON.parse(Buffer.concat(chunks))
843
    } catch(e) {
844
      return self.respondError(400, e.stack)
845
    }
846
    return self.publishPkg2(pkgName, data || {})
847
  })
848
}
849
850
Req.prototype.publishPkg2 = function (name, data) {
851
  var self = this
852
  if (data.users) console.trace('[npm-registry] users property is not supported')
853
  var attachments = data._attachments || {}
854
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
855
  var done = multicb()
856
  function addAttachmentAsBlob(filename, cb) {
857
    var data = attachments[filename].data
858
    var tarball = new Buffer(data, 'base64')
859
    var length = attachments[filename].length
860
    if (length && length !== tarball.length) return self.respondError(400,
861
      'Length mismatch for attachment \'' + filename + '\'')
862
    self.server.sbot.blobs.add(function (err, id) {
863
      if (err) return cb(err)
864
      self.blobsToPush.push(id)
865
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
866
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
867
      cb()
868
    })(pull.once(tarball))
869
  }
870
  for (var filename in attachments) {
871
    addAttachmentAsBlob(filename, done())
872
  }
873
  done(function (err) {
874
    if (err) return self.respondError(500, err.stack || err)
875
    try {
876
      self.publishPkg3(name, data, links)
877
    } catch(e) {
878
      self.respondError(500, e.stack || e)
879
    }
880
  })
881
}
882
883
Req.prototype.publishPkg3 = function (name, data, links) {
884
  var self = this
885
  var versions = data.versions || {}
886
  var linksByVersion = {/* <version>: link */}
887
888
  // associate tarball blobs with versions
889
  for (var version in versions) {
890
    var pkg = versions[version]
891
    if (!pkg) return self.respondError(400, 'Bad package object')
892
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
893
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
894
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
895
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
896
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
897
    var filename = m[1]
898
    var link = links[filename]
899
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
900
    // TODO?: try to find missing tarball mentioned in other messages
901
    if (pkg.version && pkg.version !== version)
902
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
903
    linksByVersion[version] = link
904
    link.version = version
905
    link.dependencies = pkg.dependencies || {}
906
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
907
  }
908
909
  // associate blobs with dist-tags
910
  var tags = data['dist-tags'] || {}
911
  for (var tag in tags) {
912
    var version = tags[tag]
913
    var link = linksByVersion[version]
914
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
915
    // TODO?: support setting dist-tag without version,
916
    // by looking up a tarball blob for the version
917
    link.tag = tag
918
  }
919
920
  // compute blob links to publish
921
  var mentions = []
922
  for (var filename in links) {
923
    var link = links[filename] || {}
924
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
925
    mentions.push({
926
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
927
      link: link.link,
928
      size: link.size,
929
      shasum: link.shasum,
930
      dependencies: link.dependencies,
931
      bundledDependencies: link.bundledDependencies,
932
    })
933
  }
934
  return self.publishPkgs(mentions)
935
}
936
937
Req.prototype.publishPkgs = function (mentions) {
938
  var self = this
939
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
940
    if (err) return self.respondError(500, err.stack || err)
941
    self.server.pushBlobs(self.blobsToPush, function (err) {
942
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
943
      self.respond(201)
944
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
945
    })
946
  })
947
}
948
949
Req.prototype.populatePackageJson = function (obj, cb) {
950
  var self = this
951
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
952
  var deps, bundledDeps, shasum
953
954
  // look for dependencies in links.
955
  // then fallback to getting it from the tarball blob
956
957
  pull(
958
    self.getMentionLinks(blobId),
959
    pull.drain(function (link) {
960
      if (link.dependencies) deps = link.dependencies
961
      if (link.shasum) shasum = link.shasum
962
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
963
      // how to handle multiple assignments of dependencies to a package?
964
    }, function (err) {
965
      if (err) return cb(new Error(err.stack || err))
966
      if (deps && (shasum || !self.needShasum)) {
967
        // assume that the dependencies in the links to the blob are
968
        // correct.
969
        obj.dependencies = deps
970
        obj.bundledDependencies = bundledDeps
971
        if (shasum) obj.dist.shasum = obj._shasum = shasum
972
        cb(null, obj)
973
      } else {
974
        // get dependencies from the tarball
975
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
976
          if (err) return cb(err)
977
          pkg.dist = obj.dist
978
          pkg.dist.shasum = pkg._shasum
979
          pkg.author = pkg.author || obj.author
980
          pkg.version = pkg.version || obj.version
981
          pkg.name = pkg.name || obj.name
982
          cb(null, pkg)
983
        })
984
      }
985
    })
986
  )
987
}
988
989
function getPackageJsonFromTarballBlob(sbot, id, cb) {
990
  var self = this
991
  getBlob(sbot, id, function (err, readBlob) {
992
    if (err) return cb(err)
993
    cb = once(cb)
994
    var extract = tar.extract()
995
    var pkg, shasum
996
    extract.on('entry', function (header, stream, next) {
997
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
998
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
999
          if (err) return cb(err)
1000
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1001
          catch(e) { return cb(e) }
1002
          next()
1003
        }))
1004
      } else {
1005
        stream.on('end', next)
1006
        stream.resume()
1007
      }
1008
    })
1009
    extract.on('finish', function () {
1010
      pkg._shasum = shasum
1011
      cb(null, pkg)
1012
    })
1013
    pull(
1014
      readBlob,
1015
      hash('sha1', 'hex', function (err, sum) {
1016
        if (err) return cb(err)
1017
        shasum = sum
1018
      }),
1019
      toPull(zlib.createGunzip()),
1020
      toPull(extract)
1021
    )
1022
  })
1023
}
1024