git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
77
}
78
79
exports.name = 'npm-registry'
80
exports.version = '1.0.0'
81
exports.manifest = {
82
  getAddress: 'async'
83
}
84
exports.init = function (sbot, config) {
85
  var conf = config.npm || {}
86
  var port = conf.port || 8043
87
  var host = conf.host || null
88
  var autoAuth = conf.autoAuth !== false
89
90
  var server = http.createServer(exports.respond(sbot, config))
91
  var getAddress = onceify(function (cb) {
92
    server.on('error', cb)
93
    server.listen(port, host, function () {
94
      server.removeListener('error', cb)
95
      var regHost = formatHost(host || 'localhost')
96
      var regPort = this.address().port
97
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
98
      if (autoAuth) npmLogin(regUrl, next)
99
      else next()
100
      function next(err) {
101
        cb(err, regUrl)
102
      }
103
    })
104
    sbot.on('close', function () {
105
      server.close()
106
    })
107
  })
108
109
  getAddress(function (err, addr) {
110
    if (err) return console.error('[npm-registry]', err.stack || err)
111
    console.log('[npm-registry] Listening on ' + addr)
112
  })
113
114
  return {
115
    getAddress: getAddress
116
  }
117
}
118
119
exports.respond = function (sbot, config) {
120
  var reg = new SsbNpmRegistryServer(sbot, config)
121
  return function (req, res) {
122
    new Req(reg, req, res).serve()
123
  }
124
}
125
126
function publishMsg(sbot, value, cb) {
127
  var gotExpectedPrevious = false
128
  sbot.publish(value, function next(err, msg) {
129
    if (err && /^expected previous:/.test(err.message)) {
130
      // retry once on this error
131
      if (gotExpectedPrevious) return cb(err)
132
      gotExpectedPrevious = true
133
      return sbot.publish(value, next)
134
    }
135
    cb(err, msg)
136
  })
137
}
138
139
function getDependencyBranches(sbot, link, cb) {
140
  var id = link.link
141
142
  // get ids of heads of tree of dependencyBranch message links to include all
143
  // the dependencies of the given tarball id.
144
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
145
  var msgs = {}
146
  var branched = {}
147
  var blobs = {}
148
149
  function addPkgById(id, cb) {
150
    if (blobs[id]) return cb()
151
    blobs[id] = true
152
    getPackageJsonCached(id, function (err, pkg) {
153
      if (err) return cb(err)
154
      var done = multicb()
155
      for (var name in pkg.dependencies || {}) {
156
        addPkgBySpec(name, pkg.dependencies[name], done())
157
      }
158
      done(cb)
159
    })
160
  }
161
162
  function addPkgBySpec(name, spec, cb) {
163
    var versions = {}
164
    var distTags = {}
165
    pull(
166
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
167
      pull.filter(mentionMatches(null, name, spec)),
168
      pull.map(function (mention) {
169
        // query to get the messages since links2 does not include the msg id
170
        return sbot.links({
171
          source: mention.author,
172
          dest: mention.link,
173
          rel: 'mentions',
174
          values: true,
175
        })
176
      }),
177
      pull.flatten(),
178
      pull.drain(function (msg) {
179
        var c = msg && msg.value && msg.value.content
180
        if (!c || !Array.isArray(c.mentions)) return
181
        c.mentions.forEach(function (link) {
182
          var data = link && link.name && decodeName(link.name)
183
          if (!data || data.name !== name) return
184
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
185
          if (data.distTag) distTags[data.distTag] = data.version
186
        })
187
      }, function (err) {
188
        if (err) return cb(err)
189
        var version = distTags[spec]
190
          || semver.maxSatisfying(Object.keys(versions), spec)
191
        var item = versions[version]
192
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
193
        addDepBranchMsg(item.msg)
194
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
195
        addPkgById(item.mention.link, cb)
196
      })
197
    )
198
  }
199
200
  function addDepBranchMsg(msg) {
201
    var c = msg && msg.value && msg.value.content
202
    if (!c) return
203
    if (msgs[msg.key]) return
204
    msgs[msg.key] = msg.value
205
    if (Array.isArray(c.dependencyBranch)) {
206
      for (var k = 0; k < c.dependencyBranch.length; k++) {
207
        branched[c.dependencyBranch[k]] = true
208
      }
209
    }
210
    if (Array.isArray(c.versionBranch)) {
211
      for (var k = 0; k < c.versionBranch.length; k++) {
212
        branched[c.versionBranch[k]] = true
213
      }
214
    }
215
  }
216
217
  function addPkgsByName(name, cb) {
218
    var data = decodeName(name)
219
    pull(
220
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
221
      pull.map(function (mention) {
222
        return packageLinks(sbot, mention.author, mention.link, data.name)
223
      }),
224
      pull.flatten(),
225
      pull.drain(addDepBranchMsg, cb)
226
    )
227
  }
228
229
  var done = multicb()
230
  addPkgsByName(link.name, done())
231
  addPkgById(id, done())
232
  done(function (err) {
233
    if (err) return cb(err)
234
    var ids = []
235
    for (var key in msgs) {
236
      if (!branched[key]) ids.push(key)
237
    }
238
    cb(null, ids)
239
  })
240
}
241
242
function packageLinks(sbot, feed, id, name, spec) {
243
  var matches = mentionMatches(id, name, spec)
244
  return pull(
245
    sbot.links({
246
      source: feed,
247
      dest: id,
248
      rel: 'mentions',
249
      values: true,
250
    }),
251
    pull.filter(function (msg) {
252
      var c = msg && msg.value && msg.value.content
253
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
254
    })
255
  )
256
}
257
258
// For each dependency that is not a bundledDependency, get message ids for
259
// that dependency name + version.
260
function publishSingleMention(sbot, mention, cb) {
261
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
262
  // Calculate dependencyBranch message ids.
263
  var value = {
264
    type: 'npm-packages',
265
    mentions: [mention]
266
  }
267
  var done = multicb({pluck: 1, spread: true})
268
  getDependencyBranches(sbot, mention, function (err, branches) {
269
    if (err) return cb(err)
270
    value.dependencyBranch = branches || undefined
271
    publishMsg(sbot, value, cb)
272
  })
273
}
274
275
function publishMentions(sbot, mentions, cb) {
276
  // console.error("publishing %s mentions", mentions.length)
277
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
278
  // if it is just one mention, fetch and add useful metadata
279
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
280
  publishMsg(sbot, {
281
    type: 'npm-packages',
282
    mentions: mentions,
283
  }, cb)
284
}
285
286
exports.publishPkgMentions = function (sbot, mentions, cb) {
287
  // try to fit the mentions into as few messages as possible,
288
  // while fitting under the message size limit.
289
  var msgs = []
290
  ;(function next(i, chunks) {
291
    if (i >= mentions.length) return cb(null, msgs)
292
    var chunkLen = Math.ceil(mentions.length / chunks)
293
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
294
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
295
      if (err && msgs.length) return onPartialPublish(err)
296
      if (err) return cb(err)
297
      msgs.push(msg)
298
      next(i + chunkLen, chunks)
299
    })
300
  })(0, 1)
301
  function onPartialPublish(err) {
302
    var remaining = mentions.length - i
303
    return cb(new Error('Published messages ' +
304
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
305
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
306
  }
307
}
308
309
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
310
  cb = once(cb)
311
  var waiting = 0
312
  var expandedMentions = mentions.map(function (link) {
313
    var id = link && link.link
314
    if (!id) return link
315
    waiting++
316
    var newLink = {}
317
    for (var k in link) newLink[k] = link[k]
318
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
319
      if (err) return cb(err)
320
      for (var k in props) newLink[k] = pkg[k]
321
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
322
      if (!--waiting) next()
323
    })
324
    return newLink
325
  })
326
  if (!waiting) next()
327
  function next() {
328
    cb(null, expandedMentions)
329
  }
330
}
331
332
function SsbNpmRegistryServer(sbot, config) {
333
  this.sbot = sbot
334
  this.config = config
335
  this.npmConfig = config.npm || {}
336
  this.host = this.npmConfig.host || 'localhost'
337
  this.fetchAll = this.npmConfig.fetchAll
338
  this.needShasum = this.npmConfig.needShasum
339
  var wsPort = config.ws && Number(config.ws.port) || '8989'
340
  this.wsLink = this.npmConfig.wsLink
341
    || ('http://' + (config.host || 'localhost') + ':' + wsPort)
342
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
343
}
344
345
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
346
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
347
348
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
349
  var self = this
350
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
351
  ;(function next(i) {
352
    if (i >= ids.length) return cb()
353
    self.sbot.blobs.push(ids[i], function (err) {
354
      if (err) return cb(err)
355
      next(i+1)
356
    })
357
  })(0)
358
}
359
360
function getBlob(sbot, id, cb) {
361
  var blobs = sbot.blobs
362
  blobs.size(id, function (err, size) {
363
    if (typeof size === 'number') cb(null, blobs.get(id))
364
    else blobs.want(id, function (err, got) {
365
      if (err) cb(err)
366
      else if (!got) cb('missing blob ' + id)
367
      else cb(null, blobs.get(id))
368
    })
369
  })
370
}
371
372
SsbNpmRegistryServer.prototype.blobDist = function (id) {
373
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
374
  if (!m) throw new Error('bad blob id: ' + id)
375
  return {
376
    integrity: m[2] + '-' + m[1],
377
    tarball: this.wsLink + '/blobs/get/' + id
378
  }
379
}
380
381
function getMentions(links2, name) {
382
  return links2.read({
383
    query: [
384
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
385
      {$filter: {dest: {$prefix: '&'}}},
386
      {$map: {
387
        name: ['rel', 1],
388
        size: ['rel', 2],
389
        link: 'dest',
390
        author: 'source',
391
        ts: 'ts'
392
      }}
393
    ]
394
  })
395
}
396
397
function mentionMatches(id, name, spec) {
398
  return function (mention) {
399
    var data = mention
400
      && (!id || id === mention.link)
401
      && mention.name
402
      && decodeName(mention.name)
403
    return data
404
      && data.name === name
405
      && (spec ? semver.satisfies(data.version, spec) : true)
406
  }
407
}
408
409
SsbNpmRegistryServer.prototype.getMentions = function (name) {
410
  if (!this.sbot.links2) return pull.empty()
411
  return getMentions(this.sbot.links2, name)
412
}
413
414
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
415
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
416
  else this.sbot.get(id, function (err, value) {
417
    if (err) return cb(err)
418
    cb(null, {key: id, value: value})
419
  })
420
}
421
422
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
423
  var self = this
424
  var stack = heads.slice()
425
  var seen = {}
426
  return function (abort, cb) {
427
    if (abort) return cb(abort)
428
    for (var id; stack.length && seen[id = stack.pop()];);
429
    if (!id) return cb(true)
430
    seen[id] = true
431
    // TODO: use DFS
432
    self.getMsg(id, function (err, msg) {
433
      if (err) return cb(new Error(err.stack || err))
434
      var c = msg && msg.value && msg.value.content
435
      var links = c && c[prop]
436
      if (Array.isArray(links)) {
437
        stack.push.apply(stack, links)
438
      } else if (links) {
439
        stack.push(links)
440
      }
441
      cb(null, msg)
442
    })
443
  }
444
}
445
446
function Req(server, req, res) {
447
  this.server = server
448
  this.req = req
449
  this.res = res
450
  this.blobsToPush = []
451
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
452
  var ua = this.req.headers['user-agent']
453
  var m = /\bnpm\/([0-9]*)/.exec(ua)
454
  var npmVersion = m && m[1]
455
  this.needShasum = server.needShasum != null ? server.needShasum :
456
    (npmVersion && npmVersion < 5)
457
  this.wsLink = this.server.wsLink
458
}
459
460
Req.prototype.serve = function () {
461
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
462
  this.res.setTimeout(0)
463
  var pathname = this.req.url.replace(/\?.*/, '')
464
  var m
465
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
466
    try {
467
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
468
      this.headMsgPlus = !!m[1]
469
      // ^ means also include packages published after the head message id
470
    } catch(e) {
471
      return this.respondError(400, e.stack || e)
472
    }
473
    pathname = m[3]
474
  }
475
  if (pathname === '/') return this.serveHome()
476
  if (pathname === '/robots.txt') return this.serveRobots()
477
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
478
  if (pathname === '/-/whoami') return this.serveWhoami()
479
  if (pathname === '/-/ping') return this.respond(200, true)
480
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
481
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
482
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
483
  return this.respond(404, {error: 'Not found'})
484
}
485
486
Req.prototype.respond = function (status, message) {
487
  this.res.writeHead(status, {'content-type': 'application/json'})
488
  this.res.end(message && JSON.stringify(message, 0, 2))
489
}
490
491
Req.prototype.respondError = function (status, message) {
492
  this.respond(status, {error: message})
493
}
494
495
Req.prototype.respondErrorStr = function (status, err) {
496
  this.res.writeHead(status, {'content-type': 'text/plain'})
497
  this.res.end(err.stack || err)
498
}
499
500
Req.prototype.serveHome = function () {
501
  var self = this
502
  self.res.writeHead(200, {'content-type': 'text/html'})
503
  var port = 8044
504
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
505
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
506
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
507
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
508
    '</body></html>')
509
}
510
511
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
512
  var self = this
513
  pull(
514
    self.server.sbot.links({
515
      source: feedId,
516
      dest: blobId,
517
      rel: 'mentions',
518
      values: true,
519
    }),
520
    pull.filter(function (msg) {
521
      var c = msg && msg.value && msg.value.content
522
      return c.type === 'npm-packages'
523
    }),
524
    pull.collect(function (err, msgs) {
525
      if (err) return cb(err)
526
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
527
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
528
      // TODO: make a smarter decision about which message id to use
529
      cb(null, msgs.pop().key)
530
    })
531
  )
532
}
533
534
Req.prototype.resolvePkg = function (pkgSpec, cb) {
535
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
536
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
537
  var self = this
538
  var pkgName = m[1]
539
  var spec = m[2] || '*'
540
  var versions = {}
541
  var distTags = {}
542
  pull(
543
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
544
    pull.drain(function (mention) {
545
      var data = decodeName(mention.name)
546
      if (!data.version) return
547
      if (data.distTag) {
548
        distTags[data.distTag] = data.version
549
      }
550
      versions[data.version] = {
551
        author: mention.author,
552
        name: pkgName,
553
        version: data.version,
554
        blobId: mention.link
555
      }
556
    }, function (err) {
557
      if (err) return cb(err)
558
      var version = distTags[spec]
559
        || semver.maxSatisfying(Object.keys(versions), spec)
560
      var item = versions[version]
561
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
562
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
563
        if (err) return cb(err)
564
        item.msgId = id
565
        cb(null, item)
566
      })
567
    })
568
  )
569
}
570
571
Req.prototype.resolvePkgs = function (specs, cb) {
572
  var done = multicb({pluck: 1})
573
  var self = this
574
  specs.forEach(function (spec) {
575
    self.resolvePkg(spec, done())
576
  })
577
  done(cb)
578
}
579
580
Req.prototype.serveBootstrap = function () {
581
  var self = this
582
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
583
  var ssbNpmRegistryName = require('./package.json').name
584
  var ssbNpmRegistryVersion = require('./package.json').version
585
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
586
  var done = multicb({pluck: 1, spread: true})
587
  self.resolvePkg(ssbNpmRegistrySpec, done())
588
  self.resolvePkgs(pkgs, done())
589
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
590
    if (err) return self.respondErrorStr(500, err.stack || err)
591
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
592
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
593
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
594
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
595
    var globalPkgs = pkgsInfo.map(function (info) {
596
      return info.name + '@' + info.version
597
    }).join(' ')
598
    var npmCmd = 'install -g ' + globalPkgs
599
    var pkgTmpText = '/tmp/' + ssbNpmRegistryName + '.tar.gz'
600
    var tarballLink = self.wsLink + '/blobs/get/' + ssbNpmRegistryBlobId
601
602
    var script =
603
      'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
604
      'echo \'' + ssbNpmRegistryBlobHex + '  ' + pkgTmpText + '\' | sha256sum -c &&\n' +
605
      'mkdir -p ~/.ssb/node_modules && cd ~/.ssb/node_modules &&\n' +
606
      'tar xzf ' + pkgTmpText + ' &&\n' +
607
      'mv package ' + ssbNpmRegistryName + ' &&\n' +
608
      ssbNpmRegistryName + '/bootstrap/bin.js --ws-url ' + self.wsLink + ' \\\n' +
609
        pkgMsgs.map(function (id) {
610
          return '  --branch ' + id + ' \\\n'
611
        }).join('') +
612
        '  -- ' + npmCmd + ' &&\n' +
613
      'sbot server'
614
615
    self.res.writeHead(200, {'Content-type': 'text/plain'})
616
    self.res.end(script)
617
  })
618
}
619
620
Req.prototype.serveRobots = function () {
621
  this.res.writeHead(200, {'Content-type': 'text/plain'})
622
  this.res.end('User-agent: *\nDisallow: /\n')
623
}
624
625
Req.prototype.serveWhoami = function () {
626
  var self = this
627
  self.server.sbot.whoami(function (err, feed) {
628
    if (err) return self.respondError(500, err.stack || err)
629
    self.respond(200, {username: feed.id})
630
  })
631
}
632
633
Req.prototype.serveUser1 = function () {
634
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
635
}
636
637
function decodeName(name) {
638
  var parts = String(name).replace(/\.tgz$/, '').split(':')
639
  return {
640
    name: parts[1],
641
    version: parts[2],
642
    distTag: parts[3],
643
  }
644
}
645
646
Req.prototype.getMsgMentions = function (name) {
647
  return pull(
648
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
649
    // decryption could be done here
650
    pull.map(function (msg) {
651
      var c = msg.value && msg.value.content
652
      if (!c.mentions || !Array.isArray(c.mentions)) return []
653
      return c.mentions.map(function (mention) {
654
        return {
655
          name: mention.name,
656
          size: mention.size,
657
          link: mention.link,
658
          author: msg.value.author,
659
          ts: msg.value.timestamp,
660
        }
661
      })
662
    }),
663
    pull.flatten(),
664
    pull.filter(typeof name === 'string' ? function (link) {
665
      return link.name === name
666
    } : name && name.$prefix ? function (link) {
667
      return link.name.substr(0, name.$prefix.length) === name.$prefix
668
    } : function () {
669
      throw new TypeError('unsupported name filter')
670
    })
671
  )
672
}
673
674
Req.prototype.getMentions = function (name) {
675
  var useMsgMentions = this.headMsgIds
676
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
677
  if (useServerMentions && !this.server.sbot.links2) {
678
    return this.headMsgPlus
679
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
680
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
681
  }
682
  return cat([
683
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
684
    useServerMentions ? this.server.getMentions(name) : pull.empty()
685
  ])
686
}
687
688
Req.prototype.getMentionLinks = function (blobId) {
689
  return pull(
690
    this.headMsgIds
691
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
692
      : this.server.sbot.links({
693
        dest: blobId,
694
        rel: 'mentions',
695
        values: true,
696
      }),
697
    // decryption could be done here
698
    pull.map(function (msg) {
699
      var c = msg.value && msg.value.content
700
      return c && Array.isArray(c.mentions) && c.mentions || []
701
    }),
702
    pull.flatten(),
703
    pull.filter(function (link) {
704
      return link && link.link === blobId
705
    })
706
  )
707
}
708
709
Req.prototype.servePkg = function (pathname) {
710
  var self = this
711
  var parts = pathname.split('/')
712
  var pkgName = parts.shift().replace(/%2f/i, '/')
713
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
714
  var spec = parts.shift()
715
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
716
  if (parts.length > 0) return this.respondError(404)
717
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
718
  var obj = {
719
    _id: pkgName,
720
    name: pkgName,
721
    'dist-tags': {},
722
    versions: {}
723
  }
724
  var distTags = {/* <tag>: {version, ts}*/}
725
  pull(
726
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
727
    pull.drain(function (mention) {
728
      var data = decodeName(mention.name)
729
      if (!data.version) return
730
      if (data.distTag) {
731
        var tag = distTags[data.distTag]
732
        if (!tag || mention.ts > tag.ts) {
733
          /* TODO: sort by causal order instead of only by timestamps */
734
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
735
        }
736
      }
737
      obj.versions[data.version] = {
738
        author: {
739
          url: mention.author
740
        },
741
        name: pkgName,
742
        version: data.version,
743
        dist: self.server.blobDist(mention.link)
744
      }
745
    }, function (err) {
746
      if (err) return self.respondError(500, err.stack || err)
747
      for (var tag in distTags) {
748
        obj['dist-tags'][tag] = distTags[tag].version
749
      }
750
      if (spec) resolveSpec()
751
      else if (self.fetchAll) resolveAll()
752
      else resolved()
753
    })
754
  )
755
  function resolveSpec() {
756
    var version = obj['dist-tags'][spec]
757
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
758
    obj = obj.versions[version]
759
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
760
    self.populatePackageJson(obj, function (err, pkg) {
761
      if (err) return resolved(err)
762
      obj = pkg || obj
763
      resolved()
764
    })
765
  }
766
  function resolveVersion(version, cb) {
767
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
768
      if (err) return cb(err)
769
      if (pkg) obj.versions[version] = pkg
770
      cb()
771
    })
772
  }
773
  function resolveAll() {
774
    var done = multicb()
775
    for (var version in obj.versions) {
776
      resolveVersion(version, done())
777
    }
778
    done(resolved)
779
  }
780
  function resolved(err) {
781
    if (err) return self.respondError(500, err.stack || err)
782
    self.respond(200, obj)
783
  }
784
}
785
786
Req.prototype.servePrebuild = function (name) {
787
  var self = this
788
  var getMention = self.getMentions('prebuild:' + name)
789
  var blobsByAuthor = {/* <author>: BlobId */}
790
  pull(getMention, pull.drain(function (link) {
791
    blobsByAuthor[link.author] = link.link
792
  }, function (err) {
793
    if (err) return self.respondError(500, err.stack || err)
794
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
795
    var blobId
796
    for (var feed in blobsByAuthor) {
797
      var blob = blobId = blobsByAuthor[feed]
798
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
799
      feeds.push(feed)
800
    }
801
    switch (Object.keys(authorsByLink).length) {
802
      case 0:
803
        return self.respondError(404, 'Not Found')
804
      case 1:
805
        self.res.writeHead(303, {Location: self.wsLink + '/blobs/get/' + blobId})
806
        return self.res.end()
807
      default:
808
        return self.respond(300, {choices: authorsByLink})
809
    }
810
  }))
811
}
812
813
var localhosts = {
814
  '::1': true,
815
  '127.0.0.1': true,
816
  '::ffff:127.0.0.1': true,
817
}
818
819
Req.prototype.publishPkg = function (pkgName) {
820
  var self = this
821
  var remoteAddress = self.req.socket.remoteAddress
822
  if (!(remoteAddress in localhosts)) {
823
    return self.respondError(403, 'You may not publish as this user.')
824
  }
825
826
  var chunks = []
827
  self.req.on('data', function (data) {
828
    chunks.push(data)
829
  })
830
  self.req.on('end', function () {
831
    var data
832
    try {
833
      data = JSON.parse(Buffer.concat(chunks))
834
    } catch(e) {
835
      return self.respondError(400, e.stack)
836
    }
837
    return self.publishPkg2(pkgName, data || {})
838
  })
839
}
840
841
Req.prototype.publishPkg2 = function (name, data) {
842
  var self = this
843
  if (data.users) console.trace('[npm-registry] users property is not supported')
844
  var attachments = data._attachments || {}
845
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
846
  var done = multicb()
847
  function addAttachmentAsBlob(filename, cb) {
848
    var data = attachments[filename].data
849
    var tarball = new Buffer(data, 'base64')
850
    var length = attachments[filename].length
851
    if (length && length !== tarball.length) return self.respondError(400,
852
      'Length mismatch for attachment \'' + filename + '\'')
853
    self.server.sbot.blobs.add(function (err, id) {
854
      if (err) return cb(err)
855
      self.blobsToPush.push(id)
856
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
857
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
858
      cb()
859
    })(pull.once(tarball))
860
  }
861
  for (var filename in attachments) {
862
    addAttachmentAsBlob(filename, done())
863
  }
864
  done(function (err) {
865
    if (err) return self.respondError(500, err.stack || err)
866
    try {
867
      self.publishPkg3(name, data, links)
868
    } catch(e) {
869
      self.respondError(500, e.stack || e)
870
    }
871
  })
872
}
873
874
Req.prototype.publishPkg3 = function (name, data, links) {
875
  var self = this
876
  var versions = data.versions || {}
877
  var linksByVersion = {/* <version>: link */}
878
879
  // associate tarball blobs with versions
880
  for (var version in versions) {
881
    var pkg = versions[version]
882
    if (!pkg) return self.respondError(400, 'Bad package object')
883
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
884
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
885
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
886
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
887
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
888
    var filename = m[1]
889
    var link = links[filename]
890
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
891
    // TODO?: try to find missing tarball mentioned in other messages
892
    if (pkg.version && pkg.version !== version)
893
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
894
    linksByVersion[version] = link
895
    link.version = version
896
    link.dependencies = pkg.dependencies || {}
897
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
898
  }
899
900
  // associate blobs with dist-tags
901
  var tags = data['dist-tags'] || {}
902
  for (var tag in tags) {
903
    var version = tags[tag]
904
    var link = linksByVersion[version]
905
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
906
    // TODO?: support setting dist-tag without version,
907
    // by looking up a tarball blob for the version
908
    link.tag = tag
909
  }
910
911
  // compute blob links to publish
912
  var mentions = []
913
  for (var filename in links) {
914
    var link = links[filename] || {}
915
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
916
    mentions.push({
917
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
918
      link: link.link,
919
      size: link.size,
920
      shasum: link.shasum,
921
      dependencies: link.dependencies,
922
      bundledDependencies: link.bundledDependencies,
923
    })
924
  }
925
  return self.publishPkgs(mentions)
926
}
927
928
Req.prototype.publishPkgs = function (mentions) {
929
  var self = this
930
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
931
    if (err) return self.respondError(500, err.stack || err)
932
    self.server.pushBlobs(self.blobsToPush, function (err) {
933
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
934
      self.respond(201)
935
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
936
    })
937
  })
938
}
939
940
Req.prototype.populatePackageJson = function (obj, cb) {
941
  var self = this
942
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
943
  var deps, bundledDeps, shasum
944
945
  // look for dependencies in links.
946
  // then fallback to getting it from the tarball blob
947
948
  pull(
949
    self.getMentionLinks(blobId),
950
    pull.drain(function (link) {
951
      if (link.dependencies) deps = link.dependencies
952
      if (link.shasum) shasum = link.shasum
953
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
954
      // how to handle multiple assignments of dependencies to a package?
955
    }, function (err) {
956
      if (err) return cb(new Error(err.stack || err))
957
      if (deps && (shasum || !self.needShasum)) {
958
        // assume that the dependencies in the links to the blob are
959
        // correct.
960
        obj.dependencies = deps
961
        obj.bundledDependencies = bundledDeps
962
        if (shasum) obj.dist.shasum = obj._shasum = shasum
963
        cb(null, obj)
964
      } else {
965
        // get dependencies from the tarball
966
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
967
          if (err) return cb(err)
968
          pkg.dist = obj.dist
969
          pkg.dist.shasum = pkg._shasum
970
          pkg.author = pkg.author || obj.author
971
          pkg.version = pkg.version || obj.version
972
          pkg.name = pkg.name || obj.name
973
          cb(null, pkg)
974
        })
975
      }
976
    })
977
  )
978
}
979
980
function getPackageJsonFromTarballBlob(sbot, id, cb) {
981
  var self = this
982
  getBlob(sbot, id, function (err, readBlob) {
983
    if (err) return cb(err)
984
    cb = once(cb)
985
    var extract = tar.extract()
986
    var pkg, shasum
987
    extract.on('entry', function (header, stream, next) {
988
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
989
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
990
          if (err) return cb(err)
991
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
992
          catch(e) { return cb(e) }
993
          next()
994
        }))
995
      } else {
996
        stream.on('end', next)
997
        stream.resume()
998
      }
999
    })
1000
    extract.on('finish', function () {
1001
      pkg._shasum = shasum
1002
      cb(null, pkg)
1003
    })
1004
    pull(
1005
      readBlob,
1006
      hash('sha1', 'hex', function (err, sum) {
1007
        if (err) return cb(err)
1008
        shasum = sum
1009
      }),
1010
      toPull(zlib.createGunzip()),
1011
      toPull(extract)
1012
    )
1013
  })
1014
}
1015