git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || 'localhost'
90
  var autoAuth = conf.autoAuth !== false
91
  var listenUrl
92
93
  var server = http.createServer(exports.respond(sbot, config))
94
  var getAddress = onceify(function (cb) {
95
    server.on('error', cb)
96
    server.listen(port, host, function () {
97
      server.removeListener('error', cb)
98
      var addr = this.address()
99
      var listenHost = addr.address
100
      var regHost = listenHost === '::' ? '::1' :
101
                    listenHost === '0.0.0.0' ? '127.0.0.1' :
102
                    listenHost
103
      var regUrl = 'http://' + formatHost(regHost) + ':' + addr.port
104
      listenUrl = 'http://' + formatHost(listenHost) + ':' + addr.port
105
      if (autoAuth) npmLogin(regUrl, next)
106
      else next()
107
      function next(err) {
108
        cb(err, regUrl)
109
      }
110
    })
111
    sbot.on('close', function () {
112
      server.close()
113
    })
114
  })
115
116
  /* getAddress called by local RPC is used to discover the local
117
   * ssb-npm-registry address that can be given to local npm clients. However,
118
   * when running the server we output the address the server is listening on,
119
   * to avoid misleading situations like saying listening on localhost but
120
   * actually listening on a wildcard address. */
121
  getAddress(function (err) {
122
    if (err) return console.error('[npm-registry]', err.stack || err)
123
    console.log('[npm-registry] Listening on ' + listenUrl)
124
  })
125
126
  return {
127
    getAddress: getAddress
128
  }
129
}
130
131
exports.respond = function (sbot, config) {
132
  var reg = new SsbNpmRegistryServer(sbot, config)
133
  return function (req, res) {
134
    new Req(reg, req, res).serve()
135
  }
136
}
137
138
function publishMsg(sbot, value, cb) {
139
  var gotExpectedPrevious = false
140
  sbot.publish(value, function next(err, msg) {
141
    if (err && /^expected previous:/.test(err.message)) {
142
      // retry once on this error
143
      if (gotExpectedPrevious) return cb(err)
144
      gotExpectedPrevious = true
145
      return sbot.publish(value, next)
146
    }
147
    cb(err, msg)
148
  })
149
}
150
151
function getDependencyBranches(sbot, id, cb) {
152
  // get ids of heads of tree of dependencyBranch message links to include all
153
  // the dependencies of the given tarball id.
154
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
155
  var msgs = {}
156
  var branched = {}
157
  var blobs = {}
158
159
  function addPkgById(id, cb) {
160
    if (blobs[id]) return cb()
161
    blobs[id] = true
162
    getPackageJsonCached(id, function (err, pkg) {
163
      if (err) return cb(err)
164
      var done = multicb()
165
      for (var name in pkg.dependencies || {}) {
166
        addPkgBySpec(name, pkg.dependencies[name], done())
167
      }
168
      for (var name in pkg.optionalDependencies || {}) {
169
        addPkgBySpec(name, pkg.optionalDependencies[name], done())
170
      }
171
      done(cb)
172
    })
173
  }
174
175
  function addPkgBySpec(name, spec, cb) {
176
    var versions = {}
177
    var distTags = {}
178
    pull(
179
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
180
      pull.filter(mentionMatches(null, name, spec)),
181
      pull.map(function (mention) {
182
        // query to get the messages since links2 does not include the msg id
183
        return sbot.links({
184
          source: mention.author,
185
          dest: mention.link,
186
          rel: 'mentions',
187
          values: true,
188
        })
189
      }),
190
      pull.flatten(),
191
      pull.drain(function (msg) {
192
        var c = msg && msg.value && msg.value.content
193
        if (!c || !Array.isArray(c.mentions)) return
194
        c.mentions.forEach(function (link) {
195
          var data = link && link.name && decodeName(link.name)
196
          if (!data || data.name !== name) return
197
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
198
          if (data.distTag) distTags[data.distTag] = data.version
199
        })
200
      }, function (err) {
201
        if (err) return cb(err)
202
        var version = distTags[spec]
203
          || semver.maxSatisfying(Object.keys(versions), spec)
204
        var item = versions[version]
205
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
206
        msgs[item.msg.key] = item.msg.value
207
        var c = item.msg.value.content
208
        if (Array.isArray(c.dependencyBranch)) {
209
          for (var k = 0; k < c.dependencyBranch.length; k++) {
210
            branched[c.dependencyBranch[k]] = true
211
          }
212
        }
213
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
214
        addPkgById(item.mention.link, cb)
215
      })
216
    )
217
  }
218
219
  addPkgById(id, function (err) {
220
    if (err) return cb(err)
221
    var ids = []
222
    for (var key in msgs) {
223
      if (!branched[key]) ids.push(key)
224
    }
225
    cb(null, ids)
226
  })
227
}
228
229
function packageLinks(sbot, feed, id, name, spec) {
230
  var matches = mentionMatches(id, name, spec)
231
  return pull(
232
    sbot.links({
233
      source: feed,
234
      dest: id,
235
      rel: 'mentions',
236
      values: true,
237
    }),
238
    pull.filter(function (msg) {
239
      var c = msg && msg.value && msg.value.content
240
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
241
    })
242
  )
243
}
244
245
function getVersionBranches(sbot, link, cb) {
246
  var data = decodeName(link.name)
247
  var msgs = {}, branched = {}
248
  pull(
249
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
250
    pull.map(function (mention) {
251
      return packageLinks(sbot, mention.author, mention.link, data.name)
252
    }),
253
    pull.flatten(),
254
    pull.drain(function (msg) {
255
      var c = msg && msg.value && msg.value.content
256
      if (!c) return
257
      msgs[msg.key] = msg.value
258
      if (Array.isArray(c.versionBranch)) {
259
        for (var k = 0; k < c.versionBranch.length; k++) {
260
          branched[c.versionBranch[k]] = true
261
        }
262
      }
263
    }, function (err) {
264
      if (err) return cb(err)
265
      var ids = []
266
      for (var key in msgs) {
267
        if (!branched[key]) ids.push(key)
268
      }
269
      cb(null, ids)
270
    })
271
  )
272
}
273
274
// For each dependency that is not a bundledDependency, get message ids for
275
// that dependency name + version.
276
function publishSingleMention(sbot, mention, cb) {
277
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
278
  // Calculate dependencyBranch and versionBranch message ids.
279
  var value = {
280
    type: 'npm-packages',
281
    mentions: [mention]
282
  }
283
  var done = multicb({pluck: 1, spread: true})
284
  getDependencyBranches(sbot, mention.link, done())
285
  getVersionBranches(sbot, mention, done())
286
  done(function (err, dependencyBranches, versionBranches) {
287
    if (err) return cb(err)
288
    value.dependencyBranch = dependencyBranches || undefined
289
    value.versionBranch = versionBranches || undefined
290
    publishMsg(sbot, value, cb)
291
  })
292
}
293
294
function publishMentions(sbot, mentions, cb) {
295
  // console.error("publishing %s mentions", mentions.length)
296
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
297
  // if it is just one mention, fetch and add useful metadata
298
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
299
  publishMsg(sbot, {
300
    type: 'npm-packages',
301
    mentions: mentions,
302
  }, cb)
303
}
304
305
exports.publishPkgMentions = function (sbot, mentions, cb) {
306
  // try to fit the mentions into as few messages as possible,
307
  // while fitting under the message size limit.
308
  var msgs = []
309
  ;(function next(i, chunks) {
310
    if (i >= mentions.length) return cb(null, msgs)
311
    var chunkLen = Math.ceil(mentions.length / chunks)
312
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
313
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
314
      if (err && msgs.length) return onPartialPublish(err)
315
      if (err) return cb(err)
316
      msgs.push(msg)
317
      next(i + chunkLen, chunks)
318
    })
319
  })(0, 1)
320
  function onPartialPublish(err) {
321
    var remaining = mentions.length - i
322
    return cb(new Error('Published messages ' +
323
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
324
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
325
  }
326
}
327
328
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
329
  cb = once(cb)
330
  var waiting = 0
331
  var expandedMentions = mentions.map(function (link) {
332
    var id = link && link.link
333
    if (!id) return link
334
    waiting++
335
    var newLink = {}
336
    for (var k in link) newLink[k] = link[k]
337
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
338
      if (err) return cb(err)
339
      for (var k in props) newLink[k] = pkg[k]
340
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
341
      if (!--waiting) next()
342
    })
343
    return newLink
344
  })
345
  if (!waiting) next()
346
  function next() {
347
    cb(null, expandedMentions)
348
  }
349
}
350
351
function SsbNpmRegistryServer(sbot, config) {
352
  this.sbot = sbot
353
  this.config = config
354
  this.npmConfig = config.npm || {}
355
  this.host = this.npmConfig.host || 'localhost'
356
  this.fetchAll = this.npmConfig.fetchAll
357
  this.needShasum = this.npmConfig.needShasum
358
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
359
  this.getFeedName = memo({cache: lru(100)}, this.getFeedName)
360
  if (sbot.ws) {
361
    var wsPort = config.ws && Number(config.ws.port) || '8989'
362
    this.wsUrl = 'http://localhost:' + wsPort
363
  }
364
}
365
366
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
367
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
368
369
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
370
  var self = this
371
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
372
  ;(function next(i) {
373
    if (i >= ids.length) return cb()
374
    self.sbot.blobs.push(ids[i], function (err) {
375
      if (err) return cb(err)
376
      next(i+1)
377
    })
378
  })(0)
379
}
380
381
function getBlob(sbot, id, cb) {
382
  var blobs = sbot.blobs
383
  blobs.size(id, function (err, size) {
384
    if (err && err.code !== 'ENOENT') return cb(err)
385
    if (typeof size === 'number') cb(null, blobs.get(id), size)
386
    else blobs.want(id, function (err, got) {
387
      if (err) cb(err)
388
      else if (!got) cb('missing blob ' + id)
389
      else blobs.size(id, function (err, size) {
390
        if (err) return cb(err)
391
        cb(null, blobs.get(id), size)
392
      })
393
    })
394
  })
395
}
396
397
function getMentions(links2, name) {
398
  return links2.read({
399
    query: [
400
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
401
      {$filter: {dest: {$prefix: '&'}}},
402
      {$map: {
403
        name: ['rel', 1],
404
        size: ['rel', 2],
405
        link: 'dest',
406
        author: 'source',
407
        ts: 'ts'
408
      }}
409
    ]
410
  })
411
}
412
413
function mentionMatches(id, name, spec) {
414
  return function (mention) {
415
    var data = mention
416
      && (!id || id === mention.link)
417
      && mention.name
418
      && decodeName(mention.name)
419
    return data
420
      && data.name === name
421
      && (spec ? semver.satisfies(data.version, spec) : true)
422
  }
423
}
424
425
SsbNpmRegistryServer.prototype.getMentions = function (name) {
426
  if (!this.sbot.links2) return pull.empty()
427
  return getMentions(this.sbot.links2, name)
428
}
429
430
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
431
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
432
  else this.sbot.get(id, function (err, value) {
433
    if (err) return cb(err)
434
    cb(null, {key: id, value: value})
435
  })
436
}
437
438
SsbNpmRegistryServer.prototype.getFeedName = function (id, cb) {
439
  var self = this
440
  if (self.sbot.names && self.sbot.names.getSignifier) {
441
    self.sbot.names.getSignifier(id, function (err, name) {
442
      if (err || !name) tryLinks()
443
      else cb(null, name)
444
    })
445
  } else {
446
    tryLinks()
447
  }
448
  function tryLinks() {
449
    if (!self.sbot.links) return nope()
450
    pull(
451
      self.sbot.links({
452
        source: id,
453
        dest: id,
454
        rel: 'about',
455
        values: true,
456
        limit: 1,
457
        reverse: true,
458
        keys: false,
459
        meta: false
460
      }),
461
      pull.map(function (value) {
462
        return value && value.content && value.content.name
463
      }),
464
      pull.filter(Boolean),
465
      pull.take(1),
466
      pull.collect(function (err, names) {
467
        if (err || !names.length) return nope()
468
        cb(null, names[0])
469
      })
470
    )
471
  }
472
  function nope() {
473
    cb(null, '')
474
  }
475
}
476
477
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
478
  var self = this
479
  var stack = heads.slice()
480
  var seen = {}
481
  return function (abort, cb) {
482
    if (abort) return cb(abort)
483
    for (var id; stack.length && seen[id = stack.pop()];);
484
    if (!id) return cb(true)
485
    seen[id] = true
486
    // TODO: use DFS
487
    self.getMsg(id, function (err, msg) {
488
      if (err) return cb(new Error(err.stack || err))
489
      var c = msg && msg.value && msg.value.content
490
      var links = c && c[prop]
491
      if (Array.isArray(links)) {
492
        stack.push.apply(stack, links)
493
      } else if (links) {
494
        stack.push(links)
495
      }
496
      cb(null, msg)
497
    })
498
  }
499
}
500
501
function Req(server, req, res) {
502
  this.server = server
503
  this.req = req
504
  this.res = res
505
  this.blobsToPush = []
506
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
507
  var ua = this.req.headers['user-agent']
508
  var m = /\bnpm\/([0-9]*)/.exec(ua)
509
  var npmVersion = m && m[1]
510
  this.needShasum = server.needShasum != null ? server.needShasum :
511
    (npmVersion && npmVersion < 5)
512
  this.baseUrl = this.server.npmConfig.baseUrl
513
  if (this.baseUrl) {
514
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
515
  } else {
516
    var hostname = req.headers.host
517
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
518
    this.baseUrl = 'http://' + hostname
519
  }
520
521
  // prefer serving blobs from ssb-ws, for consistency of tarball URLs.
522
  this.blobBaseUrl = this.server.wsUrl && this.isLocal()
523
    ? this.server.wsUrl + '/blobs/get/'
524
    : this.baseUrl + '/-/blobs/get/'
525
}
526
527
Req.prototype.isLocal = function () {
528
  if (this.server.config.host && this.server.config.host !== 'localhost') return false
529
  var remoteAddr = this.req.socket.remoteAddress
530
  return remoteAddr === '::1'
531
    || remoteAddr === '::ffff:127.0.0.1'
532
    || remoteAddr === '127.0.0.1'
533
}
534
535
Req.prototype.serve = function () {
536
  if (process.env.DEBUG) {
537
    console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
538
  }
539
  this.res.setTimeout(0)
540
  var pathname = this.req.url.replace(/\?.*/, '')
541
  var m
542
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
543
    try {
544
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
545
      this.headMsgPlus = !!m[1]
546
      // ^ means also include packages published after the head message id
547
    } catch(e) {
548
      return this.respondError(400, e.stack || e)
549
    }
550
    pathname = m[3]
551
  }
552
  if (pathname === '/') return this.serveHome()
553
  if (pathname === '/robots.txt') return this.serveRobots()
554
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
555
  if (pathname === '/-/whoami') return this.serveWhoami()
556
  if (pathname === '/-/ping') return this.respond(200, true)
557
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
558
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
559
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
560
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
561
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
562
  return this.respond(404, {error: 'Not found'})
563
}
564
565
Req.prototype.respond = function (status, message) {
566
  this.res.writeHead(status, {'content-type': 'application/json'})
567
  this.res.end(message && JSON.stringify(message, 0, 2))
568
}
569
570
Req.prototype.respondError = function (status, message) {
571
  this.respond(status, {error: message})
572
}
573
574
Req.prototype.respondErrorStr = function (status, err) {
575
  this.res.writeHead(status, {'content-type': 'text/plain'})
576
  this.res.end(err.stack || err)
577
}
578
579
Req.prototype.respondRaw = function (status, body) {
580
  this.res.writeHead(status)
581
  this.res.end(body)
582
}
583
584
Req.prototype.serveHome = function () {
585
  var self = this
586
  self.res.writeHead(200, {'content-type': 'text/html'})
587
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
588
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
589
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
590
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
591
    '</body></html>')
592
}
593
594
Req.prototype.blobDist = function (id) {
595
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
596
  if (!m) throw new Error('bad blob id: ' + id)
597
  return {
598
    integrity: m[2] + '-' + m[1],
599
    tarball: this.blobBaseUrl + id
600
  }
601
}
602
603
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
604
  var self = this
605
  pull(
606
    self.server.sbot.links({
607
      source: feedId,
608
      dest: blobId,
609
      rel: 'mentions',
610
      values: true,
611
    }),
612
    pull.filter(function (msg) {
613
      var c = msg && msg.value && msg.value.content
614
      return c.type === 'npm-packages'
615
    }),
616
    pull.collect(function (err, msgs) {
617
      if (err) return cb(err)
618
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
619
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
620
      // TODO: make a smarter decision about which message id to use
621
      cb(null, msgs.pop().key)
622
    })
623
  )
624
}
625
626
Req.prototype.resolvePkg = function (pkgSpec, cb) {
627
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
628
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
629
  var self = this
630
  var pkgName = m[1]
631
  var spec = m[2] || '*'
632
  var versions = {}
633
  var distTags = {}
634
  pull(
635
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
636
    pull.drain(function (mention) {
637
      var data = decodeName(mention.name)
638
      if (!data.version) return
639
      if (data.distTag) {
640
        distTags[data.distTag] = data.version
641
      }
642
      versions[data.version] = {
643
        author: mention.author,
644
        name: pkgName,
645
        version: data.version,
646
        blobId: mention.link
647
      }
648
    }, function (err) {
649
      if (err) return cb(err)
650
      var version = distTags[spec]
651
        || semver.maxSatisfying(Object.keys(versions), spec)
652
      var item = versions[version]
653
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
654
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
655
        if (err) return cb(err)
656
        item.msgId = id
657
        cb(null, item)
658
      })
659
    })
660
  )
661
}
662
663
Req.prototype.resolvePkgs = function (specs, cb) {
664
  var done = multicb({pluck: 1})
665
  var self = this
666
  specs.forEach(function (spec) {
667
    self.resolvePkg(spec, done())
668
  })
669
  done(cb)
670
}
671
672
Req.prototype.serveBootstrap = function () {
673
  var self = this
674
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
675
  var postInstallCmd = self.server.npmConfig.postInstallCmd
676
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
677
  var ssbNpmRegistryName = require('./package.json').name
678
  var ssbNpmRegistryVersion = require('./package.json').version
679
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
680
  var done = multicb({pluck: 1, spread: true})
681
  self.resolvePkg(ssbNpmRegistrySpec, done())
682
  self.resolvePkgs(pkgs, done())
683
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
684
    if (err) return self.respondErrorStr(500, err.stack || err)
685
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
686
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
687
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
688
    var ssbNpmRegistryNameVersion = ssbNpmRegistryPkgInfo.name + '-' + ssbNpmRegistryPkgInfo.version
689
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
690
    var globalPkgs = pkgsInfo.map(function (info) {
691
      return info.name + '@' + info.version
692
    }).join(' ')
693
    var npmCmd = 'install -g ' + globalPkgs
694
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
695
    var wsUrl = self.baseUrl + '/-'
696
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
697
698
    var script =
699
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
700
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
701
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
702
      'tar xzf package.tgz &&\n' +
703
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
704
        pkgMsgs.map(function (id) {
705
          return '  --branch ' + id + ' \\\n'
706
        }).join('') +
707
        '  -- ' + npmCmd +
708
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
709
710
    self.res.writeHead(200, {'Content-type': 'text/plain'})
711
    self.res.end(script)
712
  })
713
}
714
715
Req.prototype.serveRobots = function () {
716
  this.res.writeHead(200, {'Content-type': 'text/plain'})
717
  this.res.end('User-agent: *\nDisallow: /\n')
718
}
719
720
Req.prototype.serveWhoami = function () {
721
  var self = this
722
  self.server.sbot.whoami(function (err, feed) {
723
    if (err) return self.respondError(500, err.stack || err)
724
    self.respond(200, {username: feed.id})
725
  })
726
}
727
728
Req.prototype.serveUser1 = function () {
729
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
730
}
731
732
Req.prototype.serveBlob = function (id) {
733
  var self = this
734
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
735
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
736
    if (err) {
737
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
738
      else return self.respondErrorStr(500, err.message || err)
739
    }
740
    self.res.writeHead(200, {
741
      'Cache-Control': 'public, max-age=315360000',
742
      'Content-Length': size,
743
      'etag': id
744
    })
745
    pull(
746
      readBlob,
747
      toPull(self.res, function (err) {
748
        if (err) console.error('[npm-registry]', err)
749
      })
750
    )
751
  })
752
}
753
754
Req.prototype.serveMsg = function (id) {
755
  var self = this
756
  try { id = decodeURIComponent(id) }
757
  catch (e) {}
758
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
759
  self.server.sbot.get(id, function (err, value) {
760
    if (err) return self.respondError(500, err.message || err)
761
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
762
    self.res.writeHead(200, {
763
      'Content-Type': 'application/json',
764
      'Cache-Control': 'public, max-age=315360000',
765
      'Content-Length': out.length,
766
      'etag': id
767
    })
768
    self.res.end(out)
769
  })
770
}
771
772
function decodeName(name) {
773
  var parts = String(name).replace(/\.tgz$/, '').split(':')
774
  return {
775
    name: parts[1],
776
    version: parts[2],
777
    distTag: parts[3],
778
  }
779
}
780
781
Req.prototype.getMsgMentions = function (name) {
782
  return pull(
783
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
784
    // decryption could be done here
785
    pull.map(function (msg) {
786
      var c = msg.value && msg.value.content
787
      if (!c.mentions || !Array.isArray(c.mentions)) return []
788
      return c.mentions.map(function (mention) {
789
        return {
790
          name: mention.name,
791
          size: mention.size,
792
          link: mention.link,
793
          author: msg.value.author,
794
          ts: msg.value.timestamp,
795
        }
796
      })
797
    }),
798
    pull.flatten(),
799
    pull.filter(typeof name === 'string' ? function (link) {
800
      return link.name === name
801
    } : name && name.$prefix ? function (link) {
802
      return link.name.substr(0, name.$prefix.length) === name.$prefix
803
    } : function () {
804
      throw new TypeError('unsupported name filter')
805
    })
806
  )
807
}
808
809
Req.prototype.getMentions = function (name) {
810
  var useMsgMentions = this.headMsgIds
811
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
812
  if (useServerMentions && !this.server.sbot.links2) {
813
    return this.headMsgPlus
814
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
815
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
816
  }
817
  return cat([
818
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
819
    useServerMentions ? this.server.getMentions(name) : pull.empty()
820
  ])
821
}
822
823
Req.prototype.getMentionLinks = function (blobId) {
824
  return pull(
825
    this.headMsgIds
826
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
827
      : this.server.sbot.links({
828
        dest: blobId,
829
        rel: 'mentions',
830
        values: true,
831
      }),
832
    // decryption could be done here
833
    pull.map(function (msg) {
834
      var c = msg.value && msg.value.content
835
      return c && Array.isArray(c.mentions) && c.mentions || []
836
    }),
837
    pull.flatten(),
838
    pull.filter(function (link) {
839
      return link && link.link === blobId
840
    })
841
  )
842
}
843
844
Req.prototype.servePkg = function (pathname) {
845
  var self = this
846
  var parts = pathname.split('/')
847
  var pkgName = parts.shift().replace(/%2f/i, '/')
848
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
849
  var spec = parts.shift()
850
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
851
  if (parts.length > 0) return this.respondError(404)
852
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
853
  var obj = {
854
    _id: pkgName,
855
    name: pkgName,
856
    'dist-tags': {},
857
    versions: {},
858
    time: {}
859
  }
860
  var distTags = {/* <tag>: {version, ts}*/}
861
  pull(
862
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
863
    pull.drain(function (mention) {
864
      var data = decodeName(mention.name)
865
      if (!data.version) return
866
      if (data.distTag) {
867
        var tag = distTags[data.distTag]
868
        if (!tag || mention.ts > tag.ts) {
869
          /* TODO: sort by causal order instead of only by timestamps */
870
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
871
        }
872
      }
873
      obj.versions[data.version] = {
874
        author: {
875
          url: mention.author
876
        },
877
        name: pkgName,
878
        version: data.version,
879
        dist: self.blobDist(mention.link)
880
      }
881
      var ts = new Date(mention.ts)
882
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
883
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
884
      obj.time[data.version] = ts.toISOString()
885
    }, function (err) {
886
      if (err) return self.respondError(500, err.stack || err)
887
      for (var tag in distTags) {
888
        obj['dist-tags'][tag] = distTags[tag].version
889
      }
890
      if (spec) resolveSpec()
891
      else if (self.fetchAll) resolveAll()
892
      else resolved()
893
    })
894
  )
895
  function resolveSpec() {
896
    var version = obj['dist-tags'][spec]
897
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
898
    obj = obj.versions[version]
899
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
900
    self.populatePackageJson(obj, function (err, pkg) {
901
      if (err) return resolved(err)
902
      obj = pkg || obj
903
      resolved()
904
    })
905
  }
906
  function resolveVersion(version, cb) {
907
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
908
      if (err) return cb(err)
909
      if (pkg) obj.versions[version] = pkg
910
      if (pkg && pkg.license && !obj.license) obj.license = pkg.license
911
      cb()
912
    })
913
  }
914
  function resolveAll() {
915
    var done = multicb()
916
    for (var version in obj.versions) {
917
      resolveVersion(version, done())
918
    }
919
    done(resolved)
920
  }
921
  function resolved(err) {
922
    if (err) return self.respondError(500, err.stack || err)
923
    self.respond(200, obj)
924
  }
925
}
926
927
Req.prototype.servePrebuild = function (name) {
928
  var self = this
929
  var getMention = self.getMentions('prebuild:' + name)
930
  var blobsByAuthor = {/* <author>: BlobId */}
931
  pull(getMention, pull.drain(function (link) {
932
    blobsByAuthor[link.author] = link.link
933
  }, function (err) {
934
    if (err) return self.respondError(500, err.stack || err)
935
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
936
    var blobId
937
    for (var feed in blobsByAuthor) {
938
      var blob = blobId = blobsByAuthor[feed]
939
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
940
      feeds.push(feed)
941
    }
942
    switch (Object.keys(authorsByLink).length) {
943
      case 0:
944
        return self.respondError(404, 'Not Found')
945
      case 1:
946
        return self.serveBlob(blobId)
947
      default:
948
        return self.respond(300, {choices: authorsByLink})
949
    }
950
  }))
951
}
952
953
var localhosts = {
954
  '::1': true,
955
  '127.0.0.1': true,
956
  '::ffff:127.0.0.1': true,
957
}
958
959
Req.prototype.publishPkg = function (pkgName) {
960
  var self = this
961
  var remoteAddress = self.req.socket.remoteAddress
962
  if (!(remoteAddress in localhosts)) {
963
    return self.respondError(403, 'You may not publish as this user.')
964
  }
965
966
  var chunks = []
967
  self.req.on('data', function (data) {
968
    chunks.push(data)
969
  })
970
  self.req.on('end', function () {
971
    var data
972
    try {
973
      data = JSON.parse(Buffer.concat(chunks))
974
    } catch(e) {
975
      return self.respondError(400, e.stack)
976
    }
977
    return self.publishPkg2(pkgName, data || {})
978
  })
979
}
980
981
Req.prototype.publishPkg2 = function (name, data) {
982
  var self = this
983
  if (data.users) console.trace('[npm-registry] users property is not supported')
984
  var attachments = data._attachments || {}
985
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
986
  var done = multicb()
987
  function addAttachmentAsBlob(filename, cb) {
988
    var data = attachments[filename].data
989
    var tarball = new Buffer(data, 'base64')
990
    var length = attachments[filename].length
991
    if (length && length !== tarball.length) return self.respondError(400,
992
      'Length mismatch for attachment \'' + filename + '\'')
993
    self.server.sbot.blobs.add(function (err, id) {
994
      if (err) return cb(err)
995
      self.blobsToPush.push(id)
996
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
997
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
998
      cb()
999
    })(pull.once(tarball))
1000
  }
1001
  for (var filename in attachments) {
1002
    addAttachmentAsBlob(filename, done())
1003
  }
1004
  done(function (err) {
1005
    if (err) return self.respondError(500, err.stack || err)
1006
    try {
1007
      self.publishPkg3(name, data, links)
1008
    } catch(e) {
1009
      self.respondError(500, e.stack || e)
1010
    }
1011
  })
1012
}
1013
1014
Req.prototype.publishPkg3 = function (name, data, links) {
1015
  var self = this
1016
  var versions = data.versions || {}
1017
  var linksByVersion = {/* <version>: link */}
1018
1019
  // associate tarball blobs with versions
1020
  for (var version in versions) {
1021
    var pkg = versions[version]
1022
    if (!pkg) return self.respondError(400, 'Bad package object')
1023
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
1024
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
1025
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
1026
    var m = /\/-\/([^\/]+|@[^\/]+\/[^\/]+)$/.exec(pkg.dist.tarball)
1027
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
1028
    var filename = m[1]
1029
    var link = links[filename]
1030
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
1031
    // TODO?: try to find missing tarball mentioned in other messages
1032
    if (pkg.version && pkg.version !== version)
1033
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
1034
    linksByVersion[version] = link
1035
    link.version = version
1036
    link.license = pkg.license
1037
    link.dependencies = pkg.dependencies || {}
1038
    link.optionalDependencies = pkg.optionalDependencies
1039
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
1040
  }
1041
1042
  // associate blobs with dist-tags
1043
  var tags = data['dist-tags'] || {}
1044
  for (var tag in tags) {
1045
    var version = tags[tag]
1046
    var link = linksByVersion[version]
1047
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
1048
    // TODO?: support setting dist-tag without version,
1049
    // by looking up a tarball blob for the version
1050
    link.tag = tag
1051
  }
1052
1053
  // compute blob links to publish
1054
  var mentions = []
1055
  for (var filename in links) {
1056
    var link = links[filename] || {}
1057
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
1058
    mentions.push({
1059
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
1060
      link: link.link,
1061
      size: link.size,
1062
      shasum: link.shasum,
1063
      license: link.license,
1064
      dependencies: link.dependencies,
1065
      optionalDependencies: link.optionalDependencies,
1066
      bundledDependencies: link.bundledDependencies,
1067
    })
1068
  }
1069
  return self.publishPkgs(mentions)
1070
}
1071
1072
Req.prototype.publishPkgs = function (mentions) {
1073
  var self = this
1074
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
1075
    if (err) return self.respondError(500, err.stack || err)
1076
    self.server.pushBlobs(self.blobsToPush, function (err) {
1077
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
1078
      self.respond(201)
1079
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
1080
    })
1081
  })
1082
}
1083
1084
Req.prototype.populatePackageJson = function (obj, cb) {
1085
  var self = this
1086
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1087
  var deps, depsWithOptionalDeps, bundledDeps, shasum, license
1088
1089
  // Combine metadata from mentions of the blob, to construct most accurate
1090
  // metadata, including dependencies info. If needed, fall back to fetching
1091
  // metadata from the tarball blob.
1092
1093
  pull(
1094
    self.getMentionLinks(blobId),
1095
    pull.drain(function (link) {
1096
      if (link.dependencies) deps = link.dependencies
1097
      // if optionalDependencies is present, let the dependencies value
1098
      // override that from other links. because old versions that didn't
1099
      // publish optionalDependencies are missing optionalDependencies from
1100
      // their dependencies.
1101
      if (link.optionalDependencies) depsWithOptionalDeps = {
1102
        deps: link.dependencies,
1103
        optionalDeps: link.optionalDependencies
1104
      }
1105
      if (link.shasum) shasum = link.shasum
1106
      if (link.license) license = link.license
1107
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1108
      // how to handle multiple assignments of dependencies to a package?
1109
    }, function (err) {
1110
      if (err) return cb(new Error(err.stack || err))
1111
      if (deps && (shasum || !self.needShasum)) {
1112
        // assume that the dependencies in the links to the blob are
1113
        // correct.
1114
        if (depsWithOptionalDeps) {
1115
          obj.dependencies = depsWithOptionalDeps.deps || deps
1116
          obj.optionalDependencies = depsWithOptionalDeps.optionalDeps
1117
        } else {
1118
          obj.dependencies = deps
1119
        }
1120
        obj.bundledDependencies = bundledDeps
1121
        obj.license = license
1122
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1123
        next(obj)
1124
      } else {
1125
        // get dependencies from the tarball
1126
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1127
          if (err) return cb(err)
1128
          pkg.dist = obj.dist
1129
          pkg.dist.shasum = pkg._shasum
1130
          pkg.author = pkg.author || obj.author
1131
          pkg.version = pkg.version || obj.version
1132
          pkg.name = pkg.name || obj.name
1133
          pkg.license = pkg.license || obj.license
1134
          next(pkg)
1135
        })
1136
      }
1137
    })
1138
  )
1139
  function next(pkg) {
1140
    var feedId = obj.author.url
1141
    self.server.getFeedName(feedId, function (err, name) {
1142
      if (err) console.error('[npm-registry]', err.stack || err), name = ''
1143
      pkg._npmUser = {
1144
        email: feedId,
1145
        name: name
1146
      }
1147
      cb(null, pkg)
1148
    })
1149
  }
1150
}
1151
1152
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1153
  var self = this
1154
  getBlob(sbot, id, function (err, readBlob) {
1155
    if (err) return cb(err)
1156
    cb = once(cb)
1157
    var extract = tar.extract()
1158
    var pkg, shasum
1159
    extract.on('entry', function (header, stream, next) {
1160
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1161
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1162
          if (err) return cb(err)
1163
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1164
          catch(e) { return cb(e) }
1165
          next()
1166
        }))
1167
      } else {
1168
        stream.on('end', next)
1169
        stream.resume()
1170
      }
1171
    })
1172
    extract.on('finish', function () {
1173
      pkg._shasum = shasum
1174
      cb(null, pkg)
1175
    })
1176
    pull(
1177
      readBlob,
1178
      hash('sha1', 'hex', function (err, sum) {
1179
        if (err) return cb(err)
1180
        shasum = sum
1181
      }),
1182
      toPull(zlib.createGunzip()),
1183
      toPull(extract)
1184
    )
1185
  })
1186
}
1187