git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || 'localhost'
90
  var autoAuth = conf.autoAuth !== false
91
  var listenUrl
92
93
  var server = http.createServer(exports.respond(sbot, config))
94
  var getAddress = onceify(function (cb) {
95
    server.on('error', cb)
96
    server.listen(port, host, function () {
97
      server.removeListener('error', cb)
98
      var addr = this.address()
99
      var listenHost = addr.address
100
      var regHost = listenHost === '::' ? '::1' :
101
                    listenHost === '0.0.0.0' ? '127.0.0.1' :
102
                    listenHost
103
      var regUrl = 'http://' + formatHost(regHost) + ':' + addr.port
104
      listenUrl = 'http://' + formatHost(listenHost) + ':' + addr.port
105
      if (autoAuth) npmLogin(regUrl, next)
106
      else next()
107
      function next(err) {
108
        cb(err, regUrl)
109
      }
110
    })
111
    sbot.on('close', function () {
112
      server.close()
113
    })
114
  })
115
116
  /* getAddress called by local RPC is used to discover the local
117
   * ssb-npm-registry address that can be given to local npm clients. However,
118
   * when running the server we output the address the server is listening on,
119
   * to avoid misleading situations like saying listening on localhost but
120
   * actually listening on a wildcard address. */
121
  getAddress(function (err) {
122
    if (err) return console.error('[npm-registry]', err.stack || err)
123
    console.log('[npm-registry] Listening on ' + listenUrl)
124
  })
125
126
  return {
127
    getAddress: getAddress
128
  }
129
}
130
131
exports.respond = function (sbot, config) {
132
  var reg = new SsbNpmRegistryServer(sbot, config)
133
  return function (req, res) {
134
    new Req(reg, req, res).serve()
135
  }
136
}
137
138
function publishMsg(sbot, value, cb) {
139
  var gotExpectedPrevious = false
140
  sbot.publish(value, function next(err, msg) {
141
    if (err && /^expected previous:/.test(err.message)) {
142
      // retry once on this error
143
      if (gotExpectedPrevious) return cb(err)
144
      gotExpectedPrevious = true
145
      return sbot.publish(value, next)
146
    }
147
    cb(err, msg)
148
  })
149
}
150
151
function getDependencyBranches(sbot, id, cb) {
152
  // get ids of heads of tree of dependencyBranch message links to include all
153
  // the dependencies of the given tarball id.
154
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
155
  var msgs = {}
156
  var branched = {}
157
  var blobs = {}
158
159
  function addPkgById(id, cb) {
160
    if (blobs[id]) return cb()
161
    blobs[id] = true
162
    getPackageJsonCached(id, function (err, pkg) {
163
      if (err) return cb(err)
164
      var done = multicb()
165
      for (var name in pkg.dependencies || {}) {
166
        addPkgBySpec(name, pkg.dependencies[name], done())
167
      }
168
      for (var name in pkg.optionalDependencies || {}) {
169
        addPkgBySpec(name, pkg.optionalDependencies[name], done())
170
      }
171
      done(cb)
172
    })
173
  }
174
175
  function addPkgBySpec(name, spec, cb) {
176
    var versions = {}
177
    var distTags = {}
178
    pull(
179
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
180
      pull.filter(mentionMatches(null, name, spec)),
181
      pull.map(function (mention) {
182
        // query to get the messages since links2 does not include the msg id
183
        return sbot.links({
184
          source: mention.author,
185
          dest: mention.link,
186
          rel: 'mentions',
187
          values: true,
188
        })
189
      }),
190
      pull.flatten(),
191
      pull.drain(function (msg) {
192
        var c = msg && msg.value && msg.value.content
193
        if (!c || !Array.isArray(c.mentions)) return
194
        c.mentions.forEach(function (link) {
195
          var data = link && link.name && decodeName(link.name)
196
          if (!data || data.name !== name) return
197
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
198
          if (data.distTag) distTags[data.distTag] = data.version
199
        })
200
      }, function (err) {
201
        if (err) return cb(err)
202
        var version = distTags[spec]
203
          || semver.maxSatisfying(Object.keys(versions), spec)
204
        var item = versions[version]
205
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
206
        msgs[item.msg.key] = item.msg.value
207
        var c = item.msg.value.content
208
        if (Array.isArray(c.dependencyBranch)) {
209
          for (var k = 0; k < c.dependencyBranch.length; k++) {
210
            branched[c.dependencyBranch[k]] = true
211
          }
212
        }
213
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
214
        addPkgById(item.mention.link, cb)
215
      })
216
    )
217
  }
218
219
  addPkgById(id, function (err) {
220
    if (err) return cb(err)
221
    var ids = []
222
    for (var key in msgs) {
223
      if (!branched[key]) ids.push(key)
224
    }
225
    cb(null, ids)
226
  })
227
}
228
229
function packageLinks(sbot, feed, id, name, spec) {
230
  var matches = mentionMatches(id, name, spec)
231
  return pull(
232
    sbot.links({
233
      source: feed,
234
      dest: id,
235
      rel: 'mentions',
236
      values: true,
237
    }),
238
    pull.filter(function (msg) {
239
      var c = msg && msg.value && msg.value.content
240
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
241
    })
242
  )
243
}
244
245
function getVersionBranches(sbot, link, cb) {
246
  var data = decodeName(link.name)
247
  var msgs = {}, branched = {}
248
  pull(
249
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
250
    pull.map(function (mention) {
251
      return packageLinks(sbot, mention.author, mention.link, data.name)
252
    }),
253
    pull.flatten(),
254
    pull.drain(function (msg) {
255
      var c = msg && msg.value && msg.value.content
256
      if (!c) return
257
      msgs[msg.key] = msg.value
258
      if (Array.isArray(c.versionBranch)) {
259
        for (var k = 0; k < c.versionBranch.length; k++) {
260
          branched[c.versionBranch[k]] = true
261
        }
262
      }
263
    }, function (err) {
264
      if (err) return cb(err)
265
      var ids = []
266
      for (var key in msgs) {
267
        if (!branched[key]) ids.push(key)
268
      }
269
      cb(null, ids)
270
    })
271
  )
272
}
273
274
// For each dependency that is not a bundledDependency, get message ids for
275
// that dependency name + version.
276
function publishSingleMention(sbot, mention, cb) {
277
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
278
  // Calculate dependencyBranch and versionBranch message ids.
279
  var value = {
280
    type: 'npm-packages',
281
    mentions: [mention]
282
  }
283
  var done = multicb({pluck: 1, spread: true})
284
  getDependencyBranches(sbot, mention.link, done())
285
  getVersionBranches(sbot, mention, done())
286
  done(function (err, dependencyBranches, versionBranches) {
287
    if (err) return cb(err)
288
    value.dependencyBranch = dependencyBranches || undefined
289
    value.versionBranch = versionBranches || undefined
290
    publishMsg(sbot, value, cb)
291
  })
292
}
293
294
function publishMentions(sbot, mentions, cb) {
295
  // console.error("publishing %s mentions", mentions.length)
296
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
297
  // if it is just one mention, fetch and add useful metadata
298
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
299
  publishMsg(sbot, {
300
    type: 'npm-packages',
301
    mentions: mentions,
302
  }, cb)
303
}
304
305
exports.publishPkgMentions = function (sbot, mentions, cb) {
306
  // try to fit the mentions into as few messages as possible,
307
  // while fitting under the message size limit.
308
  var msgs = []
309
  ;(function next(i, chunks) {
310
    if (i >= mentions.length) return cb(null, msgs)
311
    var chunkLen = Math.ceil(mentions.length / chunks)
312
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
313
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
314
      if (err && msgs.length) return onPartialPublish(err)
315
      if (err) return cb(err)
316
      msgs.push(msg)
317
      next(i + chunkLen, chunks)
318
    })
319
  })(0, 1)
320
  function onPartialPublish(err) {
321
    var remaining = mentions.length - i
322
    return cb(new Error('Published messages ' +
323
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
324
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
325
  }
326
}
327
328
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
329
  cb = once(cb)
330
  var waiting = 0
331
  var expandedMentions = mentions.map(function (link) {
332
    var id = link && link.link
333
    if (!id) return link
334
    waiting++
335
    var newLink = {}
336
    for (var k in link) newLink[k] = link[k]
337
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
338
      if (err) return cb(err)
339
      for (var k in props) newLink[k] = pkg[k]
340
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
341
      if (!--waiting) next()
342
    })
343
    return newLink
344
  })
345
  if (!waiting) next()
346
  function next() {
347
    cb(null, expandedMentions)
348
  }
349
}
350
351
function SsbNpmRegistryServer(sbot, config) {
352
  this.sbot = sbot
353
  this.config = config
354
  this.npmConfig = config.npm || {}
355
  this.host = this.npmConfig.host || 'localhost'
356
  this.fetchAll = this.npmConfig.fetchAll
357
  this.needShasum = this.npmConfig.needShasum
358
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
359
  this.getFeedName = memo({cache: lru(100)}, this.getFeedName)
360
  if (sbot.ws) {
361
    var wsPort = config.ws && Number(config.ws.port) || '8989'
362
    this.wsUrl = 'http://localhost:' + wsPort
363
  }
364
}
365
366
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
367
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
368
369
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
370
  var self = this
371
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
372
  ;(function next(i) {
373
    if (i >= ids.length) return cb()
374
    self.sbot.blobs.push(ids[i], function (err) {
375
      if (err) return cb(err)
376
      next(i+1)
377
    })
378
  })(0)
379
}
380
381
function getBlob(sbot, id, cb) {
382
  var blobs = sbot.blobs
383
  blobs.size(id, function (err, size) {
384
    if (err && err.code !== 'ENOENT') return cb(err)
385
    if (typeof size === 'number') cb(null, blobs.get(id), size)
386
    else blobs.want(id, function (err, got) {
387
      if (err) cb(err)
388
      else if (!got) cb('missing blob ' + id)
389
      else blobs.size(id, function (err, size) {
390
        if (err) return cb(err)
391
        cb(null, blobs.get(id), size)
392
      })
393
    })
394
  })
395
}
396
397
function getMentions(links2, name) {
398
  return links2.read({
399
    query: [
400
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
401
      {$filter: {dest: {$prefix: '&'}}},
402
      {$map: {
403
        name: ['rel', 1],
404
        size: ['rel', 2],
405
        link: 'dest',
406
        author: 'source',
407
        ts: 'ts'
408
      }}
409
    ]
410
  })
411
}
412
413
function mentionMatches(id, name, spec) {
414
  return function (mention) {
415
    var data = mention
416
      && (!id || id === mention.link)
417
      && mention.name
418
      && decodeName(mention.name)
419
    return data
420
      && data.name === name
421
      && (spec ? semver.satisfies(data.version, spec) : true)
422
  }
423
}
424
425
SsbNpmRegistryServer.prototype.getMentions = function (name) {
426
  if (!this.sbot.links2) return pull.empty()
427
  return getMentions(this.sbot.links2, name)
428
}
429
430
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
431
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
432
  else this.sbot.get(id, function (err, value) {
433
    if (err) return cb(err)
434
    cb(null, {key: id, value: value})
435
  })
436
}
437
438
SsbNpmRegistryServer.prototype.getFeedName = function (id, cb) {
439
  var self = this
440
  if (self.sbot.names && self.sbot.names.getSignifier) {
441
    self.sbot.names.getSignifier(id, function (err, name) {
442
      if (err || !name) tryLinks()
443
      else cb(null, name)
444
    })
445
  } else {
446
    tryLinks()
447
  }
448
  function tryLinks() {
449
    if (!self.sbot.links) return nope()
450
    pull(
451
      self.sbot.links({
452
        source: id,
453
        dest: id,
454
        rel: 'about',
455
        values: true,
456
        limit: 1,
457
        reverse: true,
458
        keys: false,
459
        meta: false
460
      }),
461
      pull.map(function (value) {
462
        return value && value.content && value.content.name
463
      }),
464
      pull.filter(Boolean),
465
      pull.take(1),
466
      pull.collect(function (err, names) {
467
        if (err || !names.length) return nope()
468
        cb(null, names[0])
469
      })
470
    )
471
  }
472
  function nope() {
473
    cb(null, '')
474
  }
475
}
476
477
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
478
  var self = this
479
  var stack = heads.slice()
480
  var seen = {}
481
  return function (abort, cb) {
482
    if (abort) return cb(abort)
483
    for (var id; stack.length && seen[id = stack.pop()];);
484
    if (!id) return cb(true)
485
    seen[id] = true
486
    // TODO: use DFS
487
    self.getMsg(id, function (err, msg) {
488
      if (err) return cb(new Error(err.stack || err))
489
      var c = msg && msg.value && msg.value.content
490
      var links = c && c[prop]
491
      if (Array.isArray(links)) {
492
        stack.push.apply(stack, links)
493
      } else if (links) {
494
        stack.push(links)
495
      }
496
      cb(null, msg)
497
    })
498
  }
499
}
500
501
function Req(server, req, res) {
502
  this.server = server
503
  this.req = req
504
  this.res = res
505
  this.blobsToPush = []
506
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
507
  var ua = this.req.headers['user-agent']
508
  var m = /\bnpm\/([0-9]*)/.exec(ua)
509
  var npmVersion = m && m[1]
510
  this.needShasum = server.needShasum != null ? server.needShasum :
511
    (npmVersion && npmVersion < 5)
512
  this.baseUrl = this.server.npmConfig.baseUrl
513
  if (this.baseUrl) {
514
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
515
  } else {
516
    var hostname = req.headers.host
517
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
518
    this.baseUrl = 'http://' + hostname
519
  }
520
521
  // prefer serving blobs from ssb-ws, for consistency of tarball URLs.
522
  this.blobBaseUrl = this.server.wsUrl && this.isLocal()
523
    ? this.server.wsUrl + '/blobs/get/'
524
    : this.baseUrl + '/-/blobs/get/'
525
}
526
527
Req.prototype.isLocal = function () {
528
  var remoteAddr = this.req.socket.remoteAddress
529
  return remoteAddr === '::1'
530
    || remoteAddr === '::ffff:127.0.0.1'
531
    || remoteAddr === '127.0.0.1'
532
}
533
534
Req.prototype.serve = function () {
535
  if (process.env.DEBUG) {
536
    console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
537
  }
538
  this.res.setTimeout(0)
539
  var pathname = this.req.url.replace(/\?.*/, '')
540
  var m
541
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
542
    try {
543
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
544
      this.headMsgPlus = !!m[1]
545
      // ^ means also include packages published after the head message id
546
    } catch(e) {
547
      return this.respondError(400, e.stack || e)
548
    }
549
    pathname = m[3]
550
  }
551
  if (pathname === '/') return this.serveHome()
552
  if (pathname === '/robots.txt') return this.serveRobots()
553
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
554
  if (pathname === '/-/whoami') return this.serveWhoami()
555
  if (pathname === '/-/ping') return this.respond(200, true)
556
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
557
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
558
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
559
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
560
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
561
  return this.respond(404, {error: 'Not found'})
562
}
563
564
Req.prototype.respond = function (status, message) {
565
  this.res.writeHead(status, {'content-type': 'application/json'})
566
  this.res.end(message && JSON.stringify(message, 0, 2))
567
}
568
569
Req.prototype.respondError = function (status, message) {
570
  this.respond(status, {error: message})
571
}
572
573
Req.prototype.respondErrorStr = function (status, err) {
574
  this.res.writeHead(status, {'content-type': 'text/plain'})
575
  this.res.end(err.stack || err)
576
}
577
578
Req.prototype.respondRaw = function (status, body) {
579
  this.res.writeHead(status)
580
  this.res.end(body)
581
}
582
583
Req.prototype.serveHome = function () {
584
  var self = this
585
  self.res.writeHead(200, {'content-type': 'text/html'})
586
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
587
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
588
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
589
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
590
    '</body></html>')
591
}
592
593
Req.prototype.blobDist = function (id) {
594
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
595
  if (!m) throw new Error('bad blob id: ' + id)
596
  return {
597
    integrity: m[2] + '-' + m[1],
598
    tarball: this.blobBaseUrl + id
599
  }
600
}
601
602
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
603
  var self = this
604
  pull(
605
    self.server.sbot.links({
606
      source: feedId,
607
      dest: blobId,
608
      rel: 'mentions',
609
      values: true,
610
    }),
611
    pull.filter(function (msg) {
612
      var c = msg && msg.value && msg.value.content
613
      return c.type === 'npm-packages'
614
    }),
615
    pull.collect(function (err, msgs) {
616
      if (err) return cb(err)
617
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
618
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
619
      // TODO: make a smarter decision about which message id to use
620
      cb(null, msgs.pop().key)
621
    })
622
  )
623
}
624
625
Req.prototype.resolvePkg = function (pkgSpec, cb) {
626
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
627
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
628
  var self = this
629
  var pkgName = m[1]
630
  var spec = m[2] || '*'
631
  var versions = {}
632
  var distTags = {}
633
  pull(
634
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
635
    pull.drain(function (mention) {
636
      var data = decodeName(mention.name)
637
      if (!data.version) return
638
      if (data.distTag) {
639
        distTags[data.distTag] = data.version
640
      }
641
      versions[data.version] = {
642
        author: mention.author,
643
        name: pkgName,
644
        version: data.version,
645
        blobId: mention.link
646
      }
647
    }, function (err) {
648
      if (err) return cb(err)
649
      var version = distTags[spec]
650
        || semver.maxSatisfying(Object.keys(versions), spec)
651
      var item = versions[version]
652
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
653
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
654
        if (err) return cb(err)
655
        item.msgId = id
656
        cb(null, item)
657
      })
658
    })
659
  )
660
}
661
662
Req.prototype.resolvePkgs = function (specs, cb) {
663
  var done = multicb({pluck: 1})
664
  var self = this
665
  specs.forEach(function (spec) {
666
    self.resolvePkg(spec, done())
667
  })
668
  done(cb)
669
}
670
671
Req.prototype.serveBootstrap = function () {
672
  var self = this
673
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
674
  var postInstallCmd = self.server.npmConfig.postInstallCmd
675
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
676
  var ssbNpmRegistryName = require('./package.json').name
677
  var ssbNpmRegistryVersion = require('./package.json').version
678
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
679
  var done = multicb({pluck: 1, spread: true})
680
  self.resolvePkg(ssbNpmRegistrySpec, done())
681
  self.resolvePkgs(pkgs, done())
682
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
683
    if (err) return self.respondErrorStr(500, err.stack || err)
684
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
685
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
686
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
687
    var ssbNpmRegistryNameVersion = ssbNpmRegistryPkgInfo.name + '-' + ssbNpmRegistryPkgInfo.version
688
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
689
    var globalPkgs = pkgsInfo.map(function (info) {
690
      return info.name + '@' + info.version
691
    }).join(' ')
692
    var npmCmd = 'install -g ' + globalPkgs
693
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
694
    var wsUrl = self.baseUrl + '/-'
695
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
696
697
    var script =
698
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
699
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
700
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
701
      'tar xzf package.tgz &&\n' +
702
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
703
        pkgMsgs.map(function (id) {
704
          return '  --branch ' + id + ' \\\n'
705
        }).join('') +
706
        '  -- ' + npmCmd +
707
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
708
709
    self.res.writeHead(200, {'Content-type': 'text/plain'})
710
    self.res.end(script)
711
  })
712
}
713
714
Req.prototype.serveRobots = function () {
715
  this.res.writeHead(200, {'Content-type': 'text/plain'})
716
  this.res.end('User-agent: *\nDisallow: /\n')
717
}
718
719
Req.prototype.serveWhoami = function () {
720
  var self = this
721
  self.server.sbot.whoami(function (err, feed) {
722
    if (err) return self.respondError(500, err.stack || err)
723
    self.respond(200, {username: feed.id})
724
  })
725
}
726
727
Req.prototype.serveUser1 = function () {
728
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
729
}
730
731
Req.prototype.serveBlob = function (id) {
732
  var self = this
733
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
734
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
735
    if (err) {
736
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
737
      else return self.respondErrorStr(500, err.message || err)
738
    }
739
    self.res.writeHead(200, {
740
      'Cache-Control': 'public, max-age=315360000',
741
      'Content-Length': size,
742
      'etag': id
743
    })
744
    pull(
745
      readBlob,
746
      toPull(self.res, function (err) {
747
        if (err) console.error('[npm-registry]', err)
748
      })
749
    )
750
  })
751
}
752
753
Req.prototype.serveMsg = function (id) {
754
  var self = this
755
  try { id = decodeURIComponent(id) }
756
  catch (e) {}
757
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
758
  self.server.sbot.get(id, function (err, value) {
759
    if (err) return self.respondError(500, err.message || err)
760
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
761
    self.res.writeHead(200, {
762
      'Content-Type': 'application/json',
763
      'Cache-Control': 'public, max-age=315360000',
764
      'Content-Length': out.length,
765
      'etag': id
766
    })
767
    self.res.end(out)
768
  })
769
}
770
771
function decodeName(name) {
772
  var parts = String(name).replace(/\.tgz$/, '').split(':')
773
  return {
774
    name: parts[1],
775
    version: parts[2],
776
    distTag: parts[3],
777
  }
778
}
779
780
Req.prototype.getMsgMentions = function (name) {
781
  return pull(
782
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
783
    // decryption could be done here
784
    pull.map(function (msg) {
785
      var c = msg.value && msg.value.content
786
      if (!c.mentions || !Array.isArray(c.mentions)) return []
787
      return c.mentions.map(function (mention) {
788
        return {
789
          name: mention.name,
790
          size: mention.size,
791
          link: mention.link,
792
          author: msg.value.author,
793
          ts: msg.value.timestamp,
794
        }
795
      })
796
    }),
797
    pull.flatten(),
798
    pull.filter(typeof name === 'string' ? function (link) {
799
      return link.name === name
800
    } : name && name.$prefix ? function (link) {
801
      return link.name.substr(0, name.$prefix.length) === name.$prefix
802
    } : function () {
803
      throw new TypeError('unsupported name filter')
804
    })
805
  )
806
}
807
808
Req.prototype.getMentions = function (name) {
809
  var useMsgMentions = this.headMsgIds
810
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
811
  if (useServerMentions && !this.server.sbot.links2) {
812
    return this.headMsgPlus
813
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
814
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
815
  }
816
  return cat([
817
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
818
    useServerMentions ? this.server.getMentions(name) : pull.empty()
819
  ])
820
}
821
822
Req.prototype.getMentionLinks = function (blobId) {
823
  return pull(
824
    this.headMsgIds
825
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
826
      : this.server.sbot.links({
827
        dest: blobId,
828
        rel: 'mentions',
829
        values: true,
830
      }),
831
    // decryption could be done here
832
    pull.map(function (msg) {
833
      var c = msg.value && msg.value.content
834
      return c && Array.isArray(c.mentions) && c.mentions || []
835
    }),
836
    pull.flatten(),
837
    pull.filter(function (link) {
838
      return link && link.link === blobId
839
    })
840
  )
841
}
842
843
Req.prototype.servePkg = function (pathname) {
844
  var self = this
845
  var parts = pathname.split('/')
846
  var pkgName = parts.shift().replace(/%2f/i, '/')
847
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
848
  var spec = parts.shift()
849
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
850
  if (parts.length > 0) return this.respondError(404)
851
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
852
  var obj = {
853
    _id: pkgName,
854
    name: pkgName,
855
    'dist-tags': {},
856
    versions: {},
857
    time: {}
858
  }
859
  var distTags = {/* <tag>: {version, ts}*/}
860
  pull(
861
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
862
    pull.drain(function (mention) {
863
      var data = decodeName(mention.name)
864
      if (!data.version) return
865
      if (data.distTag) {
866
        var tag = distTags[data.distTag]
867
        if (!tag || mention.ts > tag.ts) {
868
          /* TODO: sort by causal order instead of only by timestamps */
869
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
870
        }
871
      }
872
      obj.versions[data.version] = {
873
        author: {
874
          url: mention.author
875
        },
876
        name: pkgName,
877
        version: data.version,
878
        dist: self.blobDist(mention.link)
879
      }
880
      var ts = new Date(mention.ts)
881
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
882
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
883
      obj.time[data.version] = ts.toISOString()
884
    }, function (err) {
885
      if (err) return self.respondError(500, err.stack || err)
886
      for (var tag in distTags) {
887
        obj['dist-tags'][tag] = distTags[tag].version
888
      }
889
      if (spec) resolveSpec()
890
      else if (self.fetchAll) resolveAll()
891
      else resolved()
892
    })
893
  )
894
  function resolveSpec() {
895
    var version = obj['dist-tags'][spec]
896
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
897
    obj = obj.versions[version]
898
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
899
    self.populatePackageJson(obj, function (err, pkg) {
900
      if (err) return resolved(err)
901
      obj = pkg || obj
902
      resolved()
903
    })
904
  }
905
  function resolveVersion(version, cb) {
906
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
907
      if (err) return cb(err)
908
      if (pkg) obj.versions[version] = pkg
909
      if (pkg && pkg.license && !obj.license) obj.license = pkg.license
910
      cb()
911
    })
912
  }
913
  function resolveAll() {
914
    var done = multicb()
915
    for (var version in obj.versions) {
916
      resolveVersion(version, done())
917
    }
918
    done(resolved)
919
  }
920
  function resolved(err) {
921
    if (err) return self.respondError(500, err.stack || err)
922
    self.respond(200, obj)
923
  }
924
}
925
926
Req.prototype.servePrebuild = function (name) {
927
  var self = this
928
  var getMention = self.getMentions('prebuild:' + name)
929
  var blobsByAuthor = {/* <author>: BlobId */}
930
  pull(getMention, pull.drain(function (link) {
931
    blobsByAuthor[link.author] = link.link
932
  }, function (err) {
933
    if (err) return self.respondError(500, err.stack || err)
934
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
935
    var blobId
936
    for (var feed in blobsByAuthor) {
937
      var blob = blobId = blobsByAuthor[feed]
938
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
939
      feeds.push(feed)
940
    }
941
    switch (Object.keys(authorsByLink).length) {
942
      case 0:
943
        return self.respondError(404, 'Not Found')
944
      case 1:
945
        return self.serveBlob(blobId)
946
      default:
947
        return self.respond(300, {choices: authorsByLink})
948
    }
949
  }))
950
}
951
952
var localhosts = {
953
  '::1': true,
954
  '127.0.0.1': true,
955
  '::ffff:127.0.0.1': true,
956
}
957
958
Req.prototype.publishPkg = function (pkgName) {
959
  var self = this
960
  var remoteAddress = self.req.socket.remoteAddress
961
  if (!(remoteAddress in localhosts)) {
962
    return self.respondError(403, 'You may not publish as this user.')
963
  }
964
965
  var chunks = []
966
  self.req.on('data', function (data) {
967
    chunks.push(data)
968
  })
969
  self.req.on('end', function () {
970
    var data
971
    try {
972
      data = JSON.parse(Buffer.concat(chunks))
973
    } catch(e) {
974
      return self.respondError(400, e.stack)
975
    }
976
    return self.publishPkg2(pkgName, data || {})
977
  })
978
}
979
980
Req.prototype.publishPkg2 = function (name, data) {
981
  var self = this
982
  if (data.users) console.trace('[npm-registry] users property is not supported')
983
  var attachments = data._attachments || {}
984
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
985
  var done = multicb()
986
  function addAttachmentAsBlob(filename, cb) {
987
    var data = attachments[filename].data
988
    var tarball = new Buffer(data, 'base64')
989
    var length = attachments[filename].length
990
    if (length && length !== tarball.length) return self.respondError(400,
991
      'Length mismatch for attachment \'' + filename + '\'')
992
    self.server.sbot.blobs.add(function (err, id) {
993
      if (err) return cb(err)
994
      self.blobsToPush.push(id)
995
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
996
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
997
      cb()
998
    })(pull.once(tarball))
999
  }
1000
  for (var filename in attachments) {
1001
    addAttachmentAsBlob(filename, done())
1002
  }
1003
  done(function (err) {
1004
    if (err) return self.respondError(500, err.stack || err)
1005
    try {
1006
      self.publishPkg3(name, data, links)
1007
    } catch(e) {
1008
      self.respondError(500, e.stack || e)
1009
    }
1010
  })
1011
}
1012
1013
Req.prototype.publishPkg3 = function (name, data, links) {
1014
  var self = this
1015
  var versions = data.versions || {}
1016
  var linksByVersion = {/* <version>: link */}
1017
1018
  // associate tarball blobs with versions
1019
  for (var version in versions) {
1020
    var pkg = versions[version]
1021
    if (!pkg) return self.respondError(400, 'Bad package object')
1022
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
1023
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
1024
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
1025
    var m = /\/-\/([^\/]+|@[^\/]+\/[^\/]+)$/.exec(pkg.dist.tarball)
1026
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
1027
    var filename = m[1]
1028
    var link = links[filename]
1029
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
1030
    // TODO?: try to find missing tarball mentioned in other messages
1031
    if (pkg.version && pkg.version !== version)
1032
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
1033
    linksByVersion[version] = link
1034
    link.version = version
1035
    link.license = pkg.license
1036
    link.dependencies = pkg.dependencies || {}
1037
    link.optionalDependencies = pkg.optionalDependencies
1038
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
1039
  }
1040
1041
  // associate blobs with dist-tags
1042
  var tags = data['dist-tags'] || {}
1043
  for (var tag in tags) {
1044
    var version = tags[tag]
1045
    var link = linksByVersion[version]
1046
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
1047
    // TODO?: support setting dist-tag without version,
1048
    // by looking up a tarball blob for the version
1049
    link.tag = tag
1050
  }
1051
1052
  // compute blob links to publish
1053
  var mentions = []
1054
  for (var filename in links) {
1055
    var link = links[filename] || {}
1056
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
1057
    mentions.push({
1058
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
1059
      link: link.link,
1060
      size: link.size,
1061
      shasum: link.shasum,
1062
      license: link.license,
1063
      dependencies: link.dependencies,
1064
      optionalDependencies: link.optionalDependencies,
1065
      bundledDependencies: link.bundledDependencies,
1066
    })
1067
  }
1068
  return self.publishPkgs(mentions)
1069
}
1070
1071
Req.prototype.publishPkgs = function (mentions) {
1072
  var self = this
1073
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
1074
    if (err) return self.respondError(500, err.stack || err)
1075
    self.server.pushBlobs(self.blobsToPush, function (err) {
1076
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
1077
      self.respond(201)
1078
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
1079
    })
1080
  })
1081
}
1082
1083
Req.prototype.populatePackageJson = function (obj, cb) {
1084
  var self = this
1085
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1086
  var deps, depsWithOptionalDeps, bundledDeps, shasum, license
1087
1088
  // Combine metadata from mentions of the blob, to construct most accurate
1089
  // metadata, including dependencies info. If needed, fall back to fetching
1090
  // metadata from the tarball blob.
1091
1092
  pull(
1093
    self.getMentionLinks(blobId),
1094
    pull.drain(function (link) {
1095
      if (link.dependencies) deps = link.dependencies
1096
      // if optionalDependencies is present, let the dependencies value
1097
      // override that from other links. because old versions that didn't
1098
      // publish optionalDependencies are missing optionalDependencies from
1099
      // their dependencies.
1100
      if (link.optionalDependencies) depsWithOptionalDeps = {
1101
        deps: link.dependencies,
1102
        optionalDeps: link.optionalDependencies
1103
      }
1104
      if (link.shasum) shasum = link.shasum
1105
      if (link.license) license = link.license
1106
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1107
      // how to handle multiple assignments of dependencies to a package?
1108
    }, function (err) {
1109
      if (err) return cb(new Error(err.stack || err))
1110
      if (deps && (shasum || !self.needShasum)) {
1111
        // assume that the dependencies in the links to the blob are
1112
        // correct.
1113
        if (depsWithOptionalDeps) {
1114
          obj.dependencies = depsWithOptionalDeps.deps || deps
1115
          obj.optionalDependencies = depsWithOptionalDeps.optionalDeps
1116
        } else {
1117
          obj.dependencies = deps
1118
        }
1119
        obj.bundledDependencies = bundledDeps
1120
        obj.license = license
1121
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1122
        next(obj)
1123
      } else {
1124
        // get dependencies from the tarball
1125
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1126
          if (err) return cb(err)
1127
          pkg.dist = obj.dist
1128
          pkg.dist.shasum = pkg._shasum
1129
          pkg.author = pkg.author || obj.author
1130
          pkg.version = pkg.version || obj.version
1131
          pkg.name = pkg.name || obj.name
1132
          pkg.license = pkg.license || obj.license
1133
          next(pkg)
1134
        })
1135
      }
1136
    })
1137
  )
1138
  function next(pkg) {
1139
    var feedId = obj.author.url
1140
    self.server.getFeedName(feedId, function (err, name) {
1141
      if (err) console.error('[npm-registry]', err.stack || err), name = ''
1142
      pkg._npmUser = {
1143
        email: feedId,
1144
        name: name
1145
      }
1146
      cb(null, pkg)
1147
    })
1148
  }
1149
}
1150
1151
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1152
  var self = this
1153
  getBlob(sbot, id, function (err, readBlob) {
1154
    if (err) return cb(err)
1155
    cb = once(cb)
1156
    var extract = tar.extract()
1157
    var pkg, shasum
1158
    extract.on('entry', function (header, stream, next) {
1159
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1160
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1161
          if (err) return cb(err)
1162
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1163
          catch(e) { return cb(e) }
1164
          next()
1165
        }))
1166
      } else {
1167
        stream.on('end', next)
1168
        stream.resume()
1169
      }
1170
    })
1171
    extract.on('finish', function () {
1172
      pkg._shasum = shasum
1173
      cb(null, pkg)
1174
    })
1175
    pull(
1176
      readBlob,
1177
      hash('sha1', 'hex', function (err, sum) {
1178
        if (err) return cb(err)
1179
        shasum = sum
1180
      }),
1181
      toPull(zlib.createGunzip()),
1182
      toPull(extract)
1183
    )
1184
  })
1185
}
1186