git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || 'localhost'
90
  var autoAuth = conf.autoAuth !== false
91
  var listenUrl
92
93
  var server = http.createServer(exports.respond(sbot, config))
94
  var getAddress = onceify(function (cb) {
95
    server.on('error', cb)
96
    server.listen(port, host, function () {
97
      server.removeListener('error', cb)
98
      var addr = this.address()
99
      var listenHost = addr.address
100
      var regHost = listenHost === '::' ? '::1' :
101
                    listenHost === '0.0.0.0' ? '127.0.0.1' :
102
                    listenHost
103
      var regUrl = 'http://' + formatHost(regHost) + ':' + addr.port
104
      listenUrl = 'http://' + formatHost(listenHost) + ':' + addr.port
105
      if (autoAuth) npmLogin(regUrl, next)
106
      else next()
107
      function next(err) {
108
        cb(err, regUrl)
109
      }
110
    })
111
    sbot.on('close', function () {
112
      server.close()
113
    })
114
  })
115
116
  /* getAddress called by local RPC is used to discover the local
117
   * ssb-npm-registry address that can be given to local npm clients. However,
118
   * when running the server we output the address the server is listening on,
119
   * to avoid misleading situations like saying listening on localhost but
120
   * actually listening on a wildcard address. */
121
  getAddress(function (err) {
122
    if (err) return console.error('[npm-registry]', err.stack || err)
123
    console.log('[npm-registry] Listening on ' + listenUrl)
124
  })
125
126
  return {
127
    getAddress: getAddress
128
  }
129
}
130
131
exports.respond = function (sbot, config) {
132
  var reg = new SsbNpmRegistryServer(sbot, config)
133
  return function (req, res) {
134
    new Req(reg, req, res).serve()
135
  }
136
}
137
138
function publishMsg(sbot, value, cb) {
139
  var gotExpectedPrevious = false
140
  sbot.publish(value, function next(err, msg) {
141
    if (err && /^expected previous:/.test(err.message)) {
142
      // retry once on this error
143
      if (gotExpectedPrevious) return cb(err)
144
      gotExpectedPrevious = true
145
      return sbot.publish(value, next)
146
    }
147
    cb(err, msg)
148
  })
149
}
150
151
function getDependencyBranches(sbot, link, cb) {
152
  var id = link.link
153
154
  // get ids of heads of tree of dependencyBranch message links to include all
155
  // the dependencies of the given tarball id.
156
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
157
  var msgs = {}
158
  var branched = {}
159
  var blobs = {}
160
161
  function addPkgById(id, cb) {
162
    if (blobs[id]) return cb()
163
    blobs[id] = true
164
    getPackageJsonCached(id, function (err, pkg) {
165
      if (err) return cb(err)
166
      var done = multicb()
167
      for (var name in pkg.dependencies || {}) {
168
        addPkgBySpec(name, pkg.dependencies[name], done())
169
      }
170
      for (var name in pkg.optionalDependencies || {}) {
171
        addPkgBySpec(name, pkg.optionalDependencies[name], done())
172
      }
173
      done(cb)
174
    })
175
  }
176
177
  function addPkgBySpec(name, spec, cb) {
178
    var versions = {}
179
    var distTags = {}
180
    pull(
181
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
182
      pull.filter(mentionMatches(null, name, spec)),
183
      pull.map(function (mention) {
184
        // query to get the messages since links2 does not include the msg id
185
        return sbot.links({
186
          source: mention.author,
187
          dest: mention.link,
188
          rel: 'mentions',
189
          values: true,
190
        })
191
      }),
192
      pull.flatten(),
193
      pull.drain(function (msg) {
194
        var c = msg && msg.value && msg.value.content
195
        if (!c || !Array.isArray(c.mentions)) return
196
        c.mentions.forEach(function (link) {
197
          var data = link && link.name && decodeName(link.name)
198
          if (!data || data.name !== name) return
199
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
200
          if (data.distTag) distTags[data.distTag] = data.version
201
        })
202
      }, function (err) {
203
        if (err) return cb(err)
204
        var version = distTags[spec]
205
          || semver.maxSatisfying(Object.keys(versions), spec)
206
        var item = versions[version]
207
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
208
        addDepBranchMsg(item.msg)
209
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
210
        addPkgById(item.mention.link, cb)
211
      })
212
    )
213
  }
214
215
  function addDepBranchMsg(msg) {
216
    var c = msg && msg.value && msg.value.content
217
    if (!c) return
218
    if (msgs[msg.key]) return
219
    msgs[msg.key] = msg.value
220
    if (Array.isArray(c.dependencyBranch)) {
221
      for (var k = 0; k < c.dependencyBranch.length; k++) {
222
        branched[c.dependencyBranch[k]] = true
223
      }
224
    }
225
    if (Array.isArray(c.versionBranch)) {
226
      for (var k = 0; k < c.versionBranch.length; k++) {
227
        branched[c.versionBranch[k]] = true
228
      }
229
    }
230
  }
231
232
  function addPkgsByName(name, cb) {
233
    var data = decodeName(name)
234
    pull(
235
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
236
      pull.map(function (mention) {
237
        return packageLinks(sbot, mention.author, mention.link, data.name)
238
      }),
239
      pull.flatten(),
240
      pull.drain(addDepBranchMsg, cb)
241
    )
242
  }
243
244
  var done = multicb()
245
  addPkgsByName(link.name, done())
246
  addPkgById(id, done())
247
  done(function (err) {
248
    if (err) return cb(err)
249
    var ids = []
250
    for (var key in msgs) {
251
      if (!branched[key]) ids.push(key)
252
    }
253
    cb(null, ids)
254
  })
255
}
256
257
function packageLinks(sbot, feed, id, name, spec) {
258
  var matches = mentionMatches(id, name, spec)
259
  return pull(
260
    sbot.links({
261
      source: feed,
262
      dest: id,
263
      rel: 'mentions',
264
      values: true,
265
    }),
266
    pull.filter(function (msg) {
267
      var c = msg && msg.value && msg.value.content
268
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
269
    })
270
  )
271
}
272
273
// For each dependency that is not a bundledDependency, get message ids for
274
// that dependency name + version.
275
function publishSingleMention(sbot, mention, cb) {
276
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
277
  // Calculate dependencyBranch message ids.
278
  var value = {
279
    type: 'npm-packages',
280
    mentions: [mention]
281
  }
282
  var done = multicb({pluck: 1, spread: true})
283
  getDependencyBranches(sbot, mention, function (err, branches) {
284
    if (err) return cb(err)
285
    value.dependencyBranch = branches || undefined
286
    publishMsg(sbot, value, cb)
287
  })
288
}
289
290
function publishMentions(sbot, mentions, cb) {
291
  // console.error("publishing %s mentions", mentions.length)
292
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
293
  // if it is just one mention, fetch and add useful metadata
294
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
295
  publishMsg(sbot, {
296
    type: 'npm-packages',
297
    mentions: mentions,
298
  }, cb)
299
}
300
301
exports.publishPkgMentions = function (sbot, mentions, cb) {
302
  // try to fit the mentions into as few messages as possible,
303
  // while fitting under the message size limit.
304
  var msgs = []
305
  ;(function next(i, chunks) {
306
    if (i >= mentions.length) return cb(null, msgs)
307
    var chunkLen = Math.ceil(mentions.length / chunks)
308
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
309
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
310
      if (err && msgs.length) return onPartialPublish(err)
311
      if (err) return cb(err)
312
      msgs.push(msg)
313
      next(i + chunkLen, chunks)
314
    })
315
  })(0, 1)
316
  function onPartialPublish(err) {
317
    var remaining = mentions.length - i
318
    return cb(new Error('Published messages ' +
319
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
320
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
321
  }
322
}
323
324
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
325
  cb = once(cb)
326
  var waiting = 0
327
  var expandedMentions = mentions.map(function (link) {
328
    var id = link && link.link
329
    if (!id) return link
330
    waiting++
331
    var newLink = {}
332
    for (var k in link) newLink[k] = link[k]
333
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
334
      if (err) return cb(err)
335
      for (var k in props) newLink[k] = pkg[k]
336
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
337
      if (!--waiting) next()
338
    })
339
    return newLink
340
  })
341
  if (!waiting) next()
342
  function next() {
343
    cb(null, expandedMentions)
344
  }
345
}
346
347
function SsbNpmRegistryServer(sbot, config) {
348
  this.sbot = sbot
349
  this.config = config
350
  this.npmConfig = config.npm || {}
351
  this.host = this.npmConfig.host || 'localhost'
352
  this.fetchAll = this.npmConfig.fetchAll
353
  this.needShasum = this.npmConfig.needShasum
354
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
355
  this.getFeedName = memo({cache: lru(100)}, this.getFeedName)
356
  if (sbot.ws) {
357
    var wsPort = config.ws && Number(config.ws.port) || '8989'
358
    this.wsUrl = 'http://localhost:' + wsPort
359
  }
360
}
361
362
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
363
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
364
365
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
366
  var self = this
367
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
368
  ;(function next(i) {
369
    if (i >= ids.length) return cb()
370
    self.sbot.blobs.push(ids[i], function (err) {
371
      if (err) return cb(err)
372
      next(i+1)
373
    })
374
  })(0)
375
}
376
377
function getBlob(sbot, id, cb) {
378
  var blobs = sbot.blobs
379
  blobs.size(id, function (err, size) {
380
    if (err && err.code !== 'ENOENT') return cb(err)
381
    if (typeof size === 'number') cb(null, blobs.get(id), size)
382
    else blobs.want(id, function (err, got) {
383
      if (err) cb(err)
384
      else if (!got) cb('missing blob ' + id)
385
      else blobs.size(id, function (err, size) {
386
        if (err) return cb(err)
387
        cb(null, blobs.get(id), size)
388
      })
389
    })
390
  })
391
}
392
393
function getMentions(links2, name) {
394
  return links2.read({
395
    query: [
396
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
397
      {$filter: {dest: {$prefix: '&'}}},
398
      {$map: {
399
        name: ['rel', 1],
400
        size: ['rel', 2],
401
        link: 'dest',
402
        author: 'source',
403
        ts: 'ts'
404
      }}
405
    ]
406
  })
407
}
408
409
function mentionMatches(id, name, spec) {
410
  return function (mention) {
411
    var data = mention
412
      && (!id || id === mention.link)
413
      && mention.name
414
      && decodeName(mention.name)
415
    return data
416
      && data.name === name
417
      && (spec ? semver.satisfies(data.version, spec) : true)
418
  }
419
}
420
421
SsbNpmRegistryServer.prototype.getMentions = function (name) {
422
  if (!this.sbot.links2) return pull.empty()
423
  return getMentions(this.sbot.links2, name)
424
}
425
426
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
427
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
428
  else this.sbot.get(id, function (err, value) {
429
    if (err) return cb(err)
430
    cb(null, {key: id, value: value})
431
  })
432
}
433
434
SsbNpmRegistryServer.prototype.getFeedName = function (id, cb) {
435
  var self = this
436
  if (self.sbot.names && self.sbot.names.getSignifier) {
437
    self.sbot.names.getSignifier(id, function (err, name) {
438
      if (err || !name) tryLinks()
439
      else cb(null, name)
440
    })
441
  } else {
442
    tryLinks()
443
  }
444
  function tryLinks() {
445
    if (!self.sbot.links) return nope()
446
    pull(
447
      self.sbot.links({
448
        source: id,
449
        dest: id,
450
        rel: 'about',
451
        values: true,
452
        limit: 1,
453
        reverse: true,
454
        keys: false,
455
        meta: false
456
      }),
457
      pull.map(function (value) {
458
        return value && value.content && value.content.name
459
      }),
460
      pull.filter(Boolean),
461
      pull.take(1),
462
      pull.collect(function (err, names) {
463
        if (err || !names.length) return nope()
464
        cb(null, names[0])
465
      })
466
    )
467
  }
468
  function nope() {
469
    cb(null, '')
470
  }
471
}
472
473
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
474
  var self = this
475
  var stack = heads.slice()
476
  var seen = {}
477
  return function (abort, cb) {
478
    if (abort) return cb(abort)
479
    for (var id; stack.length && seen[id = stack.pop()];);
480
    if (!id) return cb(true)
481
    seen[id] = true
482
    // TODO: use DFS
483
    self.getMsg(id, function (err, msg) {
484
      if (err) return cb(new Error(err.stack || err))
485
      var c = msg && msg.value && msg.value.content
486
      var links = c && c[prop]
487
      if (Array.isArray(links)) {
488
        stack.push.apply(stack, links)
489
      } else if (links) {
490
        stack.push(links)
491
      }
492
      cb(null, msg)
493
    })
494
  }
495
}
496
497
function Req(server, req, res) {
498
  this.server = server
499
  this.req = req
500
  this.res = res
501
  this.blobsToPush = []
502
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
503
  var ua = this.req.headers['user-agent']
504
  var m = /\bnpm\/([0-9]*)/.exec(ua)
505
  var npmVersion = m && m[1]
506
  this.needShasum = server.needShasum != null ? server.needShasum :
507
    (npmVersion && npmVersion < 5)
508
  this.baseUrl = this.server.npmConfig.baseUrl
509
  if (this.baseUrl) {
510
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
511
  } else {
512
    var hostname = req.headers.host
513
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
514
    this.baseUrl = 'http://' + hostname
515
  }
516
517
  // prefer serving blobs from ssb-ws, for consistency of tarball URLs.
518
  this.blobBaseUrl = this.server.wsUrl && this.isLocal()
519
    ? this.server.wsUrl + '/blobs/get/'
520
    : this.baseUrl + '/-/blobs/get/'
521
}
522
523
Req.prototype.isLocal = function () {
524
  var remoteAddr = this.req.socket.remoteAddress
525
  return remoteAddr === '::1'
526
    || remoteAddr === '::ffff:127.0.0.1'
527
    || remoteAddr === '127.0.0.1'
528
}
529
530
Req.prototype.serve = function () {
531
  if (process.env.DEBUG) {
532
    console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
533
  }
534
  this.res.setTimeout(0)
535
  var pathname = this.req.url.replace(/\?.*/, '')
536
  var m
537
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
538
    try {
539
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
540
      this.headMsgPlus = !!m[1]
541
      // ^ means also include packages published after the head message id
542
    } catch(e) {
543
      return this.respondError(400, e.stack || e)
544
    }
545
    pathname = m[3]
546
  }
547
  if (pathname === '/') return this.serveHome()
548
  if (pathname === '/robots.txt') return this.serveRobots()
549
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
550
  if (pathname === '/-/whoami') return this.serveWhoami()
551
  if (pathname === '/-/ping') return this.respond(200, true)
552
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
553
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
554
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
555
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
556
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
557
  return this.respond(404, {error: 'Not found'})
558
}
559
560
Req.prototype.respond = function (status, message) {
561
  this.res.writeHead(status, {'content-type': 'application/json'})
562
  this.res.end(message && JSON.stringify(message, 0, 2))
563
}
564
565
Req.prototype.respondError = function (status, message) {
566
  this.respond(status, {error: message})
567
}
568
569
Req.prototype.respondErrorStr = function (status, err) {
570
  this.res.writeHead(status, {'content-type': 'text/plain'})
571
  this.res.end(err.stack || err)
572
}
573
574
Req.prototype.respondRaw = function (status, body) {
575
  this.res.writeHead(status)
576
  this.res.end(body)
577
}
578
579
Req.prototype.serveHome = function () {
580
  var self = this
581
  self.res.writeHead(200, {'content-type': 'text/html'})
582
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
583
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
584
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
585
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
586
    '</body></html>')
587
}
588
589
Req.prototype.blobDist = function (id) {
590
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
591
  if (!m) throw new Error('bad blob id: ' + id)
592
  return {
593
    integrity: m[2] + '-' + m[1],
594
    tarball: this.blobBaseUrl + id
595
  }
596
}
597
598
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
599
  var self = this
600
  pull(
601
    self.server.sbot.links({
602
      source: feedId,
603
      dest: blobId,
604
      rel: 'mentions',
605
      values: true,
606
    }),
607
    pull.filter(function (msg) {
608
      var c = msg && msg.value && msg.value.content
609
      return c.type === 'npm-packages'
610
    }),
611
    pull.collect(function (err, msgs) {
612
      if (err) return cb(err)
613
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
614
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
615
      // TODO: make a smarter decision about which message id to use
616
      cb(null, msgs.pop().key)
617
    })
618
  )
619
}
620
621
Req.prototype.resolvePkg = function (pkgSpec, cb) {
622
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
623
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
624
  var self = this
625
  var pkgName = m[1]
626
  var spec = m[2] || '*'
627
  var versions = {}
628
  var distTags = {}
629
  pull(
630
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
631
    pull.drain(function (mention) {
632
      var data = decodeName(mention.name)
633
      if (!data.version) return
634
      if (data.distTag) {
635
        distTags[data.distTag] = data.version
636
      }
637
      versions[data.version] = {
638
        author: mention.author,
639
        name: pkgName,
640
        version: data.version,
641
        blobId: mention.link
642
      }
643
    }, function (err) {
644
      if (err) return cb(err)
645
      var version = distTags[spec]
646
        || semver.maxSatisfying(Object.keys(versions), spec)
647
      var item = versions[version]
648
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
649
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
650
        if (err) return cb(err)
651
        item.msgId = id
652
        cb(null, item)
653
      })
654
    })
655
  )
656
}
657
658
Req.prototype.resolvePkgs = function (specs, cb) {
659
  var done = multicb({pluck: 1})
660
  var self = this
661
  specs.forEach(function (spec) {
662
    self.resolvePkg(spec, done())
663
  })
664
  done(cb)
665
}
666
667
Req.prototype.serveBootstrap = function () {
668
  var self = this
669
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
670
  var postInstallCmd = self.server.npmConfig.postInstallCmd
671
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
672
  var ssbNpmRegistryName = require('./package.json').name
673
  var ssbNpmRegistryVersion = require('./package.json').version
674
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
675
  var done = multicb({pluck: 1, spread: true})
676
  self.resolvePkg(ssbNpmRegistrySpec, done())
677
  self.resolvePkgs(pkgs, done())
678
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
679
    if (err) return self.respondErrorStr(500, err.stack || err)
680
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
681
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
682
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
683
    var ssbNpmRegistryNameVersion = ssbNpmRegistryPkgInfo.name + '-' + ssbNpmRegistryPkgInfo.version
684
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
685
    var globalPkgs = pkgsInfo.map(function (info) {
686
      return info.name + '@' + info.version
687
    }).join(' ')
688
    var npmCmd = 'install -g ' + globalPkgs
689
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
690
    var wsUrl = self.baseUrl + '/-'
691
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
692
693
    var script =
694
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
695
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
696
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
697
      'tar xzf package.tgz &&\n' +
698
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
699
        pkgMsgs.map(function (id) {
700
          return '  --branch ' + id + ' \\\n'
701
        }).join('') +
702
        '  -- ' + npmCmd +
703
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
704
705
    self.res.writeHead(200, {'Content-type': 'text/plain'})
706
    self.res.end(script)
707
  })
708
}
709
710
Req.prototype.serveRobots = function () {
711
  this.res.writeHead(200, {'Content-type': 'text/plain'})
712
  this.res.end('User-agent: *\nDisallow: /\n')
713
}
714
715
Req.prototype.serveWhoami = function () {
716
  var self = this
717
  self.server.sbot.whoami(function (err, feed) {
718
    if (err) return self.respondError(500, err.stack || err)
719
    self.respond(200, {username: feed.id})
720
  })
721
}
722
723
Req.prototype.serveUser1 = function () {
724
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
725
}
726
727
Req.prototype.serveBlob = function (id) {
728
  var self = this
729
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
730
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
731
    if (err) {
732
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
733
      else return self.respondErrorStr(500, err.message || err)
734
    }
735
    self.res.writeHead(200, {
736
      'Cache-Control': 'public, max-age=315360000',
737
      'Content-Length': size,
738
      'etag': id
739
    })
740
    pull(
741
      readBlob,
742
      toPull(self.res, function (err) {
743
        if (err) console.error('[npm-registry]', err)
744
      })
745
    )
746
  })
747
}
748
749
Req.prototype.serveMsg = function (id) {
750
  var self = this
751
  try { id = decodeURIComponent(id) }
752
  catch (e) {}
753
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
754
  self.server.sbot.get(id, function (err, value) {
755
    if (err) return self.respondError(500, err.message || err)
756
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
757
    self.res.writeHead(200, {
758
      'Content-Type': 'application/json',
759
      'Cache-Control': 'public, max-age=315360000',
760
      'Content-Length': out.length,
761
      'etag': id
762
    })
763
    self.res.end(out)
764
  })
765
}
766
767
function decodeName(name) {
768
  var parts = String(name).replace(/\.tgz$/, '').split(':')
769
  return {
770
    name: parts[1],
771
    version: parts[2],
772
    distTag: parts[3],
773
  }
774
}
775
776
Req.prototype.getMsgMentions = function (name) {
777
  return pull(
778
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
779
    // decryption could be done here
780
    pull.map(function (msg) {
781
      var c = msg.value && msg.value.content
782
      if (!c.mentions || !Array.isArray(c.mentions)) return []
783
      return c.mentions.map(function (mention) {
784
        return {
785
          name: mention.name,
786
          size: mention.size,
787
          link: mention.link,
788
          author: msg.value.author,
789
          ts: msg.value.timestamp,
790
        }
791
      })
792
    }),
793
    pull.flatten(),
794
    pull.filter(typeof name === 'string' ? function (link) {
795
      return link.name === name
796
    } : name && name.$prefix ? function (link) {
797
      return link.name.substr(0, name.$prefix.length) === name.$prefix
798
    } : function () {
799
      throw new TypeError('unsupported name filter')
800
    })
801
  )
802
}
803
804
Req.prototype.getMentions = function (name) {
805
  var useMsgMentions = this.headMsgIds
806
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
807
  if (useServerMentions && !this.server.sbot.links2) {
808
    return this.headMsgPlus
809
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
810
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
811
  }
812
  return cat([
813
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
814
    useServerMentions ? this.server.getMentions(name) : pull.empty()
815
  ])
816
}
817
818
Req.prototype.getMentionLinks = function (blobId) {
819
  return pull(
820
    this.headMsgIds
821
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
822
      : this.server.sbot.links({
823
        dest: blobId,
824
        rel: 'mentions',
825
        values: true,
826
      }),
827
    // decryption could be done here
828
    pull.map(function (msg) {
829
      var c = msg.value && msg.value.content
830
      return c && Array.isArray(c.mentions) && c.mentions || []
831
    }),
832
    pull.flatten(),
833
    pull.filter(function (link) {
834
      return link && link.link === blobId
835
    })
836
  )
837
}
838
839
Req.prototype.servePkg = function (pathname) {
840
  var self = this
841
  var parts = pathname.split('/')
842
  var pkgName = parts.shift().replace(/%2f/i, '/')
843
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
844
  var spec = parts.shift()
845
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
846
  if (parts.length > 0) return this.respondError(404)
847
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
848
  var obj = {
849
    _id: pkgName,
850
    name: pkgName,
851
    'dist-tags': {},
852
    versions: {},
853
    time: {}
854
  }
855
  var distTags = {/* <tag>: {version, ts}*/}
856
  pull(
857
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
858
    pull.drain(function (mention) {
859
      var data = decodeName(mention.name)
860
      if (!data.version) return
861
      if (data.distTag) {
862
        var tag = distTags[data.distTag]
863
        if (!tag || mention.ts > tag.ts) {
864
          /* TODO: sort by causal order instead of only by timestamps */
865
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
866
        }
867
      }
868
      obj.versions[data.version] = {
869
        author: {
870
          url: mention.author
871
        },
872
        name: pkgName,
873
        version: data.version,
874
        dist: self.blobDist(mention.link)
875
      }
876
      var ts = new Date(mention.ts)
877
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
878
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
879
      obj.time[data.version] = ts.toISOString()
880
    }, function (err) {
881
      if (err) return self.respondError(500, err.stack || err)
882
      for (var tag in distTags) {
883
        obj['dist-tags'][tag] = distTags[tag].version
884
      }
885
      if (spec) resolveSpec()
886
      else if (self.fetchAll) resolveAll()
887
      else resolved()
888
    })
889
  )
890
  function resolveSpec() {
891
    var version = obj['dist-tags'][spec]
892
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
893
    obj = obj.versions[version]
894
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
895
    self.populatePackageJson(obj, function (err, pkg) {
896
      if (err) return resolved(err)
897
      obj = pkg || obj
898
      resolved()
899
    })
900
  }
901
  function resolveVersion(version, cb) {
902
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
903
      if (err) return cb(err)
904
      if (pkg) obj.versions[version] = pkg
905
      if (pkg && pkg.license && !obj.license) obj.license = pkg.license
906
      cb()
907
    })
908
  }
909
  function resolveAll() {
910
    var done = multicb()
911
    for (var version in obj.versions) {
912
      resolveVersion(version, done())
913
    }
914
    done(resolved)
915
  }
916
  function resolved(err) {
917
    if (err) return self.respondError(500, err.stack || err)
918
    self.respond(200, obj)
919
  }
920
}
921
922
Req.prototype.servePrebuild = function (name) {
923
  var self = this
924
  var getMention = self.getMentions('prebuild:' + name)
925
  var blobsByAuthor = {/* <author>: BlobId */}
926
  pull(getMention, pull.drain(function (link) {
927
    blobsByAuthor[link.author] = link.link
928
  }, function (err) {
929
    if (err) return self.respondError(500, err.stack || err)
930
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
931
    var blobId
932
    for (var feed in blobsByAuthor) {
933
      var blob = blobId = blobsByAuthor[feed]
934
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
935
      feeds.push(feed)
936
    }
937
    switch (Object.keys(authorsByLink).length) {
938
      case 0:
939
        return self.respondError(404, 'Not Found')
940
      case 1:
941
        return self.serveBlob(blobId)
942
      default:
943
        return self.respond(300, {choices: authorsByLink})
944
    }
945
  }))
946
}
947
948
var localhosts = {
949
  '::1': true,
950
  '127.0.0.1': true,
951
  '::ffff:127.0.0.1': true,
952
}
953
954
Req.prototype.publishPkg = function (pkgName) {
955
  var self = this
956
  var remoteAddress = self.req.socket.remoteAddress
957
  if (!(remoteAddress in localhosts)) {
958
    return self.respondError(403, 'You may not publish as this user.')
959
  }
960
961
  var chunks = []
962
  self.req.on('data', function (data) {
963
    chunks.push(data)
964
  })
965
  self.req.on('end', function () {
966
    var data
967
    try {
968
      data = JSON.parse(Buffer.concat(chunks))
969
    } catch(e) {
970
      return self.respondError(400, e.stack)
971
    }
972
    return self.publishPkg2(pkgName, data || {})
973
  })
974
}
975
976
Req.prototype.publishPkg2 = function (name, data) {
977
  var self = this
978
  if (data.users) console.trace('[npm-registry] users property is not supported')
979
  var attachments = data._attachments || {}
980
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
981
  var done = multicb()
982
  function addAttachmentAsBlob(filename, cb) {
983
    var data = attachments[filename].data
984
    var tarball = new Buffer(data, 'base64')
985
    var length = attachments[filename].length
986
    if (length && length !== tarball.length) return self.respondError(400,
987
      'Length mismatch for attachment \'' + filename + '\'')
988
    self.server.sbot.blobs.add(function (err, id) {
989
      if (err) return cb(err)
990
      self.blobsToPush.push(id)
991
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
992
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
993
      cb()
994
    })(pull.once(tarball))
995
  }
996
  for (var filename in attachments) {
997
    addAttachmentAsBlob(filename, done())
998
  }
999
  done(function (err) {
1000
    if (err) return self.respondError(500, err.stack || err)
1001
    try {
1002
      self.publishPkg3(name, data, links)
1003
    } catch(e) {
1004
      self.respondError(500, e.stack || e)
1005
    }
1006
  })
1007
}
1008
1009
Req.prototype.publishPkg3 = function (name, data, links) {
1010
  var self = this
1011
  var versions = data.versions || {}
1012
  var linksByVersion = {/* <version>: link */}
1013
1014
  // associate tarball blobs with versions
1015
  for (var version in versions) {
1016
    var pkg = versions[version]
1017
    if (!pkg) return self.respondError(400, 'Bad package object')
1018
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
1019
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
1020
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
1021
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
1022
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
1023
    var filename = m[1]
1024
    var link = links[filename]
1025
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
1026
    // TODO?: try to find missing tarball mentioned in other messages
1027
    if (pkg.version && pkg.version !== version)
1028
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
1029
    linksByVersion[version] = link
1030
    link.version = version
1031
    link.license = pkg.license
1032
    link.dependencies = pkg.dependencies || {}
1033
    link.optionalDependencies = pkg.optionalDependencies
1034
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
1035
  }
1036
1037
  // associate blobs with dist-tags
1038
  var tags = data['dist-tags'] || {}
1039
  for (var tag in tags) {
1040
    var version = tags[tag]
1041
    var link = linksByVersion[version]
1042
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
1043
    // TODO?: support setting dist-tag without version,
1044
    // by looking up a tarball blob for the version
1045
    link.tag = tag
1046
  }
1047
1048
  // compute blob links to publish
1049
  var mentions = []
1050
  for (var filename in links) {
1051
    var link = links[filename] || {}
1052
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
1053
    mentions.push({
1054
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
1055
      link: link.link,
1056
      size: link.size,
1057
      shasum: link.shasum,
1058
      license: link.license,
1059
      dependencies: link.dependencies,
1060
      optionalDependencies: link.optionalDependencies,
1061
      bundledDependencies: link.bundledDependencies,
1062
    })
1063
  }
1064
  return self.publishPkgs(mentions)
1065
}
1066
1067
Req.prototype.publishPkgs = function (mentions) {
1068
  var self = this
1069
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
1070
    if (err) return self.respondError(500, err.stack || err)
1071
    self.server.pushBlobs(self.blobsToPush, function (err) {
1072
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
1073
      self.respond(201)
1074
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
1075
    })
1076
  })
1077
}
1078
1079
Req.prototype.populatePackageJson = function (obj, cb) {
1080
  var self = this
1081
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1082
  var deps, depsWithOptionalDeps, bundledDeps, shasum, license
1083
1084
  // Combine metadata from mentions of the blob, to construct most accurate
1085
  // metadata, including dependencies info. If needed, fall back to fetching
1086
  // metadata from the tarball blob.
1087
1088
  pull(
1089
    self.getMentionLinks(blobId),
1090
    pull.drain(function (link) {
1091
      if (link.dependencies) deps = link.dependencies
1092
      // if optionalDependencies is present, let the dependencies value
1093
      // override that from other links. because old versions that didn't
1094
      // publish optionalDependencies are missing optionalDependencies from
1095
      // their dependencies.
1096
      if (link.optionalDependencies) depsWithOptionalDeps = {
1097
        deps: link.dependencies,
1098
        optionalDeps: link.optionalDependencies
1099
      }
1100
      if (link.shasum) shasum = link.shasum
1101
      if (link.license) license = link.license
1102
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1103
      // how to handle multiple assignments of dependencies to a package?
1104
    }, function (err) {
1105
      if (err) return cb(new Error(err.stack || err))
1106
      if (deps && (shasum || !self.needShasum)) {
1107
        // assume that the dependencies in the links to the blob are
1108
        // correct.
1109
        if (depsWithOptionalDeps) {
1110
          obj.dependencies = depsWithOptionalDeps.deps || deps
1111
          obj.optionalDependencies = depsWithOptionalDeps.optionalDeps
1112
        } else {
1113
          obj.dependencies = deps
1114
        }
1115
        obj.bundledDependencies = bundledDeps
1116
        obj.license = license
1117
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1118
        next(obj)
1119
      } else {
1120
        // get dependencies from the tarball
1121
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1122
          if (err) return cb(err)
1123
          pkg.dist = obj.dist
1124
          pkg.dist.shasum = pkg._shasum
1125
          pkg.author = pkg.author || obj.author
1126
          pkg.version = pkg.version || obj.version
1127
          pkg.name = pkg.name || obj.name
1128
          pkg.license = pkg.license || obj.license
1129
          next(pkg)
1130
        })
1131
      }
1132
    })
1133
  )
1134
  function next(pkg) {
1135
    var feedId = obj.author.url
1136
    self.server.getFeedName(feedId, function (err, name) {
1137
      if (err) console.error('[npm-registry]', err.stack || err), name = ''
1138
      pkg._npmUser = {
1139
        email: feedId,
1140
        name: name
1141
      }
1142
      cb(null, pkg)
1143
    })
1144
  }
1145
}
1146
1147
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1148
  var self = this
1149
  getBlob(sbot, id, function (err, readBlob) {
1150
    if (err) return cb(err)
1151
    cb = once(cb)
1152
    var extract = tar.extract()
1153
    var pkg, shasum
1154
    extract.on('entry', function (header, stream, next) {
1155
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1156
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1157
          if (err) return cb(err)
1158
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1159
          catch(e) { return cb(e) }
1160
          next()
1161
        }))
1162
      } else {
1163
        stream.on('end', next)
1164
        stream.resume()
1165
      }
1166
    })
1167
    extract.on('finish', function () {
1168
      pkg._shasum = shasum
1169
      cb(null, pkg)
1170
    })
1171
    pull(
1172
      readBlob,
1173
      hash('sha1', 'hex', function (err, sum) {
1174
        if (err) return cb(err)
1175
        shasum = sum
1176
      }),
1177
      toPull(zlib.createGunzip()),
1178
      toPull(extract)
1179
    )
1180
  })
1181
}
1182