git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || 'localhost'
90
  var autoAuth = conf.autoAuth !== false
91
  var listenUrl
92
93
  var server = http.createServer(exports.respond(sbot, config))
94
  var getAddress = onceify(function (cb) {
95
    server.on('error', cb)
96
    server.listen(port, host, function () {
97
      server.removeListener('error', cb)
98
      var addr = this.address()
99
      var listenHost = addr.address
100
      var regHost = listenHost === '::' ? '::1' :
101
                    listenHost === '0.0.0.0' ? '127.0.0.1' :
102
                    listenHost
103
      var regUrl = 'http://' + formatHost(regHost) + ':' + addr.port
104
      listenUrl = 'http://' + formatHost(listenHost) + ':' + addr.port
105
      if (autoAuth) npmLogin(regUrl, next)
106
      else next()
107
      function next(err) {
108
        cb(err, regUrl)
109
      }
110
    })
111
    sbot.on('close', function () {
112
      server.close()
113
    })
114
  })
115
116
  /* getAddress called by local RPC is used to discover the local
117
   * ssb-npm-registry address that can be given to local npm clients. However,
118
   * when running the server we output the address the server is listening on,
119
   * to avoid misleading situations like saying listening on localhost but
120
   * actually listening on a wildcard address. */
121
  getAddress(function (err) {
122
    if (err) return console.error('[npm-registry]', err.stack || err)
123
    console.log('[npm-registry] Listening on ' + listenUrl)
124
  })
125
126
  return {
127
    getAddress: getAddress
128
  }
129
}
130
131
exports.respond = function (sbot, config) {
132
  var reg = new SsbNpmRegistryServer(sbot, config)
133
  return function (req, res) {
134
    new Req(reg, req, res).serve()
135
  }
136
}
137
138
function publishMsg(sbot, value, cb) {
139
  var gotExpectedPrevious = false
140
  sbot.publish(value, function next(err, msg) {
141
    if (err && /^expected previous:/.test(err.message)) {
142
      // retry once on this error
143
      if (gotExpectedPrevious) return cb(err)
144
      gotExpectedPrevious = true
145
      return sbot.publish(value, next)
146
    }
147
    cb(err, msg)
148
  })
149
}
150
151
function getDependencyBranches(sbot, link, cb) {
152
  var id = link.link
153
154
  // get ids of heads of tree of dependencyBranch message links to include all
155
  // the dependencies of the given tarball id.
156
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
157
  var msgs = {}
158
  var branched = {}
159
  var blobs = {}
160
161
  function addPkgById(id, cb) {
162
    if (blobs[id]) return cb()
163
    blobs[id] = true
164
    getPackageJsonCached(id, function (err, pkg) {
165
      if (err) return cb(err)
166
      var done = multicb()
167
      for (var name in pkg.dependencies || {}) {
168
        addPkgBySpec(name, pkg.dependencies[name], done())
169
      }
170
      done(cb)
171
    })
172
  }
173
174
  function addPkgBySpec(name, spec, cb) {
175
    var versions = {}
176
    var distTags = {}
177
    pull(
178
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
179
      pull.filter(mentionMatches(null, name, spec)),
180
      pull.map(function (mention) {
181
        // query to get the messages since links2 does not include the msg id
182
        return sbot.links({
183
          source: mention.author,
184
          dest: mention.link,
185
          rel: 'mentions',
186
          values: true,
187
        })
188
      }),
189
      pull.flatten(),
190
      pull.drain(function (msg) {
191
        var c = msg && msg.value && msg.value.content
192
        if (!c || !Array.isArray(c.mentions)) return
193
        c.mentions.forEach(function (link) {
194
          var data = link && link.name && decodeName(link.name)
195
          if (!data || data.name !== name) return
196
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
197
          if (data.distTag) distTags[data.distTag] = data.version
198
        })
199
      }, function (err) {
200
        if (err) return cb(err)
201
        var version = distTags[spec]
202
          || semver.maxSatisfying(Object.keys(versions), spec)
203
        var item = versions[version]
204
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
205
        addDepBranchMsg(item.msg)
206
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
207
        addPkgById(item.mention.link, cb)
208
      })
209
    )
210
  }
211
212
  function addDepBranchMsg(msg) {
213
    var c = msg && msg.value && msg.value.content
214
    if (!c) return
215
    if (msgs[msg.key]) return
216
    msgs[msg.key] = msg.value
217
    if (Array.isArray(c.dependencyBranch)) {
218
      for (var k = 0; k < c.dependencyBranch.length; k++) {
219
        branched[c.dependencyBranch[k]] = true
220
      }
221
    }
222
    if (Array.isArray(c.versionBranch)) {
223
      for (var k = 0; k < c.versionBranch.length; k++) {
224
        branched[c.versionBranch[k]] = true
225
      }
226
    }
227
  }
228
229
  function addPkgsByName(name, cb) {
230
    var data = decodeName(name)
231
    pull(
232
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
233
      pull.map(function (mention) {
234
        return packageLinks(sbot, mention.author, mention.link, data.name)
235
      }),
236
      pull.flatten(),
237
      pull.drain(addDepBranchMsg, cb)
238
    )
239
  }
240
241
  var done = multicb()
242
  addPkgsByName(link.name, done())
243
  addPkgById(id, done())
244
  done(function (err) {
245
    if (err) return cb(err)
246
    var ids = []
247
    for (var key in msgs) {
248
      if (!branched[key]) ids.push(key)
249
    }
250
    cb(null, ids)
251
  })
252
}
253
254
function packageLinks(sbot, feed, id, name, spec) {
255
  var matches = mentionMatches(id, name, spec)
256
  return pull(
257
    sbot.links({
258
      source: feed,
259
      dest: id,
260
      rel: 'mentions',
261
      values: true,
262
    }),
263
    pull.filter(function (msg) {
264
      var c = msg && msg.value && msg.value.content
265
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
266
    })
267
  )
268
}
269
270
// For each dependency that is not a bundledDependency, get message ids for
271
// that dependency name + version.
272
function publishSingleMention(sbot, mention, cb) {
273
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
274
  // Calculate dependencyBranch message ids.
275
  var value = {
276
    type: 'npm-packages',
277
    mentions: [mention]
278
  }
279
  var done = multicb({pluck: 1, spread: true})
280
  getDependencyBranches(sbot, mention, function (err, branches) {
281
    if (err) return cb(err)
282
    value.dependencyBranch = branches || undefined
283
    publishMsg(sbot, value, cb)
284
  })
285
}
286
287
function publishMentions(sbot, mentions, cb) {
288
  // console.error("publishing %s mentions", mentions.length)
289
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
290
  // if it is just one mention, fetch and add useful metadata
291
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
292
  publishMsg(sbot, {
293
    type: 'npm-packages',
294
    mentions: mentions,
295
  }, cb)
296
}
297
298
exports.publishPkgMentions = function (sbot, mentions, cb) {
299
  // try to fit the mentions into as few messages as possible,
300
  // while fitting under the message size limit.
301
  var msgs = []
302
  ;(function next(i, chunks) {
303
    if (i >= mentions.length) return cb(null, msgs)
304
    var chunkLen = Math.ceil(mentions.length / chunks)
305
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
306
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
307
      if (err && msgs.length) return onPartialPublish(err)
308
      if (err) return cb(err)
309
      msgs.push(msg)
310
      next(i + chunkLen, chunks)
311
    })
312
  })(0, 1)
313
  function onPartialPublish(err) {
314
    var remaining = mentions.length - i
315
    return cb(new Error('Published messages ' +
316
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
317
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
318
  }
319
}
320
321
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
322
  cb = once(cb)
323
  var waiting = 0
324
  var expandedMentions = mentions.map(function (link) {
325
    var id = link && link.link
326
    if (!id) return link
327
    waiting++
328
    var newLink = {}
329
    for (var k in link) newLink[k] = link[k]
330
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
331
      if (err) return cb(err)
332
      for (var k in props) newLink[k] = pkg[k]
333
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
334
      if (!--waiting) next()
335
    })
336
    return newLink
337
  })
338
  if (!waiting) next()
339
  function next() {
340
    cb(null, expandedMentions)
341
  }
342
}
343
344
function SsbNpmRegistryServer(sbot, config) {
345
  this.sbot = sbot
346
  this.config = config
347
  this.npmConfig = config.npm || {}
348
  this.host = this.npmConfig.host || 'localhost'
349
  this.fetchAll = this.npmConfig.fetchAll
350
  this.needShasum = this.npmConfig.needShasum
351
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
352
  this.getFeedName = memo({cache: lru(100)}, this.getFeedName)
353
  if (sbot.ws) {
354
    var wsPort = config.ws && Number(config.ws.port) || '8989'
355
    this.wsUrl = 'http://localhost:' + wsPort
356
  }
357
}
358
359
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
360
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
361
362
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
363
  var self = this
364
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
365
  ;(function next(i) {
366
    if (i >= ids.length) return cb()
367
    self.sbot.blobs.push(ids[i], function (err) {
368
      if (err) return cb(err)
369
      next(i+1)
370
    })
371
  })(0)
372
}
373
374
function getBlob(sbot, id, cb) {
375
  var blobs = sbot.blobs
376
  blobs.size(id, function (err, size) {
377
    if (err) return cb(err)
378
    if (typeof size === 'number') cb(null, blobs.get(id), size)
379
    else blobs.want(id, function (err, got) {
380
      if (err) cb(err)
381
      else if (!got) cb('missing blob ' + id)
382
      else blobs.size(id, function (err, size) {
383
        if (err) return cb(err)
384
        cb(null, blobs.get(id), size)
385
      })
386
    })
387
  })
388
}
389
390
function getMentions(links2, name) {
391
  return links2.read({
392
    query: [
393
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
394
      {$filter: {dest: {$prefix: '&'}}},
395
      {$map: {
396
        name: ['rel', 1],
397
        size: ['rel', 2],
398
        link: 'dest',
399
        author: 'source',
400
        ts: 'ts'
401
      }}
402
    ]
403
  })
404
}
405
406
function mentionMatches(id, name, spec) {
407
  return function (mention) {
408
    var data = mention
409
      && (!id || id === mention.link)
410
      && mention.name
411
      && decodeName(mention.name)
412
    return data
413
      && data.name === name
414
      && (spec ? semver.satisfies(data.version, spec) : true)
415
  }
416
}
417
418
SsbNpmRegistryServer.prototype.getMentions = function (name) {
419
  if (!this.sbot.links2) return pull.empty()
420
  return getMentions(this.sbot.links2, name)
421
}
422
423
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
424
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
425
  else this.sbot.get(id, function (err, value) {
426
    if (err) return cb(err)
427
    cb(null, {key: id, value: value})
428
  })
429
}
430
431
SsbNpmRegistryServer.prototype.getFeedName = function (id, cb) {
432
  var self = this
433
  if (self.sbot.names && self.sbot.names.getSignifier) {
434
    self.sbot.names.getSignifier(id, function (err, name) {
435
      if (err || !name) tryLinks()
436
      else cb(null, name)
437
    })
438
  } else {
439
    tryLinks()
440
  }
441
  function tryLinks() {
442
    if (!self.sbot.links) return nope()
443
    pull(
444
      self.sbot.links({
445
        source: id,
446
        dest: id,
447
        rel: 'about',
448
        values: true,
449
        limit: 1,
450
        reverse: true,
451
        keys: false,
452
        meta: false
453
      }),
454
      pull.map(function (value) {
455
        return value && value.content && value.content.name
456
      }),
457
      pull.filter(Boolean),
458
      pull.take(1),
459
      pull.collect(function (err, names) {
460
        if (err || !names.length) return nope()
461
        cb(null, names[0])
462
      })
463
    )
464
  }
465
  function nope() {
466
    cb(null, '')
467
  }
468
}
469
470
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
471
  var self = this
472
  var stack = heads.slice()
473
  var seen = {}
474
  return function (abort, cb) {
475
    if (abort) return cb(abort)
476
    for (var id; stack.length && seen[id = stack.pop()];);
477
    if (!id) return cb(true)
478
    seen[id] = true
479
    // TODO: use DFS
480
    self.getMsg(id, function (err, msg) {
481
      if (err) return cb(new Error(err.stack || err))
482
      var c = msg && msg.value && msg.value.content
483
      var links = c && c[prop]
484
      if (Array.isArray(links)) {
485
        stack.push.apply(stack, links)
486
      } else if (links) {
487
        stack.push(links)
488
      }
489
      cb(null, msg)
490
    })
491
  }
492
}
493
494
function Req(server, req, res) {
495
  this.server = server
496
  this.req = req
497
  this.res = res
498
  this.blobsToPush = []
499
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
500
  var ua = this.req.headers['user-agent']
501
  var m = /\bnpm\/([0-9]*)/.exec(ua)
502
  var npmVersion = m && m[1]
503
  this.needShasum = server.needShasum != null ? server.needShasum :
504
    (npmVersion && npmVersion < 5)
505
  this.baseUrl = this.server.npmConfig.baseUrl
506
  if (this.baseUrl) {
507
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
508
  } else {
509
    var hostname = req.headers.host
510
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
511
    this.baseUrl = 'http://' + hostname
512
  }
513
514
  // prefer serving blobs from ssb-ws, for consistency of tarball URLs.
515
  this.blobBaseUrl = this.server.wsUrl && this.isLocal()
516
    ? this.server.wsUrl + '/blobs/get/'
517
    : this.baseUrl + '/-/blobs/get/'
518
}
519
520
Req.prototype.isLocal = function () {
521
  var remoteAddr = this.req.socket.remoteAddress
522
  return remoteAddr === '::1'
523
    || remoteAddr === '::ffff:127.0.0.1'
524
    || remoteAddr === '127.0.0.1'
525
}
526
527
Req.prototype.serve = function () {
528
  console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
529
  this.res.setTimeout(0)
530
  var pathname = this.req.url.replace(/\?.*/, '')
531
  var m
532
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
533
    try {
534
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
535
      this.headMsgPlus = !!m[1]
536
      // ^ means also include packages published after the head message id
537
    } catch(e) {
538
      return this.respondError(400, e.stack || e)
539
    }
540
    pathname = m[3]
541
  }
542
  if (pathname === '/') return this.serveHome()
543
  if (pathname === '/robots.txt') return this.serveRobots()
544
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
545
  if (pathname === '/-/whoami') return this.serveWhoami()
546
  if (pathname === '/-/ping') return this.respond(200, true)
547
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
548
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
549
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
550
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
551
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
552
  return this.respond(404, {error: 'Not found'})
553
}
554
555
Req.prototype.respond = function (status, message) {
556
  this.res.writeHead(status, {'content-type': 'application/json'})
557
  this.res.end(message && JSON.stringify(message, 0, 2))
558
}
559
560
Req.prototype.respondError = function (status, message) {
561
  this.respond(status, {error: message})
562
}
563
564
Req.prototype.respondErrorStr = function (status, err) {
565
  this.res.writeHead(status, {'content-type': 'text/plain'})
566
  this.res.end(err.stack || err)
567
}
568
569
Req.prototype.respondRaw = function (status, body) {
570
  this.res.writeHead(status)
571
  this.res.end(body)
572
}
573
574
Req.prototype.serveHome = function () {
575
  var self = this
576
  self.res.writeHead(200, {'content-type': 'text/html'})
577
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
578
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
579
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
580
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
581
    '</body></html>')
582
}
583
584
Req.prototype.blobDist = function (id) {
585
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
586
  if (!m) throw new Error('bad blob id: ' + id)
587
  return {
588
    integrity: m[2] + '-' + m[1],
589
    tarball: this.blobBaseUrl + id
590
  }
591
}
592
593
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
594
  var self = this
595
  pull(
596
    self.server.sbot.links({
597
      source: feedId,
598
      dest: blobId,
599
      rel: 'mentions',
600
      values: true,
601
    }),
602
    pull.filter(function (msg) {
603
      var c = msg && msg.value && msg.value.content
604
      return c.type === 'npm-packages'
605
    }),
606
    pull.collect(function (err, msgs) {
607
      if (err) return cb(err)
608
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
609
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
610
      // TODO: make a smarter decision about which message id to use
611
      cb(null, msgs.pop().key)
612
    })
613
  )
614
}
615
616
Req.prototype.resolvePkg = function (pkgSpec, cb) {
617
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
618
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
619
  var self = this
620
  var pkgName = m[1]
621
  var spec = m[2] || '*'
622
  var versions = {}
623
  var distTags = {}
624
  pull(
625
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
626
    pull.drain(function (mention) {
627
      var data = decodeName(mention.name)
628
      if (!data.version) return
629
      if (data.distTag) {
630
        distTags[data.distTag] = data.version
631
      }
632
      versions[data.version] = {
633
        author: mention.author,
634
        name: pkgName,
635
        version: data.version,
636
        blobId: mention.link
637
      }
638
    }, function (err) {
639
      if (err) return cb(err)
640
      var version = distTags[spec]
641
        || semver.maxSatisfying(Object.keys(versions), spec)
642
      var item = versions[version]
643
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
644
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
645
        if (err) return cb(err)
646
        item.msgId = id
647
        cb(null, item)
648
      })
649
    })
650
  )
651
}
652
653
Req.prototype.resolvePkgs = function (specs, cb) {
654
  var done = multicb({pluck: 1})
655
  var self = this
656
  specs.forEach(function (spec) {
657
    self.resolvePkg(spec, done())
658
  })
659
  done(cb)
660
}
661
662
Req.prototype.serveBootstrap = function () {
663
  var self = this
664
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
665
  var postInstallCmd = self.server.npmConfig.postInstallCmd
666
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
667
  var ssbNpmRegistryName = require('./package.json').name
668
  var ssbNpmRegistryVersion = require('./package.json').version
669
  var ssbNpmRegistryNameVersion = ssbNpmRegistryName + '-' + ssbNpmRegistryVersion
670
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
671
  var done = multicb({pluck: 1, spread: true})
672
  self.resolvePkg(ssbNpmRegistrySpec, done())
673
  self.resolvePkgs(pkgs, done())
674
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
675
    if (err) return self.respondErrorStr(500, err.stack || err)
676
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
677
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
678
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
679
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
680
    var globalPkgs = pkgsInfo.map(function (info) {
681
      return info.name + '@' + info.version
682
    }).join(' ')
683
    var npmCmd = 'install -g ' + globalPkgs
684
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
685
    var wsUrl = self.baseUrl + '/-'
686
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
687
688
    var script =
689
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
690
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
691
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
692
      'tar xzf package.tgz &&\n' +
693
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
694
        pkgMsgs.map(function (id) {
695
          return '  --branch ' + id + ' \\\n'
696
        }).join('') +
697
        '  -- ' + npmCmd +
698
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
699
700
    self.res.writeHead(200, {'Content-type': 'text/plain'})
701
    self.res.end(script)
702
  })
703
}
704
705
Req.prototype.serveRobots = function () {
706
  this.res.writeHead(200, {'Content-type': 'text/plain'})
707
  this.res.end('User-agent: *\nDisallow: /\n')
708
}
709
710
Req.prototype.serveWhoami = function () {
711
  var self = this
712
  self.server.sbot.whoami(function (err, feed) {
713
    if (err) return self.respondError(500, err.stack || err)
714
    self.respond(200, {username: feed.id})
715
  })
716
}
717
718
Req.prototype.serveUser1 = function () {
719
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
720
}
721
722
Req.prototype.serveBlob = function (id) {
723
  var self = this
724
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
725
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
726
    if (err) {
727
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
728
      else return self.respondErrorStr(500, err.message || err)
729
    }
730
    self.res.writeHead(200, {
731
      'Cache-Control': 'public, max-age=315360000',
732
      'Content-Length': size,
733
      'etag': id
734
    })
735
    pull(
736
      readBlob,
737
      toPull(self.res, function (err) {
738
        if (err) console.error('[npm-registry]', err)
739
      })
740
    )
741
  })
742
}
743
744
Req.prototype.serveMsg = function (id) {
745
  var self = this
746
  try { id = decodeURIComponent(id) }
747
  catch (e) {}
748
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
749
  self.server.sbot.get(id, function (err, value) {
750
    if (err) return self.respondError(500, err.message || err)
751
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
752
    self.res.writeHead(200, {
753
      'Content-Type': 'application/json',
754
      'Cache-Control': 'public, max-age=315360000',
755
      'Content-Length': out.length,
756
      'etag': id
757
    })
758
    self.res.end(out)
759
  })
760
}
761
762
function decodeName(name) {
763
  var parts = String(name).replace(/\.tgz$/, '').split(':')
764
  return {
765
    name: parts[1],
766
    version: parts[2],
767
    distTag: parts[3],
768
  }
769
}
770
771
Req.prototype.getMsgMentions = function (name) {
772
  return pull(
773
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
774
    // decryption could be done here
775
    pull.map(function (msg) {
776
      var c = msg.value && msg.value.content
777
      if (!c.mentions || !Array.isArray(c.mentions)) return []
778
      return c.mentions.map(function (mention) {
779
        return {
780
          name: mention.name,
781
          size: mention.size,
782
          link: mention.link,
783
          author: msg.value.author,
784
          ts: msg.value.timestamp,
785
        }
786
      })
787
    }),
788
    pull.flatten(),
789
    pull.filter(typeof name === 'string' ? function (link) {
790
      return link.name === name
791
    } : name && name.$prefix ? function (link) {
792
      return link.name.substr(0, name.$prefix.length) === name.$prefix
793
    } : function () {
794
      throw new TypeError('unsupported name filter')
795
    })
796
  )
797
}
798
799
Req.prototype.getMentions = function (name) {
800
  var useMsgMentions = this.headMsgIds
801
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
802
  if (useServerMentions && !this.server.sbot.links2) {
803
    return this.headMsgPlus
804
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
805
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
806
  }
807
  return cat([
808
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
809
    useServerMentions ? this.server.getMentions(name) : pull.empty()
810
  ])
811
}
812
813
Req.prototype.getMentionLinks = function (blobId) {
814
  return pull(
815
    this.headMsgIds
816
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
817
      : this.server.sbot.links({
818
        dest: blobId,
819
        rel: 'mentions',
820
        values: true,
821
      }),
822
    // decryption could be done here
823
    pull.map(function (msg) {
824
      var c = msg.value && msg.value.content
825
      return c && Array.isArray(c.mentions) && c.mentions || []
826
    }),
827
    pull.flatten(),
828
    pull.filter(function (link) {
829
      return link && link.link === blobId
830
    })
831
  )
832
}
833
834
Req.prototype.servePkg = function (pathname) {
835
  var self = this
836
  var parts = pathname.split('/')
837
  var pkgName = parts.shift().replace(/%2f/i, '/')
838
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
839
  var spec = parts.shift()
840
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
841
  if (parts.length > 0) return this.respondError(404)
842
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
843
  var obj = {
844
    _id: pkgName,
845
    name: pkgName,
846
    'dist-tags': {},
847
    versions: {},
848
    time: {}
849
  }
850
  var distTags = {/* <tag>: {version, ts}*/}
851
  pull(
852
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
853
    pull.drain(function (mention) {
854
      var data = decodeName(mention.name)
855
      if (!data.version) return
856
      if (data.distTag) {
857
        var tag = distTags[data.distTag]
858
        if (!tag || mention.ts > tag.ts) {
859
          /* TODO: sort by causal order instead of only by timestamps */
860
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
861
        }
862
      }
863
      obj.versions[data.version] = {
864
        author: {
865
          url: mention.author
866
        },
867
        name: pkgName,
868
        version: data.version,
869
        dist: self.blobDist(mention.link)
870
      }
871
      var ts = new Date(mention.ts)
872
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
873
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
874
      obj.time[data.version] = ts.toISOString()
875
    }, function (err) {
876
      if (err) return self.respondError(500, err.stack || err)
877
      for (var tag in distTags) {
878
        obj['dist-tags'][tag] = distTags[tag].version
879
      }
880
      if (spec) resolveSpec()
881
      else if (self.fetchAll) resolveAll()
882
      else resolved()
883
    })
884
  )
885
  function resolveSpec() {
886
    var version = obj['dist-tags'][spec]
887
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
888
    obj = obj.versions[version]
889
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
890
    self.populatePackageJson(obj, function (err, pkg) {
891
      if (err) return resolved(err)
892
      obj = pkg || obj
893
      resolved()
894
    })
895
  }
896
  function resolveVersion(version, cb) {
897
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
898
      if (err) return cb(err)
899
      if (pkg) obj.versions[version] = pkg
900
      if (pkg && pkg.license && !obj.license) obj.license = pkg.license
901
      cb()
902
    })
903
  }
904
  function resolveAll() {
905
    var done = multicb()
906
    for (var version in obj.versions) {
907
      resolveVersion(version, done())
908
    }
909
    done(resolved)
910
  }
911
  function resolved(err) {
912
    if (err) return self.respondError(500, err.stack || err)
913
    self.respond(200, obj)
914
  }
915
}
916
917
Req.prototype.servePrebuild = function (name) {
918
  var self = this
919
  var getMention = self.getMentions('prebuild:' + name)
920
  var blobsByAuthor = {/* <author>: BlobId */}
921
  pull(getMention, pull.drain(function (link) {
922
    blobsByAuthor[link.author] = link.link
923
  }, function (err) {
924
    if (err) return self.respondError(500, err.stack || err)
925
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
926
    var blobId
927
    for (var feed in blobsByAuthor) {
928
      var blob = blobId = blobsByAuthor[feed]
929
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
930
      feeds.push(feed)
931
    }
932
    switch (Object.keys(authorsByLink).length) {
933
      case 0:
934
        return self.respondError(404, 'Not Found')
935
      case 1:
936
        return self.serveBlob(blobId)
937
      default:
938
        return self.respond(300, {choices: authorsByLink})
939
    }
940
  }))
941
}
942
943
var localhosts = {
944
  '::1': true,
945
  '127.0.0.1': true,
946
  '::ffff:127.0.0.1': true,
947
}
948
949
Req.prototype.publishPkg = function (pkgName) {
950
  var self = this
951
  var remoteAddress = self.req.socket.remoteAddress
952
  if (!(remoteAddress in localhosts)) {
953
    return self.respondError(403, 'You may not publish as this user.')
954
  }
955
956
  var chunks = []
957
  self.req.on('data', function (data) {
958
    chunks.push(data)
959
  })
960
  self.req.on('end', function () {
961
    var data
962
    try {
963
      data = JSON.parse(Buffer.concat(chunks))
964
    } catch(e) {
965
      return self.respondError(400, e.stack)
966
    }
967
    return self.publishPkg2(pkgName, data || {})
968
  })
969
}
970
971
Req.prototype.publishPkg2 = function (name, data) {
972
  var self = this
973
  if (data.users) console.trace('[npm-registry] users property is not supported')
974
  var attachments = data._attachments || {}
975
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
976
  var done = multicb()
977
  function addAttachmentAsBlob(filename, cb) {
978
    var data = attachments[filename].data
979
    var tarball = new Buffer(data, 'base64')
980
    var length = attachments[filename].length
981
    if (length && length !== tarball.length) return self.respondError(400,
982
      'Length mismatch for attachment \'' + filename + '\'')
983
    self.server.sbot.blobs.add(function (err, id) {
984
      if (err) return cb(err)
985
      self.blobsToPush.push(id)
986
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
987
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
988
      cb()
989
    })(pull.once(tarball))
990
  }
991
  for (var filename in attachments) {
992
    addAttachmentAsBlob(filename, done())
993
  }
994
  done(function (err) {
995
    if (err) return self.respondError(500, err.stack || err)
996
    try {
997
      self.publishPkg3(name, data, links)
998
    } catch(e) {
999
      self.respondError(500, e.stack || e)
1000
    }
1001
  })
1002
}
1003
1004
Req.prototype.publishPkg3 = function (name, data, links) {
1005
  var self = this
1006
  var versions = data.versions || {}
1007
  var linksByVersion = {/* <version>: link */}
1008
1009
  // associate tarball blobs with versions
1010
  for (var version in versions) {
1011
    var pkg = versions[version]
1012
    if (!pkg) return self.respondError(400, 'Bad package object')
1013
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
1014
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
1015
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
1016
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
1017
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
1018
    var filename = m[1]
1019
    var link = links[filename]
1020
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
1021
    // TODO?: try to find missing tarball mentioned in other messages
1022
    if (pkg.version && pkg.version !== version)
1023
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
1024
    linksByVersion[version] = link
1025
    link.version = version
1026
    link.license = pkg.license
1027
    link.dependencies = pkg.dependencies || {}
1028
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
1029
  }
1030
1031
  // associate blobs with dist-tags
1032
  var tags = data['dist-tags'] || {}
1033
  for (var tag in tags) {
1034
    var version = tags[tag]
1035
    var link = linksByVersion[version]
1036
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
1037
    // TODO?: support setting dist-tag without version,
1038
    // by looking up a tarball blob for the version
1039
    link.tag = tag
1040
  }
1041
1042
  // compute blob links to publish
1043
  var mentions = []
1044
  for (var filename in links) {
1045
    var link = links[filename] || {}
1046
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
1047
    mentions.push({
1048
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
1049
      link: link.link,
1050
      size: link.size,
1051
      shasum: link.shasum,
1052
      license: link.license,
1053
      dependencies: link.dependencies,
1054
      bundledDependencies: link.bundledDependencies,
1055
    })
1056
  }
1057
  return self.publishPkgs(mentions)
1058
}
1059
1060
Req.prototype.publishPkgs = function (mentions) {
1061
  var self = this
1062
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
1063
    if (err) return self.respondError(500, err.stack || err)
1064
    self.server.pushBlobs(self.blobsToPush, function (err) {
1065
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
1066
      self.respond(201)
1067
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
1068
    })
1069
  })
1070
}
1071
1072
Req.prototype.populatePackageJson = function (obj, cb) {
1073
  var self = this
1074
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1075
  var deps, bundledDeps, shasum
1076
1077
  // look for dependencies in links.
1078
  // then fallback to getting it from the tarball blob
1079
1080
  pull(
1081
    self.getMentionLinks(blobId),
1082
    pull.drain(function (link) {
1083
      if (link.dependencies) deps = link.dependencies
1084
      if (link.shasum) shasum = link.shasum
1085
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1086
      // how to handle multiple assignments of dependencies to a package?
1087
    }, function (err) {
1088
      if (err) return cb(new Error(err.stack || err))
1089
      if (deps && (shasum || !self.needShasum)) {
1090
        // assume that the dependencies in the links to the blob are
1091
        // correct.
1092
        obj.dependencies = deps
1093
        obj.bundledDependencies = bundledDeps
1094
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1095
        next(obj)
1096
      } else {
1097
        // get dependencies from the tarball
1098
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1099
          if (err) return cb(err)
1100
          pkg.dist = obj.dist
1101
          pkg.dist.shasum = pkg._shasum
1102
          pkg.author = pkg.author || obj.author
1103
          pkg.version = pkg.version || obj.version
1104
          pkg.name = pkg.name || obj.name
1105
          pkg.license = pkg.license || obj.license
1106
          next(pkg)
1107
        })
1108
      }
1109
    })
1110
  )
1111
  function next(pkg) {
1112
    var feedId = obj.author.url
1113
    self.server.getFeedName(feedId, function (err, name) {
1114
      if (err) console.error('[npm-registry]', err.stack || err), name = ''
1115
      pkg._npmUser = {
1116
        email: feedId,
1117
        name: name
1118
      }
1119
      cb(null, pkg)
1120
    })
1121
  }
1122
}
1123
1124
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1125
  var self = this
1126
  getBlob(sbot, id, function (err, readBlob) {
1127
    if (err) return cb(err)
1128
    cb = once(cb)
1129
    var extract = tar.extract()
1130
    var pkg, shasum
1131
    extract.on('entry', function (header, stream, next) {
1132
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1133
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1134
          if (err) return cb(err)
1135
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1136
          catch(e) { return cb(e) }
1137
          next()
1138
        }))
1139
      } else {
1140
        stream.on('end', next)
1141
        stream.resume()
1142
      }
1143
    })
1144
    extract.on('finish', function () {
1145
      pkg._shasum = shasum
1146
      cb(null, pkg)
1147
    })
1148
    pull(
1149
      readBlob,
1150
      hash('sha1', 'hex', function (err, sum) {
1151
        if (err) return cb(err)
1152
        shasum = sum
1153
      }),
1154
      toPull(zlib.createGunzip()),
1155
      toPull(extract)
1156
    )
1157
  })
1158
}
1159