git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || null
90
  var autoAuth = conf.autoAuth !== false
91
92
  var server = http.createServer(exports.respond(sbot, config))
93
  var getAddress = onceify(function (cb) {
94
    server.on('error', cb)
95
    server.listen(port, host, function () {
96
      server.removeListener('error', cb)
97
      var regHost = formatHost(host || 'localhost')
98
      var regPort = this.address().port
99
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
100
      if (autoAuth) npmLogin(regUrl, next)
101
      else next()
102
      function next(err) {
103
        cb(err, regUrl)
104
      }
105
    })
106
    sbot.on('close', function () {
107
      server.close()
108
    })
109
  })
110
111
  getAddress(function (err, addr) {
112
    if (err) return console.error('[npm-registry]', err.stack || err)
113
    console.log('[npm-registry] Listening on ' + addr)
114
  })
115
116
  return {
117
    getAddress: getAddress
118
  }
119
}
120
121
exports.respond = function (sbot, config) {
122
  var reg = new SsbNpmRegistryServer(sbot, config)
123
  return function (req, res) {
124
    new Req(reg, req, res).serve()
125
  }
126
}
127
128
function publishMsg(sbot, value, cb) {
129
  var gotExpectedPrevious = false
130
  sbot.publish(value, function next(err, msg) {
131
    if (err && /^expected previous:/.test(err.message)) {
132
      // retry once on this error
133
      if (gotExpectedPrevious) return cb(err)
134
      gotExpectedPrevious = true
135
      return sbot.publish(value, next)
136
    }
137
    cb(err, msg)
138
  })
139
}
140
141
function getDependencyBranches(sbot, link, cb) {
142
  var id = link.link
143
144
  // get ids of heads of tree of dependencyBranch message links to include all
145
  // the dependencies of the given tarball id.
146
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
147
  var msgs = {}
148
  var branched = {}
149
  var blobs = {}
150
151
  function addPkgById(id, cb) {
152
    if (blobs[id]) return cb()
153
    blobs[id] = true
154
    getPackageJsonCached(id, function (err, pkg) {
155
      if (err) return cb(err)
156
      var done = multicb()
157
      for (var name in pkg.dependencies || {}) {
158
        addPkgBySpec(name, pkg.dependencies[name], done())
159
      }
160
      done(cb)
161
    })
162
  }
163
164
  function addPkgBySpec(name, spec, cb) {
165
    var versions = {}
166
    var distTags = {}
167
    pull(
168
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
169
      pull.filter(mentionMatches(null, name, spec)),
170
      pull.map(function (mention) {
171
        // query to get the messages since links2 does not include the msg id
172
        return sbot.links({
173
          source: mention.author,
174
          dest: mention.link,
175
          rel: 'mentions',
176
          values: true,
177
        })
178
      }),
179
      pull.flatten(),
180
      pull.drain(function (msg) {
181
        var c = msg && msg.value && msg.value.content
182
        if (!c || !Array.isArray(c.mentions)) return
183
        c.mentions.forEach(function (link) {
184
          var data = link && link.name && decodeName(link.name)
185
          if (!data || data.name !== name) return
186
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
187
          if (data.distTag) distTags[data.distTag] = data.version
188
        })
189
      }, function (err) {
190
        if (err) return cb(err)
191
        var version = distTags[spec]
192
          || semver.maxSatisfying(Object.keys(versions), spec)
193
        var item = versions[version]
194
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
195
        addDepBranchMsg(item.msg)
196
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
197
        addPkgById(item.mention.link, cb)
198
      })
199
    )
200
  }
201
202
  function addDepBranchMsg(msg) {
203
    var c = msg && msg.value && msg.value.content
204
    if (!c) return
205
    if (msgs[msg.key]) return
206
    msgs[msg.key] = msg.value
207
    if (Array.isArray(c.dependencyBranch)) {
208
      for (var k = 0; k < c.dependencyBranch.length; k++) {
209
        branched[c.dependencyBranch[k]] = true
210
      }
211
    }
212
    if (Array.isArray(c.versionBranch)) {
213
      for (var k = 0; k < c.versionBranch.length; k++) {
214
        branched[c.versionBranch[k]] = true
215
      }
216
    }
217
  }
218
219
  function addPkgsByName(name, cb) {
220
    var data = decodeName(name)
221
    pull(
222
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
223
      pull.map(function (mention) {
224
        return packageLinks(sbot, mention.author, mention.link, data.name)
225
      }),
226
      pull.flatten(),
227
      pull.drain(addDepBranchMsg, cb)
228
    )
229
  }
230
231
  var done = multicb()
232
  addPkgsByName(link.name, done())
233
  addPkgById(id, done())
234
  done(function (err) {
235
    if (err) return cb(err)
236
    var ids = []
237
    for (var key in msgs) {
238
      if (!branched[key]) ids.push(key)
239
    }
240
    cb(null, ids)
241
  })
242
}
243
244
function packageLinks(sbot, feed, id, name, spec) {
245
  var matches = mentionMatches(id, name, spec)
246
  return pull(
247
    sbot.links({
248
      source: feed,
249
      dest: id,
250
      rel: 'mentions',
251
      values: true,
252
    }),
253
    pull.filter(function (msg) {
254
      var c = msg && msg.value && msg.value.content
255
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
256
    })
257
  )
258
}
259
260
// For each dependency that is not a bundledDependency, get message ids for
261
// that dependency name + version.
262
function publishSingleMention(sbot, mention, cb) {
263
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
264
  // Calculate dependencyBranch message ids.
265
  var value = {
266
    type: 'npm-packages',
267
    mentions: [mention]
268
  }
269
  var done = multicb({pluck: 1, spread: true})
270
  getDependencyBranches(sbot, mention, function (err, branches) {
271
    if (err) return cb(err)
272
    value.dependencyBranch = branches || undefined
273
    publishMsg(sbot, value, cb)
274
  })
275
}
276
277
function publishMentions(sbot, mentions, cb) {
278
  // console.error("publishing %s mentions", mentions.length)
279
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
280
  // if it is just one mention, fetch and add useful metadata
281
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
282
  publishMsg(sbot, {
283
    type: 'npm-packages',
284
    mentions: mentions,
285
  }, cb)
286
}
287
288
exports.publishPkgMentions = function (sbot, mentions, cb) {
289
  // try to fit the mentions into as few messages as possible,
290
  // while fitting under the message size limit.
291
  var msgs = []
292
  ;(function next(i, chunks) {
293
    if (i >= mentions.length) return cb(null, msgs)
294
    var chunkLen = Math.ceil(mentions.length / chunks)
295
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
296
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
297
      if (err && msgs.length) return onPartialPublish(err)
298
      if (err) return cb(err)
299
      msgs.push(msg)
300
      next(i + chunkLen, chunks)
301
    })
302
  })(0, 1)
303
  function onPartialPublish(err) {
304
    var remaining = mentions.length - i
305
    return cb(new Error('Published messages ' +
306
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
307
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
308
  }
309
}
310
311
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
312
  cb = once(cb)
313
  var waiting = 0
314
  var expandedMentions = mentions.map(function (link) {
315
    var id = link && link.link
316
    if (!id) return link
317
    waiting++
318
    var newLink = {}
319
    for (var k in link) newLink[k] = link[k]
320
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
321
      if (err) return cb(err)
322
      for (var k in props) newLink[k] = pkg[k]
323
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
324
      if (!--waiting) next()
325
    })
326
    return newLink
327
  })
328
  if (!waiting) next()
329
  function next() {
330
    cb(null, expandedMentions)
331
  }
332
}
333
334
function SsbNpmRegistryServer(sbot, config) {
335
  this.sbot = sbot
336
  this.config = config
337
  this.npmConfig = config.npm || {}
338
  this.host = this.npmConfig.host || 'localhost'
339
  this.fetchAll = this.npmConfig.fetchAll
340
  this.needShasum = this.npmConfig.needShasum
341
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
342
  this.getFeedName = memo({cache: lru(100)}, this.getFeedName)
343
}
344
345
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
346
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
347
348
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
349
  var self = this
350
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
351
  ;(function next(i) {
352
    if (i >= ids.length) return cb()
353
    self.sbot.blobs.push(ids[i], function (err) {
354
      if (err) return cb(err)
355
      next(i+1)
356
    })
357
  })(0)
358
}
359
360
function getBlob(sbot, id, cb) {
361
  var blobs = sbot.blobs
362
  blobs.size(id, function (err, size) {
363
    if (err) return cb(err)
364
    if (typeof size === 'number') cb(null, blobs.get(id), size)
365
    else blobs.want(id, function (err, got) {
366
      if (err) cb(err)
367
      else if (!got) cb('missing blob ' + id)
368
      else blobs.size(id, function (err, size) {
369
        if (err) return cb(err)
370
        cb(null, blobs.get(id), size)
371
      })
372
    })
373
  })
374
}
375
376
function getMentions(links2, name) {
377
  return links2.read({
378
    query: [
379
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
380
      {$filter: {dest: {$prefix: '&'}}},
381
      {$map: {
382
        name: ['rel', 1],
383
        size: ['rel', 2],
384
        link: 'dest',
385
        author: 'source',
386
        ts: 'ts'
387
      }}
388
    ]
389
  })
390
}
391
392
function mentionMatches(id, name, spec) {
393
  return function (mention) {
394
    var data = mention
395
      && (!id || id === mention.link)
396
      && mention.name
397
      && decodeName(mention.name)
398
    return data
399
      && data.name === name
400
      && (spec ? semver.satisfies(data.version, spec) : true)
401
  }
402
}
403
404
SsbNpmRegistryServer.prototype.getMentions = function (name) {
405
  if (!this.sbot.links2) return pull.empty()
406
  return getMentions(this.sbot.links2, name)
407
}
408
409
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
410
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
411
  else this.sbot.get(id, function (err, value) {
412
    if (err) return cb(err)
413
    cb(null, {key: id, value: value})
414
  })
415
}
416
417
SsbNpmRegistryServer.prototype.getFeedName = function (id, cb) {
418
  var self = this
419
  if (self.sbot.names && self.sbot.names.getSignifier) {
420
    self.sbot.names.getSignifier(id, function (err, name) {
421
      if (err || !name) tryLinks()
422
      else cb(null, name)
423
    })
424
  } else {
425
    tryLinks()
426
  }
427
  function tryLinks() {
428
    if (!self.sbot.links) return nope()
429
    pull(
430
      self.sbot.links({
431
        source: id,
432
        dest: id,
433
        rel: 'about',
434
        values: true,
435
        limit: 1,
436
        reverse: true,
437
        keys: false,
438
        meta: false
439
      }),
440
      pull.map(function (value) {
441
        return value && value.content && value.content.name
442
      }),
443
      pull.filter(Boolean),
444
      pull.take(1),
445
      pull.collect(function (err, names) {
446
        if (err || !names.length) return nope()
447
        cb(null, names[0])
448
      })
449
    )
450
  }
451
  function nope() {
452
    cb(null, '')
453
  }
454
}
455
456
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
457
  var self = this
458
  var stack = heads.slice()
459
  var seen = {}
460
  return function (abort, cb) {
461
    if (abort) return cb(abort)
462
    for (var id; stack.length && seen[id = stack.pop()];);
463
    if (!id) return cb(true)
464
    seen[id] = true
465
    // TODO: use DFS
466
    self.getMsg(id, function (err, msg) {
467
      if (err) return cb(new Error(err.stack || err))
468
      var c = msg && msg.value && msg.value.content
469
      var links = c && c[prop]
470
      if (Array.isArray(links)) {
471
        stack.push.apply(stack, links)
472
      } else if (links) {
473
        stack.push(links)
474
      }
475
      cb(null, msg)
476
    })
477
  }
478
}
479
480
function Req(server, req, res) {
481
  this.server = server
482
  this.req = req
483
  this.res = res
484
  this.blobsToPush = []
485
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
486
  var ua = this.req.headers['user-agent']
487
  var m = /\bnpm\/([0-9]*)/.exec(ua)
488
  var npmVersion = m && m[1]
489
  this.needShasum = server.needShasum != null ? server.needShasum :
490
    (npmVersion && npmVersion < 5)
491
  this.baseUrl = this.server.npmConfig.baseUrl
492
  if (this.baseUrl) {
493
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
494
  } else {
495
    var hostname = req.headers.host
496
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
497
    this.baseUrl = 'http://' + hostname
498
  }
499
}
500
501
Req.prototype.serve = function () {
502
  console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
503
  this.res.setTimeout(0)
504
  var pathname = this.req.url.replace(/\?.*/, '')
505
  var m
506
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
507
    try {
508
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
509
      this.headMsgPlus = !!m[1]
510
      // ^ means also include packages published after the head message id
511
    } catch(e) {
512
      return this.respondError(400, e.stack || e)
513
    }
514
    pathname = m[3]
515
  }
516
  if (pathname === '/') return this.serveHome()
517
  if (pathname === '/robots.txt') return this.serveRobots()
518
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
519
  if (pathname === '/-/whoami') return this.serveWhoami()
520
  if (pathname === '/-/ping') return this.respond(200, true)
521
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
522
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
523
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
524
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
525
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
526
  return this.respond(404, {error: 'Not found'})
527
}
528
529
Req.prototype.respond = function (status, message) {
530
  this.res.writeHead(status, {'content-type': 'application/json'})
531
  this.res.end(message && JSON.stringify(message, 0, 2))
532
}
533
534
Req.prototype.respondError = function (status, message) {
535
  this.respond(status, {error: message})
536
}
537
538
Req.prototype.respondErrorStr = function (status, err) {
539
  this.res.writeHead(status, {'content-type': 'text/plain'})
540
  this.res.end(err.stack || err)
541
}
542
543
Req.prototype.respondRaw = function (status, body) {
544
  this.res.writeHead(status)
545
  this.res.end(body)
546
}
547
548
Req.prototype.serveHome = function () {
549
  var self = this
550
  self.res.writeHead(200, {'content-type': 'text/html'})
551
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
552
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
553
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
554
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
555
    '</body></html>')
556
}
557
558
Req.prototype.blobDist = function (id) {
559
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
560
  if (!m) throw new Error('bad blob id: ' + id)
561
  return {
562
    integrity: m[2] + '-' + m[1],
563
    tarball: this.baseUrl + '/-/blobs/get/' + id
564
  }
565
}
566
567
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
568
  var self = this
569
  pull(
570
    self.server.sbot.links({
571
      source: feedId,
572
      dest: blobId,
573
      rel: 'mentions',
574
      values: true,
575
    }),
576
    pull.filter(function (msg) {
577
      var c = msg && msg.value && msg.value.content
578
      return c.type === 'npm-packages'
579
    }),
580
    pull.collect(function (err, msgs) {
581
      if (err) return cb(err)
582
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
583
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
584
      // TODO: make a smarter decision about which message id to use
585
      cb(null, msgs.pop().key)
586
    })
587
  )
588
}
589
590
Req.prototype.resolvePkg = function (pkgSpec, cb) {
591
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
592
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
593
  var self = this
594
  var pkgName = m[1]
595
  var spec = m[2] || '*'
596
  var versions = {}
597
  var distTags = {}
598
  pull(
599
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
600
    pull.drain(function (mention) {
601
      var data = decodeName(mention.name)
602
      if (!data.version) return
603
      if (data.distTag) {
604
        distTags[data.distTag] = data.version
605
      }
606
      versions[data.version] = {
607
        author: mention.author,
608
        name: pkgName,
609
        version: data.version,
610
        blobId: mention.link
611
      }
612
    }, function (err) {
613
      if (err) return cb(err)
614
      var version = distTags[spec]
615
        || semver.maxSatisfying(Object.keys(versions), spec)
616
      var item = versions[version]
617
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
618
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
619
        if (err) return cb(err)
620
        item.msgId = id
621
        cb(null, item)
622
      })
623
    })
624
  )
625
}
626
627
Req.prototype.resolvePkgs = function (specs, cb) {
628
  var done = multicb({pluck: 1})
629
  var self = this
630
  specs.forEach(function (spec) {
631
    self.resolvePkg(spec, done())
632
  })
633
  done(cb)
634
}
635
636
Req.prototype.serveBootstrap = function () {
637
  var self = this
638
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
639
  var postInstallCmd = self.server.npmConfig.postInstallCmd
640
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
641
  var ssbNpmRegistryName = require('./package.json').name
642
  var ssbNpmRegistryVersion = require('./package.json').version
643
  var ssbNpmRegistryNameVersion = ssbNpmRegistryName + '-' + ssbNpmRegistryVersion
644
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
645
  var done = multicb({pluck: 1, spread: true})
646
  self.resolvePkg(ssbNpmRegistrySpec, done())
647
  self.resolvePkgs(pkgs, done())
648
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
649
    if (err) return self.respondErrorStr(500, err.stack || err)
650
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
651
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
652
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
653
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
654
    var globalPkgs = pkgsInfo.map(function (info) {
655
      return info.name + '@' + info.version
656
    }).join(' ')
657
    var npmCmd = 'install -g ' + globalPkgs
658
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
659
    var wsUrl = self.baseUrl + '/-'
660
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
661
662
    var script =
663
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
664
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
665
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
666
      'tar xzf package.tgz &&\n' +
667
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
668
        pkgMsgs.map(function (id) {
669
          return '  --branch ' + id + ' \\\n'
670
        }).join('') +
671
        '  -- ' + npmCmd +
672
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
673
674
    self.res.writeHead(200, {'Content-type': 'text/plain'})
675
    self.res.end(script)
676
  })
677
}
678
679
Req.prototype.serveRobots = function () {
680
  this.res.writeHead(200, {'Content-type': 'text/plain'})
681
  this.res.end('User-agent: *\nDisallow: /\n')
682
}
683
684
Req.prototype.serveWhoami = function () {
685
  var self = this
686
  self.server.sbot.whoami(function (err, feed) {
687
    if (err) return self.respondError(500, err.stack || err)
688
    self.respond(200, {username: feed.id})
689
  })
690
}
691
692
Req.prototype.serveUser1 = function () {
693
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
694
}
695
696
Req.prototype.serveBlob = function (id) {
697
  var self = this
698
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
699
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
700
    if (err) {
701
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
702
      else return self.respondErrorStr(500, err.message || err)
703
    }
704
    self.res.writeHead(200, {
705
      'Cache-Control': 'public, max-age=315360000',
706
      'Content-Length': size,
707
      'etag': id
708
    })
709
    pull(
710
      readBlob,
711
      toPull(self.res, function (err) {
712
        if (err) console.error('[npm-registry]', err)
713
      })
714
    )
715
  })
716
}
717
718
Req.prototype.serveMsg = function (id) {
719
  var self = this
720
  try { id = decodeURIComponent(id) }
721
  catch (e) {}
722
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
723
  self.server.sbot.get(id, function (err, value) {
724
    if (err) return self.respondError(500, err.message || err)
725
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
726
    self.res.writeHead(200, {
727
      'Content-Type': 'application/json',
728
      'Cache-Control': 'public, max-age=315360000',
729
      'Content-Length': out.length,
730
      'etag': id
731
    })
732
    self.res.end(out)
733
  })
734
}
735
736
function decodeName(name) {
737
  var parts = String(name).replace(/\.tgz$/, '').split(':')
738
  return {
739
    name: parts[1],
740
    version: parts[2],
741
    distTag: parts[3],
742
  }
743
}
744
745
Req.prototype.getMsgMentions = function (name) {
746
  return pull(
747
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
748
    // decryption could be done here
749
    pull.map(function (msg) {
750
      var c = msg.value && msg.value.content
751
      if (!c.mentions || !Array.isArray(c.mentions)) return []
752
      return c.mentions.map(function (mention) {
753
        return {
754
          name: mention.name,
755
          size: mention.size,
756
          link: mention.link,
757
          author: msg.value.author,
758
          ts: msg.value.timestamp,
759
        }
760
      })
761
    }),
762
    pull.flatten(),
763
    pull.filter(typeof name === 'string' ? function (link) {
764
      return link.name === name
765
    } : name && name.$prefix ? function (link) {
766
      return link.name.substr(0, name.$prefix.length) === name.$prefix
767
    } : function () {
768
      throw new TypeError('unsupported name filter')
769
    })
770
  )
771
}
772
773
Req.prototype.getMentions = function (name) {
774
  var useMsgMentions = this.headMsgIds
775
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
776
  if (useServerMentions && !this.server.sbot.links2) {
777
    return this.headMsgPlus
778
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
779
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
780
  }
781
  return cat([
782
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
783
    useServerMentions ? this.server.getMentions(name) : pull.empty()
784
  ])
785
}
786
787
Req.prototype.getMentionLinks = function (blobId) {
788
  return pull(
789
    this.headMsgIds
790
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
791
      : this.server.sbot.links({
792
        dest: blobId,
793
        rel: 'mentions',
794
        values: true,
795
      }),
796
    // decryption could be done here
797
    pull.map(function (msg) {
798
      var c = msg.value && msg.value.content
799
      return c && Array.isArray(c.mentions) && c.mentions || []
800
    }),
801
    pull.flatten(),
802
    pull.filter(function (link) {
803
      return link && link.link === blobId
804
    })
805
  )
806
}
807
808
Req.prototype.servePkg = function (pathname) {
809
  var self = this
810
  var parts = pathname.split('/')
811
  var pkgName = parts.shift().replace(/%2f/i, '/')
812
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
813
  var spec = parts.shift()
814
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
815
  if (parts.length > 0) return this.respondError(404)
816
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
817
  var obj = {
818
    _id: pkgName,
819
    name: pkgName,
820
    'dist-tags': {},
821
    versions: {},
822
    time: {}
823
  }
824
  var distTags = {/* <tag>: {version, ts}*/}
825
  pull(
826
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
827
    pull.drain(function (mention) {
828
      var data = decodeName(mention.name)
829
      if (!data.version) return
830
      if (data.distTag) {
831
        var tag = distTags[data.distTag]
832
        if (!tag || mention.ts > tag.ts) {
833
          /* TODO: sort by causal order instead of only by timestamps */
834
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
835
        }
836
      }
837
      obj.versions[data.version] = {
838
        author: {
839
          url: mention.author
840
        },
841
        name: pkgName,
842
        version: data.version,
843
        dist: self.blobDist(mention.link)
844
      }
845
      var ts = new Date(mention.ts)
846
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
847
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
848
      obj.time[data.version] = ts.toISOString()
849
    }, function (err) {
850
      if (err) return self.respondError(500, err.stack || err)
851
      for (var tag in distTags) {
852
        obj['dist-tags'][tag] = distTags[tag].version
853
      }
854
      if (spec) resolveSpec()
855
      else if (self.fetchAll) resolveAll()
856
      else resolved()
857
    })
858
  )
859
  function resolveSpec() {
860
    var version = obj['dist-tags'][spec]
861
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
862
    obj = obj.versions[version]
863
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
864
    self.populatePackageJson(obj, function (err, pkg) {
865
      if (err) return resolved(err)
866
      obj = pkg || obj
867
      resolved()
868
    })
869
  }
870
  function resolveVersion(version, cb) {
871
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
872
      if (err) return cb(err)
873
      if (pkg) obj.versions[version] = pkg
874
      if (pkg && pkg.license && !obj.license) obj.license = pkg.license
875
      cb()
876
    })
877
  }
878
  function resolveAll() {
879
    var done = multicb()
880
    for (var version in obj.versions) {
881
      resolveVersion(version, done())
882
    }
883
    done(resolved)
884
  }
885
  function resolved(err) {
886
    if (err) return self.respondError(500, err.stack || err)
887
    self.respond(200, obj)
888
  }
889
}
890
891
Req.prototype.servePrebuild = function (name) {
892
  var self = this
893
  var getMention = self.getMentions('prebuild:' + name)
894
  var blobsByAuthor = {/* <author>: BlobId */}
895
  pull(getMention, pull.drain(function (link) {
896
    blobsByAuthor[link.author] = link.link
897
  }, function (err) {
898
    if (err) return self.respondError(500, err.stack || err)
899
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
900
    var blobId
901
    for (var feed in blobsByAuthor) {
902
      var blob = blobId = blobsByAuthor[feed]
903
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
904
      feeds.push(feed)
905
    }
906
    switch (Object.keys(authorsByLink).length) {
907
      case 0:
908
        return self.respondError(404, 'Not Found')
909
      case 1:
910
        return self.serveBlob(blobId)
911
      default:
912
        return self.respond(300, {choices: authorsByLink})
913
    }
914
  }))
915
}
916
917
var localhosts = {
918
  '::1': true,
919
  '127.0.0.1': true,
920
  '::ffff:127.0.0.1': true,
921
}
922
923
Req.prototype.publishPkg = function (pkgName) {
924
  var self = this
925
  var remoteAddress = self.req.socket.remoteAddress
926
  if (!(remoteAddress in localhosts)) {
927
    return self.respondError(403, 'You may not publish as this user.')
928
  }
929
930
  var chunks = []
931
  self.req.on('data', function (data) {
932
    chunks.push(data)
933
  })
934
  self.req.on('end', function () {
935
    var data
936
    try {
937
      data = JSON.parse(Buffer.concat(chunks))
938
    } catch(e) {
939
      return self.respondError(400, e.stack)
940
    }
941
    return self.publishPkg2(pkgName, data || {})
942
  })
943
}
944
945
Req.prototype.publishPkg2 = function (name, data) {
946
  var self = this
947
  if (data.users) console.trace('[npm-registry] users property is not supported')
948
  var attachments = data._attachments || {}
949
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
950
  var done = multicb()
951
  function addAttachmentAsBlob(filename, cb) {
952
    var data = attachments[filename].data
953
    var tarball = new Buffer(data, 'base64')
954
    var length = attachments[filename].length
955
    if (length && length !== tarball.length) return self.respondError(400,
956
      'Length mismatch for attachment \'' + filename + '\'')
957
    self.server.sbot.blobs.add(function (err, id) {
958
      if (err) return cb(err)
959
      self.blobsToPush.push(id)
960
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
961
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
962
      cb()
963
    })(pull.once(tarball))
964
  }
965
  for (var filename in attachments) {
966
    addAttachmentAsBlob(filename, done())
967
  }
968
  done(function (err) {
969
    if (err) return self.respondError(500, err.stack || err)
970
    try {
971
      self.publishPkg3(name, data, links)
972
    } catch(e) {
973
      self.respondError(500, e.stack || e)
974
    }
975
  })
976
}
977
978
Req.prototype.publishPkg3 = function (name, data, links) {
979
  var self = this
980
  var versions = data.versions || {}
981
  var linksByVersion = {/* <version>: link */}
982
983
  // associate tarball blobs with versions
984
  for (var version in versions) {
985
    var pkg = versions[version]
986
    if (!pkg) return self.respondError(400, 'Bad package object')
987
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
988
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
989
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
990
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
991
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
992
    var filename = m[1]
993
    var link = links[filename]
994
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
995
    // TODO?: try to find missing tarball mentioned in other messages
996
    if (pkg.version && pkg.version !== version)
997
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
998
    linksByVersion[version] = link
999
    link.version = version
1000
    link.license = pkg.license
1001
    link.dependencies = pkg.dependencies || {}
1002
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
1003
  }
1004
1005
  // associate blobs with dist-tags
1006
  var tags = data['dist-tags'] || {}
1007
  for (var tag in tags) {
1008
    var version = tags[tag]
1009
    var link = linksByVersion[version]
1010
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
1011
    // TODO?: support setting dist-tag without version,
1012
    // by looking up a tarball blob for the version
1013
    link.tag = tag
1014
  }
1015
1016
  // compute blob links to publish
1017
  var mentions = []
1018
  for (var filename in links) {
1019
    var link = links[filename] || {}
1020
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
1021
    mentions.push({
1022
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
1023
      link: link.link,
1024
      size: link.size,
1025
      shasum: link.shasum,
1026
      license: link.license,
1027
      dependencies: link.dependencies,
1028
      bundledDependencies: link.bundledDependencies,
1029
    })
1030
  }
1031
  return self.publishPkgs(mentions)
1032
}
1033
1034
Req.prototype.publishPkgs = function (mentions) {
1035
  var self = this
1036
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
1037
    if (err) return self.respondError(500, err.stack || err)
1038
    self.server.pushBlobs(self.blobsToPush, function (err) {
1039
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
1040
      self.respond(201)
1041
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
1042
    })
1043
  })
1044
}
1045
1046
Req.prototype.populatePackageJson = function (obj, cb) {
1047
  var self = this
1048
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1049
  var deps, bundledDeps, shasum
1050
1051
  // look for dependencies in links.
1052
  // then fallback to getting it from the tarball blob
1053
1054
  pull(
1055
    self.getMentionLinks(blobId),
1056
    pull.drain(function (link) {
1057
      if (link.dependencies) deps = link.dependencies
1058
      if (link.shasum) shasum = link.shasum
1059
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1060
      // how to handle multiple assignments of dependencies to a package?
1061
    }, function (err) {
1062
      if (err) return cb(new Error(err.stack || err))
1063
      if (deps && (shasum || !self.needShasum)) {
1064
        // assume that the dependencies in the links to the blob are
1065
        // correct.
1066
        obj.dependencies = deps
1067
        obj.bundledDependencies = bundledDeps
1068
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1069
        next(obj)
1070
      } else {
1071
        // get dependencies from the tarball
1072
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1073
          if (err) return cb(err)
1074
          pkg.dist = obj.dist
1075
          pkg.dist.shasum = pkg._shasum
1076
          pkg.author = pkg.author || obj.author
1077
          pkg.version = pkg.version || obj.version
1078
          pkg.name = pkg.name || obj.name
1079
          pkg.license = pkg.license || obj.license
1080
          next(pkg)
1081
        })
1082
      }
1083
    })
1084
  )
1085
  function next(pkg) {
1086
    var feedId = obj.author.url
1087
    self.server.getFeedName(feedId, function (err, name) {
1088
      if (err) console.error('[npm-registry]', err.stack || err), name = ''
1089
      pkg._npmUser = {
1090
        email: feedId,
1091
        name: name
1092
      }
1093
      cb(null, pkg)
1094
    })
1095
  }
1096
}
1097
1098
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1099
  var self = this
1100
  getBlob(sbot, id, function (err, readBlob) {
1101
    if (err) return cb(err)
1102
    cb = once(cb)
1103
    var extract = tar.extract()
1104
    var pkg, shasum
1105
    extract.on('entry', function (header, stream, next) {
1106
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1107
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1108
          if (err) return cb(err)
1109
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1110
          catch(e) { return cb(e) }
1111
          next()
1112
        }))
1113
      } else {
1114
        stream.on('end', next)
1115
        stream.resume()
1116
      }
1117
    })
1118
    extract.on('finish', function () {
1119
      pkg._shasum = shasum
1120
      cb(null, pkg)
1121
    })
1122
    pull(
1123
      readBlob,
1124
      hash('sha1', 'hex', function (err, sum) {
1125
        if (err) return cb(err)
1126
        shasum = sum
1127
      }),
1128
      toPull(zlib.createGunzip()),
1129
      toPull(extract)
1130
    )
1131
  })
1132
}
1133