git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function idToHex(id) {
25
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26
  return new Buffer(b64, 'base64').toString('hex')
27
}
28
29
function onceify(fn, self) {
30
  var cbs = [], err, data
31
  return function (cb) {
32
    if (fn) {
33
      cbs.push(cb)
34
      fn.call(self, function (_err, _data) {
35
        err = _err, data = _data
36
        var _cbs = cbs
37
        cbs = null
38
        while (_cbs.length) _cbs.shift()(err, data)
39
      })
40
      fn = null
41
    } else if (cbs) {
42
      cbs.push(cb)
43
    } else {
44
      cb(err, data)
45
    }
46
  }
47
}
48
49
function once(cb) {
50
  var done
51
  return function (err, result) {
52
    if (done) {
53
      if (err) console.trace(err)
54
    } else {
55
      done = true
56
      cb(err, result)
57
    }
58
  }
59
}
60
61
function npmLogin(registryAddress, cb) {
62
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63
  var filename = path.join(os.homedir(), '.npmrc')
64
  fs.readFile(filename, 'utf8', function (err, data) {
65
    if (err && err.code === 'ENOENT') data = ''
66
    else if (err) return cb(new Error(err.stack))
67
    var lines = data ? data.split('\n') : []
68
    if (lines.indexOf(tokenLine) > -1) return cb()
69
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71
    fs.appendFile(filename, line, cb)
72
  })
73
}
74
75
function formatHost(host) {
76
  if (host === '::1') return 'localhost'
77
  host = host.replace(/^::ffff:/, '')
78
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
79
}
80
81
exports.name = 'npm-registry'
82
exports.version = '1.0.0'
83
exports.manifest = {
84
  getAddress: 'async'
85
}
86
exports.init = function (sbot, config) {
87
  var conf = config.npm || {}
88
  var port = conf.port || 8043
89
  var host = conf.host || null
90
  var autoAuth = conf.autoAuth !== false
91
92
  var server = http.createServer(exports.respond(sbot, config))
93
  var getAddress = onceify(function (cb) {
94
    server.on('error', cb)
95
    server.listen(port, host, function () {
96
      server.removeListener('error', cb)
97
      var regHost = formatHost(host || 'localhost')
98
      var regPort = this.address().port
99
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
100
      if (autoAuth) npmLogin(regUrl, next)
101
      else next()
102
      function next(err) {
103
        cb(err, regUrl)
104
      }
105
    })
106
    sbot.on('close', function () {
107
      server.close()
108
    })
109
  })
110
111
  getAddress(function (err, addr) {
112
    if (err) return console.error('[npm-registry]', err.stack || err)
113
    console.log('[npm-registry] Listening on ' + addr)
114
  })
115
116
  return {
117
    getAddress: getAddress
118
  }
119
}
120
121
exports.respond = function (sbot, config) {
122
  var reg = new SsbNpmRegistryServer(sbot, config)
123
  return function (req, res) {
124
    new Req(reg, req, res).serve()
125
  }
126
}
127
128
function publishMsg(sbot, value, cb) {
129
  var gotExpectedPrevious = false
130
  sbot.publish(value, function next(err, msg) {
131
    if (err && /^expected previous:/.test(err.message)) {
132
      // retry once on this error
133
      if (gotExpectedPrevious) return cb(err)
134
      gotExpectedPrevious = true
135
      return sbot.publish(value, next)
136
    }
137
    cb(err, msg)
138
  })
139
}
140
141
function getDependencyBranches(sbot, link, cb) {
142
  var id = link.link
143
144
  // get ids of heads of tree of dependencyBranch message links to include all
145
  // the dependencies of the given tarball id.
146
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
147
  var msgs = {}
148
  var branched = {}
149
  var blobs = {}
150
151
  function addPkgById(id, cb) {
152
    if (blobs[id]) return cb()
153
    blobs[id] = true
154
    getPackageJsonCached(id, function (err, pkg) {
155
      if (err) return cb(err)
156
      var done = multicb()
157
      for (var name in pkg.dependencies || {}) {
158
        addPkgBySpec(name, pkg.dependencies[name], done())
159
      }
160
      done(cb)
161
    })
162
  }
163
164
  function addPkgBySpec(name, spec, cb) {
165
    var versions = {}
166
    var distTags = {}
167
    pull(
168
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
169
      pull.filter(mentionMatches(null, name, spec)),
170
      pull.map(function (mention) {
171
        // query to get the messages since links2 does not include the msg id
172
        return sbot.links({
173
          source: mention.author,
174
          dest: mention.link,
175
          rel: 'mentions',
176
          values: true,
177
        })
178
      }),
179
      pull.flatten(),
180
      pull.drain(function (msg) {
181
        var c = msg && msg.value && msg.value.content
182
        if (!c || !Array.isArray(c.mentions)) return
183
        c.mentions.forEach(function (link) {
184
          var data = link && link.name && decodeName(link.name)
185
          if (!data || data.name !== name) return
186
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
187
          if (data.distTag) distTags[data.distTag] = data.version
188
        })
189
      }, function (err) {
190
        if (err) return cb(err)
191
        var version = distTags[spec]
192
          || semver.maxSatisfying(Object.keys(versions), spec)
193
        var item = versions[version]
194
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
195
        addDepBranchMsg(item.msg)
196
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
197
        addPkgById(item.mention.link, cb)
198
      })
199
    )
200
  }
201
202
  function addDepBranchMsg(msg) {
203
    var c = msg && msg.value && msg.value.content
204
    if (!c) return
205
    if (msgs[msg.key]) return
206
    msgs[msg.key] = msg.value
207
    if (Array.isArray(c.dependencyBranch)) {
208
      for (var k = 0; k < c.dependencyBranch.length; k++) {
209
        branched[c.dependencyBranch[k]] = true
210
      }
211
    }
212
    if (Array.isArray(c.versionBranch)) {
213
      for (var k = 0; k < c.versionBranch.length; k++) {
214
        branched[c.versionBranch[k]] = true
215
      }
216
    }
217
  }
218
219
  function addPkgsByName(name, cb) {
220
    var data = decodeName(name)
221
    pull(
222
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
223
      pull.map(function (mention) {
224
        return packageLinks(sbot, mention.author, mention.link, data.name)
225
      }),
226
      pull.flatten(),
227
      pull.drain(addDepBranchMsg, cb)
228
    )
229
  }
230
231
  var done = multicb()
232
  addPkgsByName(link.name, done())
233
  addPkgById(id, done())
234
  done(function (err) {
235
    if (err) return cb(err)
236
    var ids = []
237
    for (var key in msgs) {
238
      if (!branched[key]) ids.push(key)
239
    }
240
    cb(null, ids)
241
  })
242
}
243
244
function packageLinks(sbot, feed, id, name, spec) {
245
  var matches = mentionMatches(id, name, spec)
246
  return pull(
247
    sbot.links({
248
      source: feed,
249
      dest: id,
250
      rel: 'mentions',
251
      values: true,
252
    }),
253
    pull.filter(function (msg) {
254
      var c = msg && msg.value && msg.value.content
255
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
256
    })
257
  )
258
}
259
260
// For each dependency that is not a bundledDependency, get message ids for
261
// that dependency name + version.
262
function publishSingleMention(sbot, mention, cb) {
263
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
264
  // Calculate dependencyBranch message ids.
265
  var value = {
266
    type: 'npm-packages',
267
    mentions: [mention]
268
  }
269
  var done = multicb({pluck: 1, spread: true})
270
  getDependencyBranches(sbot, mention, function (err, branches) {
271
    if (err) return cb(err)
272
    value.dependencyBranch = branches || undefined
273
    publishMsg(sbot, value, cb)
274
  })
275
}
276
277
function publishMentions(sbot, mentions, cb) {
278
  // console.error("publishing %s mentions", mentions.length)
279
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
280
  // if it is just one mention, fetch and add useful metadata
281
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
282
  publishMsg(sbot, {
283
    type: 'npm-packages',
284
    mentions: mentions,
285
  }, cb)
286
}
287
288
exports.publishPkgMentions = function (sbot, mentions, cb) {
289
  // try to fit the mentions into as few messages as possible,
290
  // while fitting under the message size limit.
291
  var msgs = []
292
  ;(function next(i, chunks) {
293
    if (i >= mentions.length) return cb(null, msgs)
294
    var chunkLen = Math.ceil(mentions.length / chunks)
295
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
296
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
297
      if (err && msgs.length) return onPartialPublish(err)
298
      if (err) return cb(err)
299
      msgs.push(msg)
300
      next(i + chunkLen, chunks)
301
    })
302
  })(0, 1)
303
  function onPartialPublish(err) {
304
    var remaining = mentions.length - i
305
    return cb(new Error('Published messages ' +
306
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
307
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
308
  }
309
}
310
311
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
312
  cb = once(cb)
313
  var waiting = 0
314
  var expandedMentions = mentions.map(function (link) {
315
    var id = link && link.link
316
    if (!id) return link
317
    waiting++
318
    var newLink = {}
319
    for (var k in link) newLink[k] = link[k]
320
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
321
      if (err) return cb(err)
322
      for (var k in props) newLink[k] = pkg[k]
323
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
324
      if (!--waiting) next()
325
    })
326
    return newLink
327
  })
328
  if (!waiting) next()
329
  function next() {
330
    cb(null, expandedMentions)
331
  }
332
}
333
334
function SsbNpmRegistryServer(sbot, config) {
335
  this.sbot = sbot
336
  this.config = config
337
  this.npmConfig = config.npm || {}
338
  this.host = this.npmConfig.host || 'localhost'
339
  this.fetchAll = this.npmConfig.fetchAll
340
  this.needShasum = this.npmConfig.needShasum
341
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
342
  this.getFeedName = memo({cache: lru(100)}, this.getFeedName)
343
  if (sbot.ws) {
344
    var wsPort = config.ws && Number(config.ws.port) || '8989'
345
    this.wsUrl = 'http://localhost:' + wsPort
346
  }
347
}
348
349
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
350
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
351
352
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
353
  var self = this
354
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
355
  ;(function next(i) {
356
    if (i >= ids.length) return cb()
357
    self.sbot.blobs.push(ids[i], function (err) {
358
      if (err) return cb(err)
359
      next(i+1)
360
    })
361
  })(0)
362
}
363
364
function getBlob(sbot, id, cb) {
365
  var blobs = sbot.blobs
366
  blobs.size(id, function (err, size) {
367
    if (err) return cb(err)
368
    if (typeof size === 'number') cb(null, blobs.get(id), size)
369
    else blobs.want(id, function (err, got) {
370
      if (err) cb(err)
371
      else if (!got) cb('missing blob ' + id)
372
      else blobs.size(id, function (err, size) {
373
        if (err) return cb(err)
374
        cb(null, blobs.get(id), size)
375
      })
376
    })
377
  })
378
}
379
380
function getMentions(links2, name) {
381
  return links2.read({
382
    query: [
383
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
384
      {$filter: {dest: {$prefix: '&'}}},
385
      {$map: {
386
        name: ['rel', 1],
387
        size: ['rel', 2],
388
        link: 'dest',
389
        author: 'source',
390
        ts: 'ts'
391
      }}
392
    ]
393
  })
394
}
395
396
function mentionMatches(id, name, spec) {
397
  return function (mention) {
398
    var data = mention
399
      && (!id || id === mention.link)
400
      && mention.name
401
      && decodeName(mention.name)
402
    return data
403
      && data.name === name
404
      && (spec ? semver.satisfies(data.version, spec) : true)
405
  }
406
}
407
408
SsbNpmRegistryServer.prototype.getMentions = function (name) {
409
  if (!this.sbot.links2) return pull.empty()
410
  return getMentions(this.sbot.links2, name)
411
}
412
413
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
414
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
415
  else this.sbot.get(id, function (err, value) {
416
    if (err) return cb(err)
417
    cb(null, {key: id, value: value})
418
  })
419
}
420
421
SsbNpmRegistryServer.prototype.getFeedName = function (id, cb) {
422
  var self = this
423
  if (self.sbot.names && self.sbot.names.getSignifier) {
424
    self.sbot.names.getSignifier(id, function (err, name) {
425
      if (err || !name) tryLinks()
426
      else cb(null, name)
427
    })
428
  } else {
429
    tryLinks()
430
  }
431
  function tryLinks() {
432
    if (!self.sbot.links) return nope()
433
    pull(
434
      self.sbot.links({
435
        source: id,
436
        dest: id,
437
        rel: 'about',
438
        values: true,
439
        limit: 1,
440
        reverse: true,
441
        keys: false,
442
        meta: false
443
      }),
444
      pull.map(function (value) {
445
        return value && value.content && value.content.name
446
      }),
447
      pull.filter(Boolean),
448
      pull.take(1),
449
      pull.collect(function (err, names) {
450
        if (err || !names.length) return nope()
451
        cb(null, names[0])
452
      })
453
    )
454
  }
455
  function nope() {
456
    cb(null, '')
457
  }
458
}
459
460
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
461
  var self = this
462
  var stack = heads.slice()
463
  var seen = {}
464
  return function (abort, cb) {
465
    if (abort) return cb(abort)
466
    for (var id; stack.length && seen[id = stack.pop()];);
467
    if (!id) return cb(true)
468
    seen[id] = true
469
    // TODO: use DFS
470
    self.getMsg(id, function (err, msg) {
471
      if (err) return cb(new Error(err.stack || err))
472
      var c = msg && msg.value && msg.value.content
473
      var links = c && c[prop]
474
      if (Array.isArray(links)) {
475
        stack.push.apply(stack, links)
476
      } else if (links) {
477
        stack.push(links)
478
      }
479
      cb(null, msg)
480
    })
481
  }
482
}
483
484
function Req(server, req, res) {
485
  this.server = server
486
  this.req = req
487
  this.res = res
488
  this.blobsToPush = []
489
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
490
  var ua = this.req.headers['user-agent']
491
  var m = /\bnpm\/([0-9]*)/.exec(ua)
492
  var npmVersion = m && m[1]
493
  this.needShasum = server.needShasum != null ? server.needShasum :
494
    (npmVersion && npmVersion < 5)
495
  this.baseUrl = this.server.npmConfig.baseUrl
496
  if (this.baseUrl) {
497
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
498
  } else {
499
    var hostname = req.headers.host
500
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
501
    this.baseUrl = 'http://' + hostname
502
  }
503
504
  // prefer serving blobs from ssb-ws, for consistency of tarball URLs.
505
  this.blobBaseUrl = this.server.wsUrl && this.isLocal()
506
    ? this.server.wsUrl + '/blobs/get/'
507
    : this.baseUrl + '/-/blobs/get/'
508
}
509
510
Req.prototype.isLocal = function () {
511
  var remoteAddr = this.req.socket.remoteAddress
512
  return remoteAddr === '::1'
513
    || remoteAddr === '::ffff:127.0.0.1'
514
    || remoteAddr === '127.0.0.1'
515
}
516
517
Req.prototype.serve = function () {
518
  console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
519
  this.res.setTimeout(0)
520
  var pathname = this.req.url.replace(/\?.*/, '')
521
  var m
522
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
523
    try {
524
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
525
      this.headMsgPlus = !!m[1]
526
      // ^ means also include packages published after the head message id
527
    } catch(e) {
528
      return this.respondError(400, e.stack || e)
529
    }
530
    pathname = m[3]
531
  }
532
  if (pathname === '/') return this.serveHome()
533
  if (pathname === '/robots.txt') return this.serveRobots()
534
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
535
  if (pathname === '/-/whoami') return this.serveWhoami()
536
  if (pathname === '/-/ping') return this.respond(200, true)
537
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
538
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
539
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
540
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
541
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
542
  return this.respond(404, {error: 'Not found'})
543
}
544
545
Req.prototype.respond = function (status, message) {
546
  this.res.writeHead(status, {'content-type': 'application/json'})
547
  this.res.end(message && JSON.stringify(message, 0, 2))
548
}
549
550
Req.prototype.respondError = function (status, message) {
551
  this.respond(status, {error: message})
552
}
553
554
Req.prototype.respondErrorStr = function (status, err) {
555
  this.res.writeHead(status, {'content-type': 'text/plain'})
556
  this.res.end(err.stack || err)
557
}
558
559
Req.prototype.respondRaw = function (status, body) {
560
  this.res.writeHead(status)
561
  this.res.end(body)
562
}
563
564
Req.prototype.serveHome = function () {
565
  var self = this
566
  self.res.writeHead(200, {'content-type': 'text/html'})
567
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
568
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
569
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
570
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
571
    '</body></html>')
572
}
573
574
Req.prototype.blobDist = function (id) {
575
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
576
  if (!m) throw new Error('bad blob id: ' + id)
577
  return {
578
    integrity: m[2] + '-' + m[1],
579
    tarball: this.blobBaseUrl + id
580
  }
581
}
582
583
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
584
  var self = this
585
  pull(
586
    self.server.sbot.links({
587
      source: feedId,
588
      dest: blobId,
589
      rel: 'mentions',
590
      values: true,
591
    }),
592
    pull.filter(function (msg) {
593
      var c = msg && msg.value && msg.value.content
594
      return c.type === 'npm-packages'
595
    }),
596
    pull.collect(function (err, msgs) {
597
      if (err) return cb(err)
598
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
599
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
600
      // TODO: make a smarter decision about which message id to use
601
      cb(null, msgs.pop().key)
602
    })
603
  )
604
}
605
606
Req.prototype.resolvePkg = function (pkgSpec, cb) {
607
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
608
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
609
  var self = this
610
  var pkgName = m[1]
611
  var spec = m[2] || '*'
612
  var versions = {}
613
  var distTags = {}
614
  pull(
615
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
616
    pull.drain(function (mention) {
617
      var data = decodeName(mention.name)
618
      if (!data.version) return
619
      if (data.distTag) {
620
        distTags[data.distTag] = data.version
621
      }
622
      versions[data.version] = {
623
        author: mention.author,
624
        name: pkgName,
625
        version: data.version,
626
        blobId: mention.link
627
      }
628
    }, function (err) {
629
      if (err) return cb(err)
630
      var version = distTags[spec]
631
        || semver.maxSatisfying(Object.keys(versions), spec)
632
      var item = versions[version]
633
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
634
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
635
        if (err) return cb(err)
636
        item.msgId = id
637
        cb(null, item)
638
      })
639
    })
640
  )
641
}
642
643
Req.prototype.resolvePkgs = function (specs, cb) {
644
  var done = multicb({pluck: 1})
645
  var self = this
646
  specs.forEach(function (spec) {
647
    self.resolvePkg(spec, done())
648
  })
649
  done(cb)
650
}
651
652
Req.prototype.serveBootstrap = function () {
653
  var self = this
654
  var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
655
  var postInstallCmd = self.server.npmConfig.postInstallCmd
656
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
657
  var ssbNpmRegistryName = require('./package.json').name
658
  var ssbNpmRegistryVersion = require('./package.json').version
659
  var ssbNpmRegistryNameVersion = ssbNpmRegistryName + '-' + ssbNpmRegistryVersion
660
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
661
  var done = multicb({pluck: 1, spread: true})
662
  self.resolvePkg(ssbNpmRegistrySpec, done())
663
  self.resolvePkgs(pkgs, done())
664
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
665
    if (err) return self.respondErrorStr(500, err.stack || err)
666
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
667
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
668
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
669
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
670
    var globalPkgs = pkgsInfo.map(function (info) {
671
      return info.name + '@' + info.version
672
    }).join(' ')
673
    var npmCmd = 'install -g ' + globalPkgs
674
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
675
    var wsUrl = self.baseUrl + '/-'
676
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
677
678
    var script =
679
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
680
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
681
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
682
      'tar xzf package.tgz &&\n' +
683
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
684
        pkgMsgs.map(function (id) {
685
          return '  --branch ' + id + ' \\\n'
686
        }).join('') +
687
        '  -- ' + npmCmd +
688
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
689
690
    self.res.writeHead(200, {'Content-type': 'text/plain'})
691
    self.res.end(script)
692
  })
693
}
694
695
Req.prototype.serveRobots = function () {
696
  this.res.writeHead(200, {'Content-type': 'text/plain'})
697
  this.res.end('User-agent: *\nDisallow: /\n')
698
}
699
700
Req.prototype.serveWhoami = function () {
701
  var self = this
702
  self.server.sbot.whoami(function (err, feed) {
703
    if (err) return self.respondError(500, err.stack || err)
704
    self.respond(200, {username: feed.id})
705
  })
706
}
707
708
Req.prototype.serveUser1 = function () {
709
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
710
}
711
712
Req.prototype.serveBlob = function (id) {
713
  var self = this
714
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
715
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
716
    if (err) {
717
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
718
      else return self.respondErrorStr(500, err.message || err)
719
    }
720
    self.res.writeHead(200, {
721
      'Cache-Control': 'public, max-age=315360000',
722
      'Content-Length': size,
723
      'etag': id
724
    })
725
    pull(
726
      readBlob,
727
      toPull(self.res, function (err) {
728
        if (err) console.error('[npm-registry]', err)
729
      })
730
    )
731
  })
732
}
733
734
Req.prototype.serveMsg = function (id) {
735
  var self = this
736
  try { id = decodeURIComponent(id) }
737
  catch (e) {}
738
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
739
  self.server.sbot.get(id, function (err, value) {
740
    if (err) return self.respondError(500, err.message || err)
741
    var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
742
    self.res.writeHead(200, {
743
      'Content-Type': 'application/json',
744
      'Cache-Control': 'public, max-age=315360000',
745
      'Content-Length': out.length,
746
      'etag': id
747
    })
748
    self.res.end(out)
749
  })
750
}
751
752
function decodeName(name) {
753
  var parts = String(name).replace(/\.tgz$/, '').split(':')
754
  return {
755
    name: parts[1],
756
    version: parts[2],
757
    distTag: parts[3],
758
  }
759
}
760
761
Req.prototype.getMsgMentions = function (name) {
762
  return pull(
763
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
764
    // decryption could be done here
765
    pull.map(function (msg) {
766
      var c = msg.value && msg.value.content
767
      if (!c.mentions || !Array.isArray(c.mentions)) return []
768
      return c.mentions.map(function (mention) {
769
        return {
770
          name: mention.name,
771
          size: mention.size,
772
          link: mention.link,
773
          author: msg.value.author,
774
          ts: msg.value.timestamp,
775
        }
776
      })
777
    }),
778
    pull.flatten(),
779
    pull.filter(typeof name === 'string' ? function (link) {
780
      return link.name === name
781
    } : name && name.$prefix ? function (link) {
782
      return link.name.substr(0, name.$prefix.length) === name.$prefix
783
    } : function () {
784
      throw new TypeError('unsupported name filter')
785
    })
786
  )
787
}
788
789
Req.prototype.getMentions = function (name) {
790
  var useMsgMentions = this.headMsgIds
791
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
792
  if (useServerMentions && !this.server.sbot.links2) {
793
    return this.headMsgPlus
794
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
795
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
796
  }
797
  return cat([
798
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
799
    useServerMentions ? this.server.getMentions(name) : pull.empty()
800
  ])
801
}
802
803
Req.prototype.getMentionLinks = function (blobId) {
804
  return pull(
805
    this.headMsgIds
806
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
807
      : this.server.sbot.links({
808
        dest: blobId,
809
        rel: 'mentions',
810
        values: true,
811
      }),
812
    // decryption could be done here
813
    pull.map(function (msg) {
814
      var c = msg.value && msg.value.content
815
      return c && Array.isArray(c.mentions) && c.mentions || []
816
    }),
817
    pull.flatten(),
818
    pull.filter(function (link) {
819
      return link && link.link === blobId
820
    })
821
  )
822
}
823
824
Req.prototype.servePkg = function (pathname) {
825
  var self = this
826
  var parts = pathname.split('/')
827
  var pkgName = parts.shift().replace(/%2f/i, '/')
828
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
829
  var spec = parts.shift()
830
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
831
  if (parts.length > 0) return this.respondError(404)
832
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
833
  var obj = {
834
    _id: pkgName,
835
    name: pkgName,
836
    'dist-tags': {},
837
    versions: {},
838
    time: {}
839
  }
840
  var distTags = {/* <tag>: {version, ts}*/}
841
  pull(
842
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
843
    pull.drain(function (mention) {
844
      var data = decodeName(mention.name)
845
      if (!data.version) return
846
      if (data.distTag) {
847
        var tag = distTags[data.distTag]
848
        if (!tag || mention.ts > tag.ts) {
849
          /* TODO: sort by causal order instead of only by timestamps */
850
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
851
        }
852
      }
853
      obj.versions[data.version] = {
854
        author: {
855
          url: mention.author
856
        },
857
        name: pkgName,
858
        version: data.version,
859
        dist: self.blobDist(mention.link)
860
      }
861
      var ts = new Date(mention.ts)
862
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
863
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
864
      obj.time[data.version] = ts.toISOString()
865
    }, function (err) {
866
      if (err) return self.respondError(500, err.stack || err)
867
      for (var tag in distTags) {
868
        obj['dist-tags'][tag] = distTags[tag].version
869
      }
870
      if (spec) resolveSpec()
871
      else if (self.fetchAll) resolveAll()
872
      else resolved()
873
    })
874
  )
875
  function resolveSpec() {
876
    var version = obj['dist-tags'][spec]
877
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
878
    obj = obj.versions[version]
879
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
880
    self.populatePackageJson(obj, function (err, pkg) {
881
      if (err) return resolved(err)
882
      obj = pkg || obj
883
      resolved()
884
    })
885
  }
886
  function resolveVersion(version, cb) {
887
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
888
      if (err) return cb(err)
889
      if (pkg) obj.versions[version] = pkg
890
      if (pkg && pkg.license && !obj.license) obj.license = pkg.license
891
      cb()
892
    })
893
  }
894
  function resolveAll() {
895
    var done = multicb()
896
    for (var version in obj.versions) {
897
      resolveVersion(version, done())
898
    }
899
    done(resolved)
900
  }
901
  function resolved(err) {
902
    if (err) return self.respondError(500, err.stack || err)
903
    self.respond(200, obj)
904
  }
905
}
906
907
Req.prototype.servePrebuild = function (name) {
908
  var self = this
909
  var getMention = self.getMentions('prebuild:' + name)
910
  var blobsByAuthor = {/* <author>: BlobId */}
911
  pull(getMention, pull.drain(function (link) {
912
    blobsByAuthor[link.author] = link.link
913
  }, function (err) {
914
    if (err) return self.respondError(500, err.stack || err)
915
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
916
    var blobId
917
    for (var feed in blobsByAuthor) {
918
      var blob = blobId = blobsByAuthor[feed]
919
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
920
      feeds.push(feed)
921
    }
922
    switch (Object.keys(authorsByLink).length) {
923
      case 0:
924
        return self.respondError(404, 'Not Found')
925
      case 1:
926
        return self.serveBlob(blobId)
927
      default:
928
        return self.respond(300, {choices: authorsByLink})
929
    }
930
  }))
931
}
932
933
var localhosts = {
934
  '::1': true,
935
  '127.0.0.1': true,
936
  '::ffff:127.0.0.1': true,
937
}
938
939
Req.prototype.publishPkg = function (pkgName) {
940
  var self = this
941
  var remoteAddress = self.req.socket.remoteAddress
942
  if (!(remoteAddress in localhosts)) {
943
    return self.respondError(403, 'You may not publish as this user.')
944
  }
945
946
  var chunks = []
947
  self.req.on('data', function (data) {
948
    chunks.push(data)
949
  })
950
  self.req.on('end', function () {
951
    var data
952
    try {
953
      data = JSON.parse(Buffer.concat(chunks))
954
    } catch(e) {
955
      return self.respondError(400, e.stack)
956
    }
957
    return self.publishPkg2(pkgName, data || {})
958
  })
959
}
960
961
Req.prototype.publishPkg2 = function (name, data) {
962
  var self = this
963
  if (data.users) console.trace('[npm-registry] users property is not supported')
964
  var attachments = data._attachments || {}
965
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
966
  var done = multicb()
967
  function addAttachmentAsBlob(filename, cb) {
968
    var data = attachments[filename].data
969
    var tarball = new Buffer(data, 'base64')
970
    var length = attachments[filename].length
971
    if (length && length !== tarball.length) return self.respondError(400,
972
      'Length mismatch for attachment \'' + filename + '\'')
973
    self.server.sbot.blobs.add(function (err, id) {
974
      if (err) return cb(err)
975
      self.blobsToPush.push(id)
976
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
977
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
978
      cb()
979
    })(pull.once(tarball))
980
  }
981
  for (var filename in attachments) {
982
    addAttachmentAsBlob(filename, done())
983
  }
984
  done(function (err) {
985
    if (err) return self.respondError(500, err.stack || err)
986
    try {
987
      self.publishPkg3(name, data, links)
988
    } catch(e) {
989
      self.respondError(500, e.stack || e)
990
    }
991
  })
992
}
993
994
Req.prototype.publishPkg3 = function (name, data, links) {
995
  var self = this
996
  var versions = data.versions || {}
997
  var linksByVersion = {/* <version>: link */}
998
999
  // associate tarball blobs with versions
1000
  for (var version in versions) {
1001
    var pkg = versions[version]
1002
    if (!pkg) return self.respondError(400, 'Bad package object')
1003
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
1004
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
1005
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
1006
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
1007
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
1008
    var filename = m[1]
1009
    var link = links[filename]
1010
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
1011
    // TODO?: try to find missing tarball mentioned in other messages
1012
    if (pkg.version && pkg.version !== version)
1013
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
1014
    linksByVersion[version] = link
1015
    link.version = version
1016
    link.license = pkg.license
1017
    link.dependencies = pkg.dependencies || {}
1018
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
1019
  }
1020
1021
  // associate blobs with dist-tags
1022
  var tags = data['dist-tags'] || {}
1023
  for (var tag in tags) {
1024
    var version = tags[tag]
1025
    var link = linksByVersion[version]
1026
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
1027
    // TODO?: support setting dist-tag without version,
1028
    // by looking up a tarball blob for the version
1029
    link.tag = tag
1030
  }
1031
1032
  // compute blob links to publish
1033
  var mentions = []
1034
  for (var filename in links) {
1035
    var link = links[filename] || {}
1036
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
1037
    mentions.push({
1038
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
1039
      link: link.link,
1040
      size: link.size,
1041
      shasum: link.shasum,
1042
      license: link.license,
1043
      dependencies: link.dependencies,
1044
      bundledDependencies: link.bundledDependencies,
1045
    })
1046
  }
1047
  return self.publishPkgs(mentions)
1048
}
1049
1050
Req.prototype.publishPkgs = function (mentions) {
1051
  var self = this
1052
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
1053
    if (err) return self.respondError(500, err.stack || err)
1054
    self.server.pushBlobs(self.blobsToPush, function (err) {
1055
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
1056
      self.respond(201)
1057
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
1058
    })
1059
  })
1060
}
1061
1062
Req.prototype.populatePackageJson = function (obj, cb) {
1063
  var self = this
1064
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1065
  var deps, bundledDeps, shasum
1066
1067
  // look for dependencies in links.
1068
  // then fallback to getting it from the tarball blob
1069
1070
  pull(
1071
    self.getMentionLinks(blobId),
1072
    pull.drain(function (link) {
1073
      if (link.dependencies) deps = link.dependencies
1074
      if (link.shasum) shasum = link.shasum
1075
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1076
      // how to handle multiple assignments of dependencies to a package?
1077
    }, function (err) {
1078
      if (err) return cb(new Error(err.stack || err))
1079
      if (deps && (shasum || !self.needShasum)) {
1080
        // assume that the dependencies in the links to the blob are
1081
        // correct.
1082
        obj.dependencies = deps
1083
        obj.bundledDependencies = bundledDeps
1084
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1085
        next(obj)
1086
      } else {
1087
        // get dependencies from the tarball
1088
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1089
          if (err) return cb(err)
1090
          pkg.dist = obj.dist
1091
          pkg.dist.shasum = pkg._shasum
1092
          pkg.author = pkg.author || obj.author
1093
          pkg.version = pkg.version || obj.version
1094
          pkg.name = pkg.name || obj.name
1095
          pkg.license = pkg.license || obj.license
1096
          next(pkg)
1097
        })
1098
      }
1099
    })
1100
  )
1101
  function next(pkg) {
1102
    var feedId = obj.author.url
1103
    self.server.getFeedName(feedId, function (err, name) {
1104
      if (err) console.error('[npm-registry]', err.stack || err), name = ''
1105
      pkg._npmUser = {
1106
        email: feedId,
1107
        name: name
1108
      }
1109
      cb(null, pkg)
1110
    })
1111
  }
1112
}
1113
1114
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1115
  var self = this
1116
  getBlob(sbot, id, function (err, readBlob) {
1117
    if (err) return cb(err)
1118
    cb = once(cb)
1119
    var extract = tar.extract()
1120
    var pkg, shasum
1121
    extract.on('entry', function (header, stream, next) {
1122
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1123
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1124
          if (err) return cb(err)
1125
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1126
          catch(e) { return cb(e) }
1127
          next()
1128
        }))
1129
      } else {
1130
        stream.on('end', next)
1131
        stream.resume()
1132
      }
1133
    })
1134
    extract.on('finish', function () {
1135
      pkg._shasum = shasum
1136
      cb(null, pkg)
1137
    })
1138
    pull(
1139
      readBlob,
1140
      hash('sha1', 'hex', function (err, sum) {
1141
        if (err) return cb(err)
1142
        shasum = sum
1143
      }),
1144
      toPull(zlib.createGunzip()),
1145
      toPull(extract)
1146
    )
1147
  })
1148
}
1149