git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
var chunkedBlobs = require('./largeblobs')
18
19
function escapeHTML(str) {
20
  return String(str)
21
    .replace(/</g, '&lt;')
22
    .replace(/>/g, '&gt;')
23
}
24
25
function idToHex(id) {
26
  var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
27
  return Buffer.from(b64, 'base64').toString('hex')
28
}
29
30
function onceify(fn, self) {
31
  var cbs = [], err, data
32
  return function (cb) {
33
    if (fn) {
34
      cbs.push(cb)
35
      fn.call(self, function (_err, _data) {
36
        err = _err, data = _data
37
        var _cbs = cbs
38
        cbs = null
39
        while (_cbs.length) _cbs.shift()(err, data)
40
      })
41
      fn = null
42
    } else if (cbs) {
43
      cbs.push(cb)
44
    } else {
45
      cb(err, data)
46
    }
47
  }
48
}
49
50
function once(cb) {
51
  var done
52
  return function (err, result) {
53
    if (done) {
54
      if (err) console.trace(err)
55
    } else {
56
      done = true
57
      cb(err, result)
58
    }
59
  }
60
}
61
62
function npmLogin(registryAddress, cb) {
63
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
64
  var filename = path.join(os.homedir(), '.npmrc')
65
  fs.readFile(filename, 'utf8', function (err, data) {
66
    if (err && err.code === 'ENOENT') data = ''
67
    else if (err) return cb(new Error(err.stack))
68
    var lines = data ? data.split('\n') : []
69
    if (lines.indexOf(tokenLine) > -1) return cb()
70
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
71
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
72
    fs.appendFile(filename, line, cb)
73
  })
74
}
75
76
function formatHost(host) {
77
  if (host === '::1') return 'localhost'
78
  host = host.replace(/^::ffff:/, '')
79
  return host[0] !== '[' && /:.*:/.test(host) ? '[' + host + ']' : host
80
}
81
82
var wantWarnTime = Number(process.env.WANT_WARN_TIME) || 60e3
83
84
exports.name = 'npm-registry'
85
exports.version = '1.0.0'
86
exports.manifest = {
87
  getAddress: 'async'
88
}
89
exports.init = function (sbot, config) {
90
  var conf = config.npm || {}
91
  var port = conf.port || 8043
92
  var host = conf.host || 'localhost'
93
  var autoAuth = conf.autoAuth !== false
94
  var listenUrl
95
96
  var server = http.createServer(exports.respond(sbot, config))
97
  var getAddress = onceify(function (cb) {
98
    server.on('error', cb)
99
    server.listen(port, host, function () {
100
      server.removeListener('error', cb)
101
      var addr = this.address()
102
      var listenHost = addr.address
103
      var regHost = listenHost === '::' ? '::1' :
104
                    listenHost === '0.0.0.0' ? '127.0.0.1' :
105
                    listenHost
106
      var regUrl = 'http://' + formatHost(regHost) + ':' + addr.port
107
      listenUrl = 'http://' + formatHost(listenHost) + ':' + addr.port
108
      if (autoAuth) npmLogin(regUrl, next)
109
      else next()
110
      function next(err) {
111
        cb(err, regUrl)
112
      }
113
    })
114
    sbot.on('close', function () {
115
      server.close()
116
    })
117
  })
118
119
  /* getAddress called by local RPC is used to discover the local
120
   * ssb-npm-registry address that can be given to local npm clients. However,
121
   * when running the server we output the address the server is listening on,
122
   * to avoid misleading situations like saying listening on localhost but
123
   * actually listening on a wildcard address. */
124
  getAddress(function (err) {
125
    if (err) return console.error('[npm-registry]', err.stack || err)
126
    console.log('[npm-registry] Listening on ' + listenUrl)
127
  })
128
129
  return {
130
    getAddress: getAddress
131
  }
132
}
133
134
exports.respond = function (sbot, config) {
135
  var reg = new SsbNpmRegistryServer(sbot, config)
136
  return function (req, res) {
137
    new Req(reg, req, res).serve()
138
  }
139
}
140
141
function publishMsg(sbot, value, cb) {
142
  var gotExpectedPrevious = false
143
  sbot.publish(value, function next(err, msg) {
144
    if (err && /^expected previous:/.test(err.message)) {
145
      // retry once on this error
146
      if (gotExpectedPrevious) return cb(err)
147
      gotExpectedPrevious = true
148
      return sbot.publish(value, next)
149
    }
150
    cb(err, msg)
151
  })
152
}
153
154
function getDependencyBranches(sbot, id, cb) {
155
  // get ids of heads of tree of dependencyBranch message links to include all
156
  // the dependencies of the given tarball id.
157
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
158
  var msgs = {}
159
  var branched = {}
160
  var blobs = {}
161
162
  function addPkgById(id, cb) {
163
    if (blobs[id]) return cb()
164
    blobs[id] = true
165
    getPackageJsonCached(id, function (err, pkg) {
166
      if (err) return cb(err)
167
      var done = multicb()
168
      for (var name in pkg.dependencies || {}) {
169
        addPkgBySpec(name, pkg.dependencies[name], done())
170
      }
171
      for (var name in pkg.optionalDependencies || {}) {
172
        addPkgBySpec(name, pkg.optionalDependencies[name], done())
173
      }
174
      done(cb)
175
    })
176
  }
177
178
  function addPkgBySpec(name, spec, cb) {
179
    var versions = {}
180
    var distTags = {}
181
    pull(
182
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
183
      pull.filter(mentionMatches(null, name, spec)),
184
      pull.map(function (mention) {
185
        // query to get the messages since links2 does not include the msg id
186
        return sbot.links({
187
          source: mention.author,
188
          dest: mention.link,
189
          rel: 'mentions',
190
          values: true,
191
        })
192
      }),
193
      pull.flatten(),
194
      pull.drain(function (msg) {
195
        var c = msg && msg.value && msg.value.content
196
        if (!c || !Array.isArray(c.mentions)) return
197
        c.mentions.forEach(function (link) {
198
          var data = link && link.name && decodeName(link.name)
199
          if (!data || data.name !== name) return
200
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
201
          if (data.distTag) distTags[data.distTag] = data.version
202
        })
203
      }, function (err) {
204
        if (err) return cb(err)
205
        var version = distTags[spec]
206
          || semver.maxSatisfying(Object.keys(versions), spec)
207
        var item = versions[version]
208
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
209
        msgs[item.msg.key] = item.msg.value
210
        var c = item.msg.value.content
211
        if (Array.isArray(c.dependencyBranch)) {
212
          for (var k = 0; k < c.dependencyBranch.length; k++) {
213
            branched[c.dependencyBranch[k]] = true
214
          }
215
        }
216
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
217
        addPkgById(item.mention.link, cb)
218
      })
219
    )
220
  }
221
222
  addPkgById(id, function (err) {
223
    if (err) return cb(err)
224
    var ids = []
225
    for (var key in msgs) {
226
      if (!branched[key]) ids.push(key)
227
    }
228
    cb(null, ids)
229
  })
230
}
231
232
function packageLinks(sbot, feed, id, name, spec) {
233
  var matches = mentionMatches(id, name, spec)
234
  return pull(
235
    sbot.links({
236
      source: feed,
237
      dest: id,
238
      rel: 'mentions',
239
      values: true,
240
    }),
241
    pull.filter(function (msg) {
242
      var c = msg && msg.value && msg.value.content
243
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
244
    })
245
  )
246
}
247
248
function getVersionBranches(sbot, link, cb) {
249
  var data = decodeName(link.name)
250
  var msgs = {}, branched = {}
251
  pull(
252
    getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
253
    pull.map(function (mention) {
254
      return packageLinks(sbot, mention.author, mention.link, data.name)
255
    }),
256
    pull.flatten(),
257
    pull.drain(function (msg) {
258
      var c = msg && msg.value && msg.value.content
259
      if (!c) return
260
      msgs[msg.key] = msg.value
261
      if (Array.isArray(c.versionBranch)) {
262
        for (var k = 0; k < c.versionBranch.length; k++) {
263
          branched[c.versionBranch[k]] = true
264
        }
265
      }
266
    }, function (err) {
267
      if (err) return cb(err)
268
      var ids = []
269
      for (var key in msgs) {
270
        if (!branched[key]) ids.push(key)
271
      }
272
      cb(null, ids)
273
    })
274
  )
275
}
276
277
// For each dependency that is not a bundledDependency, get message ids for
278
// that dependency name + version.
279
function publishSingleMention(sbot, mention, cb) {
280
  if (!sbot.links2) return cb(new Error('ssb-links plugin is required to publish ssb-npm packages'))
281
  // Calculate dependencyBranch and versionBranch message ids.
282
  var value = {
283
    type: 'npm-packages',
284
    mentions: [mention]
285
  }
286
  var done = multicb({pluck: 1, spread: true})
287
  getDependencyBranches(sbot, mention.link, done())
288
  getVersionBranches(sbot, mention, done())
289
  done(function (err, dependencyBranches, versionBranches) {
290
    if (err) return cb(err)
291
    value.dependencyBranch = dependencyBranches || undefined
292
    value.versionBranch = versionBranches || undefined
293
    publishMsg(sbot, value, cb)
294
  })
295
}
296
297
function publishMentions(sbot, mentions, cb) {
298
  // console.error("publishing %s mentions", mentions.length)
299
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
300
  // if it is just one mention, fetch and add useful metadata
301
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
302
  publishMsg(sbot, {
303
    type: 'npm-packages',
304
    mentions: mentions,
305
  }, cb)
306
}
307
308
exports.publishPkgMentions = function (sbot, mentions, cb) {
309
  // try to fit the mentions into as few messages as possible,
310
  // while fitting under the message size limit.
311
  var msgs = []
312
  ;(function next(i, chunks) {
313
    if (i >= mentions.length) return cb(null, msgs)
314
    var chunkLen = Math.ceil(mentions.length / chunks)
315
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
316
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
317
      if (err && msgs.length) return onPartialPublish(err)
318
      if (err) return cb(err)
319
      msgs.push(msg)
320
      next(i + chunkLen, chunks)
321
    })
322
  })(0, 1)
323
  function onPartialPublish(err) {
324
    var remaining = mentions.length - i
325
    return cb(new Error('Published messages ' +
326
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
327
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
328
  }
329
}
330
331
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
332
  cb = once(cb)
333
  var waiting = 0
334
  var expandedMentions = mentions.map(function (link) {
335
    var id = link && link.link
336
    if (!id) return link
337
    waiting++
338
    var newLink = {}
339
    for (var k in link) newLink[k] = link[k]
340
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
341
      if (err) return cb(err)
342
      for (var k in props) newLink[k] = pkg[k]
343
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
344
      if (!--waiting) next()
345
    })
346
    return newLink
347
  })
348
  if (!waiting) next()
349
  function next() {
350
    cb(null, expandedMentions)
351
  }
352
}
353
354
function SsbNpmRegistryServer(sbot, config) {
355
  this.sbot = sbot
356
  this.config = config
357
  this.npmConfig = config.npm || {}
358
  this.host = this.npmConfig.host || 'localhost'
359
  this.fetchAll = this.npmConfig.fetchAll
360
  this.needShasum = this.npmConfig.needShasum
361
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
362
  this.getFeedName = memo({cache: lru(100)}, this.getFeedName)
363
  if (sbot.ws) {
364
    var wsPort = config.ws && Number(config.ws.port) || '8989'
365
    this.wsUrl = 'http://localhost:' + wsPort
366
  }
367
}
368
369
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
370
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
371
372
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
373
  var self = this
374
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
375
  ;(function next(i) {
376
    if (i >= ids.length) return cb()
377
    self.sbot.blobs.push(ids[i], function (err) {
378
      if (err) return cb(err)
379
      next(i+1)
380
    })
381
  })(0)
382
}
383
384
function getBlob(sbot, id, cb) {
385
  var blobs = sbot.blobs
386
  blobs.size(id, function (err, size) {
387
    if (err && err.code !== 'ENOENT') return cb(err)
388
    if (typeof size === 'number') return cb(null, blobs.get(id), size)
389
    var chunks = chunkedBlobs[id]
390
    if (chunks) return getLargeBlob(sbot, chunks, cb)
391
    var timeout = wantWarnTime > 0 && setTimeout(function () {
392
      console.error('Blob taking a long time to fetch:', id)
393
    }, wantWarnTime)
394
    blobs.want(id, function (err, got) {
395
      if (timeout) clearTimeout(timeout)
396
      if (err) cb(err)
397
      else if (!got) cb('missing blob ' + id)
398
      else blobs.size(id, function (err, size) {
399
        if (err) return cb(err)
400
        cb(null, blobs.get(id), size)
401
      })
402
    })
403
  })
404
}
405
406
function getLargeBlob(sbot, ids, cb) {
407
  var done = multicb({pluck: 1})
408
  var totalSize = 0
409
  ids.forEach(function (id) {
410
    var cb = done()
411
    getBlob(sbot, id, function (err, source, size) {
412
      if (err) return cb(err)
413
      totalSize += size
414
      cb(null, source)
415
    })
416
  })
417
  done(function (err, sources) {
418
    if (err) return cb(err)
419
    cb(null, cat(sources), totalSize)
420
  })
421
}
422
423
function getMentions(links2, name) {
424
  return links2.read({
425
    query: [
426
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
427
      {$filter: {dest: {$prefix: '&'}}},
428
      {$map: {
429
        name: ['rel', 1],
430
        size: ['rel', 2],
431
        link: 'dest',
432
        author: 'source',
433
        ts: 'ts'
434
      }}
435
    ]
436
  })
437
}
438
439
function mentionMatches(id, name, spec) {
440
  return function (mention) {
441
    var data = mention
442
      && (!id || id === mention.link)
443
      && mention.name
444
      && decodeName(mention.name)
445
    return data
446
      && data.name === name
447
      && (spec ? semver.satisfies(data.version, spec) : true)
448
  }
449
}
450
451
SsbNpmRegistryServer.prototype.getMentions = function (name) {
452
  if (!this.sbot.links2) return pull.empty()
453
  return getMentions(this.sbot.links2, name)
454
}
455
456
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
457
  if (this.sbot.ooo) {
458
    var timeout = wantWarnTime > 0 && setTimeout(function () {
459
      console.error('Message taking a long time to fetch:', id)
460
    }, wantWarnTime)
461
    return this.sbot.ooo.get({id: id, timeout: 0}, timeout ? function (err, msg) {
462
      clearTimeout(timeout)
463
      cb(err, msg)
464
    } : cb)
465
  }
466
  else this.sbot.get(id, function (err, value) {
467
    if (err) console.error('Unable to get message:', id)
468
    if (err) return cb(err)
469
    cb(null, {key: id, value: value})
470
  })
471
}
472
473
SsbNpmRegistryServer.prototype.getFeedName = function (id, cb) {
474
  var self = this
475
  if (self.sbot.names && self.sbot.names.getSignifier) {
476
    self.sbot.names.getSignifier(id, function (err, name) {
477
      if (err || !name) tryLinks()
478
      else cb(null, name)
479
    })
480
  } else {
481
    tryLinks()
482
  }
483
  function tryLinks() {
484
    if (!self.sbot.links) return nope()
485
    pull(
486
      self.sbot.links({
487
        source: id,
488
        dest: id,
489
        rel: 'about',
490
        values: true,
491
        limit: 1,
492
        reverse: true,
493
        keys: false,
494
        meta: false
495
      }),
496
      pull.map(function (value) {
497
        return value && value.content && value.content.name
498
      }),
499
      pull.filter(Boolean),
500
      pull.take(1),
501
      pull.collect(function (err, names) {
502
        if (err || !names.length) return nope()
503
        cb(null, names[0])
504
      })
505
    )
506
  }
507
  function nope() {
508
    cb(null, '')
509
  }
510
}
511
512
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
513
  var self = this
514
  var stack = heads.slice()
515
  var seen = {}
516
  return function (abort, cb) {
517
    if (abort) return cb(abort)
518
    for (var id; stack.length && seen[id = stack.pop()];);
519
    if (!id) return cb(true)
520
    seen[id] = true
521
    // TODO: use DFS
522
    self.getMsg(id, function (err, msg) {
523
      if (err) return cb(new Error(err.stack || err))
524
      var c = msg && msg.value && msg.value.content
525
      var links = c && c[prop]
526
      if (Array.isArray(links)) {
527
        stack.push.apply(stack, links)
528
      } else if (links) {
529
        stack.push(links)
530
      }
531
      cb(null, msg)
532
    })
533
  }
534
}
535
536
function Req(server, req, res) {
537
  this.server = server
538
  this.req = req
539
  this.res = res
540
  this.blobsToPush = []
541
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
542
  var ua = this.req.headers['user-agent']
543
  var m = /\bnpm\/([0-9]*)/.exec(ua)
544
  var npmVersion = m && m[1]
545
  this.needShasum = server.needShasum != null ? server.needShasum :
546
    (npmVersion && npmVersion < 5)
547
  this.baseUrl = this.server.npmConfig.baseUrl
548
  if (this.baseUrl) {
549
    this.baseUrl = this.baseUrl.replace(/\/+$/, '')
550
  } else {
551
    var hostname = req.headers.host
552
      || (formatHost(req.socket.localAddress) + ':' + req.socket.localPort)
553
    this.baseUrl = 'http://' + hostname
554
  }
555
  this.blobBaseUrl = this.baseUrl + '/-/blobs/get/'
556
}
557
558
Req.prototype.serve = function () {
559
  if (process.env.DEBUG) {
560
    console.log(this.req.method, this.req.url, formatHost(this.req.socket.remoteAddress))
561
  }
562
  this.res.setTimeout(0)
563
  var pathname = this.req.url.replace(/\?.*/, '')
564
  var m
565
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
566
    try {
567
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
568
      this.headMsgPlus = !!m[1]
569
      // ^ means also include packages published after the head message id
570
    } catch(e) {
571
      return this.respondError(400, e.stack || e)
572
    }
573
    pathname = m[3]
574
  }
575
  if (pathname === '/') return this.serveHome()
576
  if (pathname === '/robots.txt') return this.serveRobots()
577
  if (pathname === '/-/bootstrap') return this.serveBootstrap()
578
  if (pathname === '/-/whoami') return this.serveWhoami()
579
  if (pathname === '/-/ping') return this.respond(200, true)
580
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
581
  if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
582
  if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
583
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
584
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
585
  return this.respond(404, {error: 'Not found'})
586
}
587
588
Req.prototype.respond = function (status, message) {
589
  this.res.writeHead(status, {'content-type': 'application/json'})
590
  this.res.end(message && JSON.stringify(message, 0, 2))
591
}
592
593
Req.prototype.respondError = function (status, message) {
594
  this.respond(status, {error: message})
595
}
596
597
Req.prototype.respondErrorStr = function (status, err) {
598
  this.res.writeHead(status, {'content-type': 'text/plain'})
599
  this.res.end(err.stack || err)
600
}
601
602
Req.prototype.respondRaw = function (status, body) {
603
  this.res.writeHead(status)
604
  this.res.end(body)
605
}
606
607
Req.prototype.serveHome = function () {
608
  var self = this
609
  self.res.writeHead(200, {'content-type': 'text/html'})
610
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
611
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
612
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
613
    '<p><a href="/-/bootstrap">Bootstrap</a></p>\n' +
614
    '</body></html>')
615
}
616
617
Req.prototype.blobDist = function (id) {
618
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
619
  if (!m) throw new Error('bad blob id: ' + id)
620
  return {
621
    integrity: m[2] + '-' + m[1],
622
    tarball: this.blobBaseUrl + id
623
  }
624
}
625
626
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
627
  var self = this
628
  pull(
629
    self.server.sbot.links({
630
      source: feedId,
631
      dest: blobId,
632
      rel: 'mentions',
633
      values: true,
634
    }),
635
    pull.filter(function (msg) {
636
      var c = msg && msg.value && msg.value.content
637
      return c.type === 'npm-packages'
638
    }),
639
    pull.collect(function (err, msgs) {
640
      if (err) return cb(err)
641
      if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
642
      if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
643
      // TODO: make a smarter decision about which message id to use
644
      cb(null, msgs.pop().key)
645
    })
646
  )
647
}
648
649
Req.prototype.resolvePkg = function (pkgSpec, cb) {
650
  var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
651
  if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
652
  var self = this
653
  var pkgName = m[1]
654
  var spec = m[2] || '*'
655
  var versions = {}
656
  var distTags = {}
657
  pull(
658
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
659
    pull.drain(function (mention) {
660
      var data = decodeName(mention.name)
661
      if (!data.version) return
662
      if (data.distTag) {
663
        distTags[data.distTag] = data.version
664
      }
665
      versions[data.version] = {
666
        author: mention.author,
667
        name: pkgName,
668
        version: data.version,
669
        blobId: mention.link
670
      }
671
    }, function (err) {
672
      if (err) return cb(err)
673
      var version = distTags[spec]
674
        || semver.maxSatisfying(Object.keys(versions), spec)
675
      var item = versions[version]
676
      if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
677
      self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
678
        if (err) return cb(err)
679
        item.msgId = id
680
        cb(null, item)
681
      })
682
    })
683
  )
684
}
685
686
Req.prototype.resolvePkgs = function (specs, cb) {
687
  var done = multicb({pluck: 1})
688
  var self = this
689
  specs.forEach(function (spec) {
690
    self.resolvePkg(spec, done())
691
  })
692
  done(cb)
693
}
694
695
Req.prototype.serveBootstrap = function () {
696
  var self = this
697
  var pkgs = self.server.npmConfig.defaultPkgs || ['ssb-npm']
698
  var postInstallCmd = self.server.npmConfig.postInstallCmd
699
  if (postInstallCmd == null) postInstallCmd = 'sbot server'
700
  var ssbNpmRegistryName = require('./package.json').name
701
  var ssbNpmRegistryVersion = require('./package.json').version
702
  var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
703
  var done = multicb({pluck: 1, spread: true})
704
  self.resolvePkg(ssbNpmRegistrySpec, done())
705
  self.resolvePkgs(pkgs, done())
706
  done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
707
    if (err) return self.respondErrorStr(500, err.stack || err)
708
    if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
709
    var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
710
    var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
711
    var ssbNpmRegistryNameVersion = ssbNpmRegistryPkgInfo.name + '-' + ssbNpmRegistryPkgInfo.version
712
    var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
713
    var globalPkgs = pkgsInfo.map(function (info) {
714
      return info.name + '@' + info.version
715
    }).join(' ')
716
    var npmCmd = 'install -g ' + globalPkgs
717
    var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
718
    var wsUrl = self.baseUrl + '/-'
719
    var tarballLink = wsUrl + '/blobs/get/' + ssbNpmRegistryBlobId
720
721
    var script =
722
      'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
723
      'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
724
      'echo \'' + ssbNpmRegistryBlobHex + '  package.tgz\' | sha256sum -c &&\n' +
725
      'tar xzf package.tgz &&\n' +
726
      './*/bootstrap/bin.js --ws-url ' + wsUrl + ' \\\n' +
727
        pkgMsgs.map(function (id) {
728
          return '  --branch ' + id + ' \\\n'
729
        }).join('') +
730
        '  -- ' + npmCmd +
731
      (postInstallCmd ? ' &&\n' +postInstallCmd : '') + '\n'
732
733
    self.res.writeHead(200, {'Content-type': 'text/plain'})
734
    self.res.end(script)
735
  })
736
}
737
738
Req.prototype.serveRobots = function () {
739
  this.res.writeHead(200, {'Content-type': 'text/plain'})
740
  this.res.end('User-agent: *\nDisallow: /\n')
741
}
742
743
Req.prototype.serveWhoami = function () {
744
  var self = this
745
  self.server.sbot.whoami(function (err, feed) {
746
    if (err) return self.respondError(500, err.stack || err)
747
    self.respond(200, {username: feed.id})
748
  })
749
}
750
751
Req.prototype.serveUser1 = function () {
752
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
753
}
754
755
Req.prototype.serveBlob = function (id) {
756
  var self = this
757
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
758
  getBlob(self.server.sbot, id, function (err, readBlob, size) {
759
    if (err) {
760
      if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
761
      else return self.respondErrorStr(500, err.message || err)
762
    }
763
    self.res.writeHead(200, {
764
      'Cache-Control': 'public, max-age=315360000',
765
      'Content-Length': size,
766
      'etag': id
767
    })
768
    pull(
769
      readBlob,
770
      toPull(self.res, function (err) {
771
        if (err) console.error('[npm-registry]', err)
772
      })
773
    )
774
  })
775
}
776
777
Req.prototype.serveMsg = function (id) {
778
  var self = this
779
  try { id = decodeURIComponent(id) }
780
  catch (e) {}
781
  if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
782
  self.server.getMsg(id, function (err, msg) {
783
    if (err) return self.respondError(500, err.message || err)
784
    var out = Buffer.from(JSON.stringify(msg, null, 2), 'utf8')
785
    self.res.writeHead(200, {
786
      'Content-Type': 'application/json',
787
      'Cache-Control': 'public, max-age=315360000',
788
      'Content-Length': out.length,
789
      'etag': id
790
    })
791
    self.res.end(out)
792
  })
793
}
794
795
function decodeName(name) {
796
  var parts = String(name).replace(/\.tgz$/, '').split(':')
797
  return {
798
    name: parts[1],
799
    version: parts[2],
800
    distTag: parts[3],
801
  }
802
}
803
804
Req.prototype.getMsgMentions = function (name) {
805
  return pull(
806
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
807
    // decryption could be done here
808
    pull.map(function (msg) {
809
      var c = msg.value && msg.value.content
810
      if (!c.mentions || !Array.isArray(c.mentions)) return []
811
      return c.mentions.map(function (mention) {
812
        return {
813
          name: mention.name,
814
          size: mention.size,
815
          link: mention.link,
816
          author: msg.value.author,
817
          ts: msg.value.timestamp,
818
        }
819
      })
820
    }),
821
    pull.flatten(),
822
    pull.filter(typeof name === 'string' ? function (link) {
823
      return link.name === name
824
    } : name && name.$prefix ? function (link) {
825
      return link.name.substr(0, name.$prefix.length) === name.$prefix
826
    } : function () {
827
      throw new TypeError('unsupported name filter')
828
    })
829
  )
830
}
831
832
Req.prototype.getMentions = function (name) {
833
  var useMsgMentions = this.headMsgIds
834
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
835
  if (useServerMentions && !this.server.sbot.links2) {
836
    return this.headMsgPlus
837
      ? pull.error(new Error('ssb-links plugin is needed for ^msgid queries'))
838
      : pull.error(new Error('ssb-links plugin is needed for non-msgid queries'))
839
  }
840
  return cat([
841
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
842
    useServerMentions ? this.server.getMentions(name) : pull.empty()
843
  ])
844
}
845
846
Req.prototype.getMentionLinks = function (blobId) {
847
  return pull(
848
    this.headMsgIds
849
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
850
      : this.server.sbot.links({
851
        dest: blobId,
852
        rel: 'mentions',
853
        values: true,
854
      }),
855
    // decryption could be done here
856
    pull.map(function (msg) {
857
      var c = msg.value && msg.value.content
858
      return c && Array.isArray(c.mentions) && c.mentions || []
859
    }),
860
    pull.flatten(),
861
    pull.filter(function (link) {
862
      return link && link.link === blobId
863
    })
864
  )
865
}
866
867
Req.prototype.servePkg = function (pathname) {
868
  var self = this
869
  var parts = pathname.split('/')
870
  var pkgName = parts.shift().replace(/%2f/i, '/')
871
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
872
  var spec = parts.shift()
873
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
874
  if (parts.length > 0) return this.respondError(404)
875
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
876
  var obj = {
877
    _id: pkgName,
878
    name: pkgName,
879
    'dist-tags': {},
880
    versions: {},
881
    time: {}
882
  }
883
  var distTags = {/* <tag>: {version, ts}*/}
884
  pull(
885
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
886
    pull.drain(function (mention) {
887
      var data = decodeName(mention.name)
888
      if (!data.version) return
889
      if (data.distTag) {
890
        var tag = distTags[data.distTag]
891
        if (!tag || mention.ts > tag.ts) {
892
          /* TODO: sort by causal order instead of only by timestamps */
893
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
894
        }
895
      }
896
      obj.versions[data.version] = {
897
        author: {
898
          url: mention.author
899
        },
900
        name: pkgName,
901
        version: data.version,
902
        dist: self.blobDist(mention.link)
903
      }
904
      var ts = new Date(mention.ts)
905
      if (ts > obj.time.updated || !obj.time.updated) obj.time.updated = ts
906
      if (ts < obj.time.created || !obj.time.created) obj.time.created = ts
907
      obj.time[data.version] = ts.toISOString()
908
    }, function (err) {
909
      if (err) return self.respondError(500, err.stack || err)
910
      for (var tag in distTags) {
911
        obj['dist-tags'][tag] = distTags[tag].version
912
      }
913
      if (spec) resolveSpec()
914
      else if (self.fetchAll) resolveAll()
915
      else resolved()
916
    })
917
  )
918
  function resolveSpec() {
919
    var version = obj['dist-tags'][spec]
920
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
921
    obj = obj.versions[version]
922
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
923
    self.populatePackageJson(obj, function (err, pkg) {
924
      if (err) return resolved(err)
925
      obj = pkg || obj
926
      resolved()
927
    })
928
  }
929
  function resolveVersion(version, cb) {
930
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
931
      if (err) return cb(err)
932
      if (pkg) obj.versions[version] = pkg
933
      if (pkg && pkg.license && !obj.license) obj.license = pkg.license
934
      cb()
935
    })
936
  }
937
  function resolveAll() {
938
    var done = multicb()
939
    for (var version in obj.versions) {
940
      resolveVersion(version, done())
941
    }
942
    done(resolved)
943
  }
944
  function resolved(err) {
945
    if (err) return self.respondError(500, err.stack || err)
946
    self.respond(200, obj)
947
  }
948
}
949
950
Req.prototype.servePrebuild = function (name) {
951
  var self = this
952
  var getMention = self.getMentions('prebuild:' + name)
953
  var blobsByAuthor = {/* <author>: BlobId */}
954
  pull(getMention, pull.drain(function (link) {
955
    blobsByAuthor[link.author] = link.link
956
  }, function (err) {
957
    if (err) return self.respondError(500, err.stack || err)
958
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
959
    var blobId
960
    for (var feed in blobsByAuthor) {
961
      var blob = blobId = blobsByAuthor[feed]
962
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
963
      feeds.push(feed)
964
    }
965
    switch (Object.keys(authorsByLink).length) {
966
      case 0:
967
        return self.respondError(404, 'Not Found')
968
      case 1:
969
        return self.serveBlob(blobId)
970
      default:
971
        return self.respond(300, {choices: authorsByLink})
972
    }
973
  }))
974
}
975
976
var localhosts = {
977
  '::1': true,
978
  '127.0.0.1': true,
979
  '::ffff:127.0.0.1': true,
980
}
981
982
Req.prototype.publishPkg = function (pkgName) {
983
  var self = this
984
  var remoteAddress = self.req.socket.remoteAddress
985
  if (!(remoteAddress in localhosts)) {
986
    return self.respondError(403, 'You may not publish as this user.')
987
  }
988
989
  var chunks = []
990
  self.req.on('data', function (data) {
991
    chunks.push(data)
992
  })
993
  self.req.on('end', function () {
994
    var data
995
    try {
996
      data = JSON.parse(Buffer.concat(chunks))
997
    } catch(e) {
998
      return self.respondError(400, e.stack)
999
    }
1000
    return self.publishPkg2(pkgName, data || {})
1001
  })
1002
}
1003
1004
Req.prototype.publishPkg2 = function (name, data) {
1005
  var self = this
1006
  if (data.users) console.trace('[npm-registry] users property is not supported')
1007
  var attachments = data._attachments || {}
1008
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
1009
  var done = multicb()
1010
  function addAttachmentAsBlob(filename, cb) {
1011
    var data = attachments[filename].data
1012
    var tarball = Buffer.from(data, 'base64')
1013
    var length = attachments[filename].length
1014
    if (length && length !== tarball.length) return self.respondError(400,
1015
      'Length mismatch for attachment \'' + filename + '\'')
1016
    self.server.sbot.blobs.add(function (err, id) {
1017
      if (err) return cb(err)
1018
      self.blobsToPush.push(id)
1019
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
1020
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
1021
      cb()
1022
    })(pull.once(tarball))
1023
  }
1024
  for (var filename in attachments) {
1025
    addAttachmentAsBlob(filename, done())
1026
  }
1027
  done(function (err) {
1028
    if (err) return self.respondError(500, err.stack || err)
1029
    try {
1030
      self.publishPkg3(name, data, links)
1031
    } catch(e) {
1032
      self.respondError(500, e.stack || e)
1033
    }
1034
  })
1035
}
1036
1037
Req.prototype.publishPkg3 = function (name, data, links) {
1038
  var self = this
1039
  var versions = data.versions || {}
1040
  var linksByVersion = {/* <version>: link */}
1041
1042
  // associate tarball blobs with versions
1043
  for (var version in versions) {
1044
    var pkg = versions[version]
1045
    if (!pkg) return self.respondError(400, 'Bad package object')
1046
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
1047
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
1048
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
1049
    var m = /\/-\/([^\/]+|@[^\/]+\/[^\/]+)$/.exec(pkg.dist.tarball)
1050
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
1051
    var filename = m[1]
1052
    var link = links[filename]
1053
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
1054
    // TODO?: try to find missing tarball mentioned in other messages
1055
    if (pkg.version && pkg.version !== version)
1056
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
1057
    linksByVersion[version] = link
1058
    link.version = version
1059
    link.license = pkg.license
1060
    link.dependencies = pkg.dependencies || {}
1061
    link.optionalDependencies = pkg.optionalDependencies
1062
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
1063
  }
1064
1065
  // associate blobs with dist-tags
1066
  var tags = data['dist-tags'] || {}
1067
  for (var tag in tags) {
1068
    var version = tags[tag]
1069
    var link = linksByVersion[version]
1070
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
1071
    // TODO?: support setting dist-tag without version,
1072
    // by looking up a tarball blob for the version
1073
    link.tag = tag
1074
  }
1075
1076
  // compute blob links to publish
1077
  var mentions = []
1078
  for (var filename in links) {
1079
    var link = links[filename] || {}
1080
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
1081
    mentions.push({
1082
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
1083
      link: link.link,
1084
      size: link.size,
1085
      shasum: link.shasum,
1086
      license: link.license,
1087
      dependencies: link.dependencies,
1088
      optionalDependencies: link.optionalDependencies,
1089
      bundledDependencies: link.bundledDependencies,
1090
    })
1091
  }
1092
  return self.publishPkgs(mentions)
1093
}
1094
1095
Req.prototype.publishPkgs = function (mentions) {
1096
  var self = this
1097
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
1098
    if (err) return self.respondError(500, err.stack || err)
1099
    self.server.pushBlobs(self.blobsToPush, function (err) {
1100
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
1101
      self.respond(201)
1102
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
1103
    })
1104
  })
1105
}
1106
1107
Req.prototype.populatePackageJson = function (obj, cb) {
1108
  var self = this
1109
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
1110
  var deps, depsWithOptionalDeps, bundledDeps, shasum, license
1111
1112
  // Combine metadata from mentions of the blob, to construct most accurate
1113
  // metadata, including dependencies info. If needed, fall back to fetching
1114
  // metadata from the tarball blob.
1115
1116
  pull(
1117
    self.getMentionLinks(blobId),
1118
    pull.drain(function (link) {
1119
      if (link.dependencies) deps = link.dependencies
1120
      // if optionalDependencies is present, let the dependencies value
1121
      // override that from other links. because old versions that didn't
1122
      // publish optionalDependencies are missing optionalDependencies from
1123
      // their dependencies.
1124
      if (link.optionalDependencies) depsWithOptionalDeps = {
1125
        deps: link.dependencies,
1126
        optionalDeps: link.optionalDependencies
1127
      }
1128
      if (link.shasum) shasum = link.shasum
1129
      if (link.license) license = link.license
1130
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
1131
      // how to handle multiple assignments of dependencies to a package?
1132
    }, function (err) {
1133
      if (err) return cb(new Error(err.stack || err))
1134
      if (deps && (shasum || !self.needShasum)) {
1135
        // assume that the dependencies in the links to the blob are
1136
        // correct.
1137
        if (depsWithOptionalDeps) {
1138
          obj.dependencies = depsWithOptionalDeps.deps || deps
1139
          obj.optionalDependencies = depsWithOptionalDeps.optionalDeps
1140
        } else {
1141
          obj.dependencies = deps
1142
        }
1143
        obj.bundledDependencies = bundledDeps
1144
        obj.license = license
1145
        if (shasum) obj.dist.shasum = obj._shasum = shasum
1146
        next(obj)
1147
      } else {
1148
        // get dependencies from the tarball
1149
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
1150
          if (err) return cb(err)
1151
          pkg.dist = obj.dist
1152
          pkg.dist.shasum = pkg._shasum
1153
          pkg.author = pkg.author || obj.author
1154
          pkg.version = pkg.version || obj.version
1155
          pkg.name = pkg.name || obj.name
1156
          pkg.license = pkg.license || obj.license
1157
          next(pkg)
1158
        })
1159
      }
1160
    })
1161
  )
1162
  function next(pkg) {
1163
    var feedId = obj.author.url
1164
    self.server.getFeedName(feedId, function (err, name) {
1165
      if (err) console.error('[npm-registry]', err.stack || err), name = ''
1166
      pkg._npmUser = {
1167
        email: feedId,
1168
        name: name
1169
      }
1170
      cb(null, pkg)
1171
    })
1172
  }
1173
}
1174
1175
function getPackageJsonFromTarballBlob(sbot, id, cb) {
1176
  var self = this
1177
  getBlob(sbot, id, function (err, readBlob) {
1178
    if (err) return cb(err)
1179
    cb = once(cb)
1180
    var extract = tar.extract()
1181
    var pkg, shasum
1182
    extract.on('entry', function (header, stream, next) {
1183
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
1184
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
1185
          if (err) return cb(err)
1186
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
1187
          catch(e) { return cb(e) }
1188
          next()
1189
        }))
1190
      } else {
1191
        stream.on('end', next)
1192
        stream.resume()
1193
      }
1194
    })
1195
    extract.on('finish', function () {
1196
      pkg._shasum = shasum
1197
      cb(null, pkg)
1198
    })
1199
    pull(
1200
      readBlob,
1201
      hash('sha1', 'hex', function (err, sum) {
1202
        if (err) return cb(err)
1203
        shasum = sum
1204
      }),
1205
      toPull(zlib.createGunzip()),
1206
      toPull(extract)
1207
    )
1208
  })
1209
}
1210