git ssb

1
var http = require('http')
2
var os = require('os')
3
var path = require('path')
4
var fs = require('fs')
5
var crypto = require('crypto')
6
var pkg = require('./package')
7
var semver = require('semver')
8
var toPull = require('stream-to-pull-stream')
9
var pull = require('pull-stream')
10
var tar = require('tar-stream')
11
var zlib = require('zlib')
12
var cat = require('pull-cat')
13
var hash = require('pull-hash')
14
var multicb = require('multicb')
15
var memo = require('asyncmemo')
16
var lru = require('hashlru')
17
18
function escapeHTML(str) {
19
  return String(str)
20
    .replace(/</g, '&lt;')
21
    .replace(/>/g, '&gt;')
22
}
23
24
function onceify(fn, self) {
25
  var cbs = [], err, data
26
  return function (cb) {
27
    if (fn) {
28
      cbs.push(cb)
29
      fn.call(self, function (_err, _data) {
30
        err = _err, data = _data
31
        var _cbs = cbs
32
        cbs = null
33
        while (_cbs.length) _cbs.shift()(err, data)
34
      })
35
      fn = null
36
    } else if (cbs) {
37
      cbs.push(cb)
38
    } else {
39
      cb(err, data)
40
    }
41
  }
42
}
43
44
function once(cb) {
45
  var done
46
  return function (err, result) {
47
    if (done) {
48
      if (err) console.trace(err)
49
    } else {
50
      done = true
51
      cb(err, result)
52
    }
53
  }
54
}
55
56
function npmLogin(registryAddress, cb) {
57
  var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
58
  var filename = path.join(os.homedir(), '.npmrc')
59
  fs.readFile(filename, 'utf8', function (err, data) {
60
    if (err && err.code === 'ENOENT') data = ''
61
    else if (err) return cb(new Error(err.stack))
62
    var lines = data ? data.split('\n') : []
63
    if (lines.indexOf(tokenLine) > -1) return cb()
64
    var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
65
    var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
66
    fs.appendFile(filename, line, cb)
67
  })
68
}
69
70
function formatHost(host) {
71
  return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
72
}
73
74
exports.name = 'npm-registry'
75
exports.version = '1.0.0'
76
exports.manifest = {
77
  getAddress: 'async'
78
}
79
exports.init = function (sbot, config) {
80
  var conf = config.npm || {}
81
  var port = conf.port || 8043
82
  var host = conf.host || null
83
  var autoAuth = conf.autoAuth !== false
84
85
  var server = http.createServer(exports.respond(sbot, config))
86
  var getAddress = onceify(function (cb) {
87
    server.on('error', cb)
88
    server.listen(port, host, function () {
89
      server.removeListener('error', cb)
90
      var regHost = formatHost(host || 'localhost')
91
      var regPort = this.address().port
92
      var regUrl = 'http://' + regHost + ':' + regPort + '/'
93
      if (autoAuth) npmLogin(regUrl, next)
94
      else next()
95
      function next(err) {
96
        cb(err, regUrl)
97
      }
98
    })
99
    sbot.on('close', function () {
100
      server.close()
101
    })
102
  })
103
104
  getAddress(function (err, addr) {
105
    if (err) return console.error(err)
106
    console.log('[npm-registry] Listening on ' + addr)
107
  })
108
109
  return {
110
    getAddress: getAddress
111
  }
112
}
113
114
exports.respond = function (sbot, config) {
115
  var reg = new SsbNpmRegistryServer(sbot, config)
116
  return function (req, res) {
117
    new Req(reg, req, res).serve()
118
  }
119
}
120
121
function publishMsg(sbot, value, cb) {
122
  var gotExpectedPrevious = false
123
  sbot.publish(value, function next(err, msg) {
124
    if (err && /^expected previous:/.test(err.message)) {
125
      // retry once on this error
126
      if (gotExpectedPrevious) return cb(err)
127
      gotExpectedPrevious = true
128
      return sbot.publish(value, next)
129
    }
130
    cb(err, msg)
131
  })
132
}
133
134
function getDependencyBranches(sbot, link, cb) {
135
  var id = link.link
136
137
  // get ids of heads of tree of dependencyBranch message links to include all
138
  // the dependencies of the given tarball id.
139
  var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
140
  var msgs = {}
141
  var branched = {}
142
  var blobs = {}
143
144
  function addPkgById(id, cb) {
145
    if (blobs[id]) return cb()
146
    blobs[id] = true
147
    getPackageJsonCached(id, function (err, pkg) {
148
      if (err) return cb(err)
149
      var done = multicb()
150
      for (var name in pkg.dependencies || {}) {
151
        addPkgBySpec(name, pkg.dependencies[name], done())
152
      }
153
      done(cb)
154
    })
155
  }
156
157
  function addPkgBySpec(name, spec, cb) {
158
    var versions = {}
159
    var distTags = {}
160
    pull(
161
      getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
162
      pull.filter(mentionMatches(null, name, spec)),
163
      pull.map(function (mention) {
164
        // query to get the messages since links2 does not include the msg id
165
        return sbot.links({
166
          source: mention.author,
167
          dest: mention.link,
168
          rel: 'mentions',
169
          values: true,
170
        })
171
      }),
172
      pull.flatten(),
173
      pull.drain(function (msg) {
174
        var c = msg && msg.value && msg.value.content
175
        if (!c || !Array.isArray(c.mentions)) return
176
        c.mentions.forEach(function (link) {
177
          var data = link && link.name && decodeName(link.name)
178
          if (!data || data.name !== name) return
179
          versions[data.version] = {msg: msg, mention: link, mentionData: data}
180
          if (data.distTag) distTags[data.distTag] = data.version
181
        })
182
      }, function (err) {
183
        if (err) return cb(err)
184
        var version = distTags[spec]
185
          || semver.maxSatisfying(Object.keys(versions), spec)
186
        var item = versions[version]
187
        if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
188
        addDepBranchMsg(item.msg)
189
        // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
190
        addPkgById(item.mention.link, cb)
191
      })
192
    )
193
  }
194
195
  function addDepBranchMsg(msg) {
196
    var c = msg && msg.value && msg.value.content
197
    if (!c) return
198
    if (msgs[msg.key]) return
199
    msgs[msg.key] = msg.value
200
    if (Array.isArray(c.dependencyBranch)) {
201
      for (var k = 0; k < c.dependencyBranch.length; k++) {
202
        branched[c.dependencyBranch[k]] = true
203
      }
204
    }
205
    if (Array.isArray(c.versionBranch)) {
206
      for (var k = 0; k < c.versionBranch.length; k++) {
207
        branched[c.versionBranch[k]] = true
208
      }
209
    }
210
  }
211
212
  function addPkgsByName(name, cb) {
213
    var data = decodeName(name)
214
    pull(
215
      getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
216
      pull.map(function (mention) {
217
        return packageLinks(sbot, mention.author, mention.link, data.name)
218
      }),
219
      pull.flatten(),
220
      pull.drain(addDepBranchMsg, cb)
221
    )
222
  }
223
224
  var done = multicb()
225
  addPkgsByName(link.name, done())
226
  addPkgById(id, done())
227
  done(function (err) {
228
    if (err) return cb(err)
229
    var ids = []
230
    for (var key in msgs) {
231
      if (!branched[key]) ids.push(key)
232
    }
233
    cb(null, ids)
234
  })
235
}
236
237
function packageLinks(sbot, feed, id, name, spec) {
238
  var matches = mentionMatches(id, name, spec)
239
  return pull(
240
    sbot.links({
241
      source: feed,
242
      dest: id,
243
      rel: 'mentions',
244
      values: true,
245
    }),
246
    pull.filter(function (msg) {
247
      var c = msg && msg.value && msg.value.content
248
      return c && Array.isArray(c.mentions) && c.mentions.some(matches)
249
    })
250
  )
251
}
252
253
function getVersionBranches(sbot, link, cb) {
254
}
255
256
// For each dependency that is not a bundledDependency, get message ids for
257
// that dependency name + version.
258
function publishSingleMention(sbot, mention, cb) {
259
  if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
260
  // Calculate dependencyBranch message ids.
261
  var value = {
262
    type: 'npm-packages',
263
    mentions: [mention]
264
  }
265
  var done = multicb({pluck: 1, spread: true})
266
  getDependencyBranches(sbot, mention, function (err, branches) {
267
    if (err) return cb(err)
268
    value.dependencyBranch = branches || undefined
269
    publishMsg(sbot, value, cb)
270
  })
271
}
272
273
function publishMentions(sbot, mentions, cb) {
274
  // console.error("publishing %s mentions", mentions.length)
275
  if (mentions.length === 0) return cb(new Error('Empty mentions list'))
276
  // if it is just one mention, fetch and add useful metadata
277
  if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
278
  publishMsg(sbot, {
279
    type: 'npm-packages',
280
    mentions: mentions,
281
  }, cb)
282
}
283
284
exports.publishPkgMentions = function (sbot, mentions, cb) {
285
  // try to fit the mentions into as few messages as possible,
286
  // while fitting under the message size limit.
287
  var msgs = []
288
  ;(function next(i, chunks) {
289
    if (i >= mentions.length) return cb(null, msgs)
290
    var chunkLen = Math.ceil(mentions.length / chunks)
291
    publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
292
      if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
293
      if (err && msgs.length) return onPartialPublish(err)
294
      if (err) return cb(err)
295
      msgs.push(msg)
296
      next(i + chunkLen, chunks)
297
    })
298
  })(0, 1)
299
  function onPartialPublish(err) {
300
    var remaining = mentions.length - i
301
    return cb(new Error('Published messages ' +
302
      msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
303
      'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
304
  }
305
}
306
307
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
308
  cb = once(cb)
309
  var waiting = 0
310
  var expandedMentions = mentions.map(function (link) {
311
    var id = link && link.link
312
    if (!id) return link
313
    waiting++
314
    var newLink = {}
315
    for (var k in link) newLink[k] = link[k]
316
    getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
317
      if (err) return cb(err)
318
      for (var k in props) newLink[k] = pkg[k]
319
      if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
320
      if (!--waiting) next()
321
    })
322
    return newLink
323
  })
324
  if (!waiting) next()
325
  function next() {
326
    cb(null, expandedMentions)
327
  }
328
}
329
330
function SsbNpmRegistryServer(sbot, config) {
331
  this.sbot = sbot
332
  this.config = config
333
  this.npmConfig = config.npm || {}
334
  this.host = this.npmConfig.host || 'localhost'
335
  this.fetchAll = this.npmConfig.fetchAll
336
  this.needShasum = this.npmConfig.needShasum
337
  this.wsPort = config.ws && Number(config.ws.port) || '8989'
338
  this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
339
    + this.wsPort + '/blobs/get/'
340
  this.getMsg = memo({cache: lru(100)}, this.getMsg)
341
}
342
343
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
344
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
345
346
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
347
  var self = this
348
  if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
349
  ;(function next(i) {
350
    if (i >= ids.length) return cb()
351
    self.sbot.blobs.push(ids[i], function (err) {
352
      if (err) return cb(err)
353
      next(i+1)
354
    })
355
  })(0)
356
}
357
358
function getBlob(sbot, id, cb) {
359
  var blobs = sbot.blobs
360
  blobs.size(id, function (err, size) {
361
    if (typeof size === 'number') cb(null, blobs.get(id))
362
    else blobs.want(id, function (err, got) {
363
      if (err) cb(err)
364
      else if (!got) cb('missing blob ' + id)
365
      else cb(null, blobs.get(id))
366
    })
367
  })
368
}
369
370
SsbNpmRegistryServer.prototype.blobDist = function (id) {
371
  var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
372
  if (!m) throw new Error('bad blob id: ' + id)
373
  return {
374
    integrity: m[2] + '-' + m[1],
375
    tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
376
  }
377
}
378
379
function getMentions(links2, name) {
380
  return links2.read({
381
    query: [
382
      {$filter: {rel: ['mentions', name, {$gt: true}]}},
383
      {$filter: {dest: {$prefix: '&'}}},
384
      {$map: {
385
        name: ['rel', 1],
386
        size: ['rel', 2],
387
        link: 'dest',
388
        author: 'source',
389
        ts: 'ts'
390
      }}
391
    ]
392
  })
393
}
394
395
function mentionMatches(id, name, spec) {
396
  return function (mention) {
397
    var data = mention
398
      && (!id || id === mention.link)
399
      && mention.name
400
      && decodeName(mention.name)
401
    return data
402
      && data.name === name
403
      && (spec ? semver.satisfies(data.version, spec) : true)
404
  }
405
}
406
407
SsbNpmRegistryServer.prototype.getMentions = function (name) {
408
  if (!this.sbot.links2) return pull.empty()
409
  return getMentions(this.sbot.links2, name)
410
}
411
412
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
413
  if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
414
  else this.sbot.get(id, function (err, value) {
415
    if (err) return cb(err)
416
    cb(null, {key: id, value: value})
417
  })
418
}
419
420
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
421
  var self = this
422
  var stack = heads.slice()
423
  var seen = {}
424
  return function (abort, cb) {
425
    if (abort) return cb(abort)
426
    for (var id; stack.length && seen[id = stack.pop()];);
427
    if (!id) return cb(true)
428
    seen[id] = true
429
    // TODO: use DFS
430
    self.getMsg(id, function (err, msg) {
431
      if (err) return cb(new Error(err.stack || err))
432
      var c = msg && msg.value && msg.value.content
433
      var links = c && c[prop]
434
      if (Array.isArray(links)) {
435
        stack.push.apply(stack, links)
436
      } else if (links) {
437
        stack.push(links)
438
      }
439
      cb(null, msg)
440
    })
441
  }
442
}
443
444
function Req(server, req, res) {
445
  this.server = server
446
  this.req = req
447
  this.res = res
448
  this.blobsToPush = []
449
  this.fetchAll = server.fetchAll != null ? server.fetchAll : true
450
  var ua = this.req.headers['user-agent']
451
  var m = /\bnpm\/([0-9]*)/.exec(ua)
452
  var npmVersion = m && m[1]
453
  this.needShasum = server.needShasum != null ? server.needShasum :
454
    (npmVersion && npmVersion < 5)
455
}
456
457
Req.prototype.serve = function () {
458
  console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
459
  this.res.setTimeout(0)
460
  var pathname = this.req.url.replace(/\?.*/, '')
461
  var m
462
  if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
463
    try {
464
      this.headMsgIds = decodeURIComponent(m[2]).split(',')
465
      this.headMsgPlus = !!m[1]
466
      // ^ means also include packages published after the head message id
467
    } catch(e) {
468
      return this.respondError(400, e.stack || e)
469
    }
470
    pathname = m[3]
471
  }
472
  if (pathname === '/') return this.serveHome()
473
  if (pathname === '/-/whoami') return this.serveWhoami()
474
  if (pathname === '/-/ping') return this.respond(200, true)
475
  if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
476
  if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
477
  if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
478
  return this.respond(404)
479
}
480
481
Req.prototype.respond = function (status, message) {
482
  this.res.writeHead(status, {'content-type': 'application/json'})
483
  this.res.end(message && JSON.stringify(message, 0, 2))
484
}
485
486
Req.prototype.respondError = function (status, message) {
487
  this.respond(status, {error: message})
488
}
489
490
Req.prototype.serveHome = function () {
491
  var self = this
492
  self.res.writeHead(200, {'content-type': 'text/html'})
493
  var port = 8044
494
  self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
495
    '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
496
    '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
497
    '</body></html>')
498
}
499
500
Req.prototype.serveWhoami = function () {
501
  var self = this
502
  self.server.sbot.whoami(function (err, feed) {
503
    if (err) return self.respondError(err.stack || err)
504
    self.respond(200, {username: feed.id})
505
  })
506
}
507
508
Req.prototype.serveUser1 = function () {
509
  this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
510
}
511
512
function decodeName(name) {
513
  var parts = String(name).replace(/\.tgz$/, '').split(':')
514
  return {
515
    name: parts[1],
516
    version: parts[2],
517
    distTag: parts[3],
518
  }
519
}
520
521
Req.prototype.getMsgMentions = function (name) {
522
  return pull(
523
    this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
524
    // decryption could be done here
525
    pull.map(function (msg) {
526
      var c = msg.value && msg.value.content
527
      if (!c.mentions || !Array.isArray(c.mentions)) return []
528
      return c.mentions.map(function (mention) {
529
        return {
530
          name: mention.name,
531
          size: mention.size,
532
          link: mention.link,
533
          author: msg.value.author,
534
          ts: msg.value.timestamp,
535
        }
536
      })
537
    }),
538
    pull.flatten(),
539
    pull.filter(typeof name === 'string' ? function (link) {
540
      return link.name === name
541
    } : name && name.$prefix ? function (link) {
542
      return link.name.substr(0, name.$prefix.length) === name.$prefix
543
    } : function () {
544
      throw new TypeError('unsupported name filter')
545
    })
546
  )
547
}
548
549
Req.prototype.getMentions = function (name) {
550
  var useMsgMentions = this.headMsgIds
551
  var useServerMentions = !this.headMsgIds || this.headMsgPlus
552
  if (useServerMentions && !this.server.sbot.links2) {
553
    return this.headMsgPlus
554
      ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
555
      : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
556
  }
557
  return cat([
558
    useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
559
    useServerMentions ? this.server.getMentions(name) : pull.empty()
560
  ])
561
}
562
563
Req.prototype.getMentionLinks = function (blobId) {
564
  return pull(
565
    this.headMsgIds
566
      ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
567
      : this.server.sbot.links({
568
        dest: blobId,
569
        rel: 'mentions',
570
        values: true,
571
      }),
572
    // decryption could be done here
573
    pull.map(function (msg) {
574
      var c = msg.value && msg.value.content
575
      return c && Array.isArray(c.mentions) && c.mentions || []
576
    }),
577
    pull.flatten(),
578
    pull.filter(function (link) {
579
      return link && link.link === blobId
580
    })
581
  )
582
}
583
584
Req.prototype.servePkg = function (pathname) {
585
  var self = this
586
  var parts = pathname.split('/')
587
  var pkgName = parts.shift().replace(/%2f/i, '/')
588
  if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
589
  var spec = parts.shift()
590
  if (spec) try { spec = decodeURIComponent(spec) } finally {}
591
  if (parts.length > 0) return this.respondError(404)
592
  if (self.req.method === 'PUT') return self.publishPkg(pkgName)
593
  var obj = {
594
    _id: pkgName,
595
    name: pkgName,
596
    'dist-tags': {},
597
    versions: {}
598
  }
599
  var distTags = {/* <tag>: {version, ts}*/}
600
  pull(
601
    self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
602
    pull.drain(function (mention) {
603
      var data = decodeName(mention.name)
604
      if (!data.version) return
605
      if (data.distTag) {
606
        var tag = distTags[data.distTag]
607
        if (!tag || mention.ts > tag.ts) {
608
          /* TODO: sort by causal order instead of only by timestamps */
609
          distTags[data.distTag] = {ts: mention.ts, version: data.version}
610
        }
611
      }
612
      obj.versions[data.version] = {
613
        author: {
614
          url: mention.author
615
        },
616
        name: pkgName,
617
        version: data.version,
618
        dist: self.server.blobDist(mention.link)
619
      }
620
    }, function (err) {
621
      if (err) return self.respondError(500, err.stack || err)
622
      for (var tag in distTags) {
623
        obj['dist-tags'][tag] = distTags[tag].version
624
      }
625
      if (spec) resolveSpec()
626
      else if (self.fetchAll) resolveAll()
627
      else resolved()
628
    })
629
  )
630
  function resolveSpec() {
631
    var version = obj['dist-tags'][spec]
632
      || semver.maxSatisfying(Object.keys(obj.versions), spec)
633
    obj = obj.versions[version]
634
    if (!obj) return self.respondError(404, 'version not found: ' + spec)
635
    self.populatePackageJson(obj, function (err, pkg) {
636
      if (err) return resolved(err)
637
      obj = pkg || obj
638
      resolved()
639
    })
640
  }
641
  function resolveVersion(version, cb) {
642
    self.populatePackageJson(obj.versions[version], function (err, pkg) {
643
      if (err) return cb(err)
644
      if (pkg) obj.versions[version] = pkg
645
      cb()
646
    })
647
  }
648
  function resolveAll() {
649
    var done = multicb()
650
    for (var version in obj.versions) {
651
      resolveVersion(version, done())
652
    }
653
    done(resolved)
654
  }
655
  function resolved(err) {
656
    if (err) return self.respondError(500, err.stack || err)
657
    self.respond(200, obj)
658
  }
659
}
660
661
Req.prototype.servePrebuild = function (name) {
662
  var self = this
663
  var getMention = self.getMentions('prebuild:' + name)
664
  var blobsByAuthor = {/* <author>: BlobId */}
665
  getMention(null, function next(err, link) {
666
    if (err === true) return done()
667
    if (err) return self.respondError(500, err.stack || err)
668
    blobsByAuthor[link.author] = link.link
669
    getMention(null, next)
670
  })
671
  function done() {
672
    var authorsByLink = {/* <BlobId>: [FeedId...] */}
673
    var blobId
674
    for (var feed in blobsByAuthor) {
675
      var blob = blobId = blobsByAuthor[feed]
676
      var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
677
      feeds.push(feed)
678
    }
679
    switch (Object.keys(authorsByLink).length) {
680
      case 0:
681
        return self.respondError(404, 'Not Found')
682
      case 1:
683
        self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
684
        return self.res.end()
685
      default:
686
        return self.respond(300, {choices: authorsByLink})
687
    }
688
  }
689
}
690
691
var localhosts = {
692
  '::1': true,
693
  '127.0.0.1': true,
694
  '::ffff:127.0.0.1': true,
695
}
696
697
Req.prototype.publishPkg = function (pkgName) {
698
  var self = this
699
  var remoteAddress = self.req.socket.remoteAddress
700
  if (!(remoteAddress in localhosts)) {
701
    return self.respondError(403, 'You may not publish as this user.')
702
  }
703
704
  var chunks = []
705
  self.req.on('data', function (data) {
706
    chunks.push(data)
707
  })
708
  self.req.on('end', function () {
709
    var data
710
    try {
711
      data = JSON.parse(Buffer.concat(chunks))
712
    } catch(e) {
713
      return self.respondError(400, e.stack)
714
    }
715
    return self.publishPkg2(pkgName, data || {})
716
  })
717
}
718
719
Req.prototype.publishPkg2 = function (name, data) {
720
  var self = this
721
  if (data.users) console.trace('[npm-registry] users property is not supported')
722
  var attachments = data._attachments || {}
723
  var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
724
  var done = multicb()
725
  function addAttachmentAsBlob(filename, cb) {
726
    var data = attachments[filename].data
727
    var tarball = new Buffer(data, 'base64')
728
    var length = attachments[filename].length
729
    if (length && length !== tarball.length) return self.respondError(400,
730
      'Length mismatch for attachment \'' + filename + '\'')
731
    self.server.sbot.blobs.add(function (err, id) {
732
      if (err) return cb(err)
733
      self.blobsToPush.push(id)
734
      var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
735
      links[filename] = {link: id, size: tarball.length, shasum: shasum}
736
      cb()
737
    })(pull.once(tarball))
738
  }
739
  for (var filename in attachments) {
740
    addAttachmentAsBlob(filename, done())
741
  }
742
  done(function (err) {
743
    if (err) return self.respondError(500, err.stack || err)
744
    try {
745
      self.publishPkg3(name, data, links)
746
    } catch(e) {
747
      self.respondError(500, e.stack || e)
748
    }
749
  })
750
}
751
752
Req.prototype.publishPkg3 = function (name, data, links) {
753
  var self = this
754
  var versions = data.versions || {}
755
  var linksByVersion = {/* <version>: link */}
756
757
  // associate tarball blobs with versions
758
  for (var version in versions) {
759
    var pkg = versions[version]
760
    if (!pkg) return self.respondError(400, 'Bad package object')
761
    if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
762
    if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
763
    if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
764
    var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
765
    if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
766
    var filename = m[1]
767
    var link = links[filename]
768
    if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
769
    // TODO?: try to find missing tarball mentioned in other messages
770
    if (pkg.version && pkg.version !== version)
771
      return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
772
    linksByVersion[version] = link
773
    link.version = version
774
    link.dependencies = pkg.dependencies || {}
775
    link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
776
  }
777
778
  // associate blobs with dist-tags
779
  var tags = data['dist-tags'] || {}
780
  for (var tag in tags) {
781
    var version = tags[tag]
782
    var link = linksByVersion[version]
783
    if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
784
    // TODO?: support setting dist-tag without version,
785
    // by looking up a tarball blob for the version
786
    link.tag = tag
787
  }
788
789
  // compute blob links to publish
790
  var mentions = []
791
  for (var filename in links) {
792
    var link = links[filename] || {}
793
    if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
794
    mentions.push({
795
      name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
796
      link: link.link,
797
      size: link.size,
798
      shasum: link.shasum,
799
      dependencies: link.dependencies,
800
      bundledDependencies: link.bundledDependencies,
801
    })
802
  }
803
  return self.publishPkgs(mentions)
804
}
805
806
Req.prototype.publishPkgs = function (mentions) {
807
  var self = this
808
  exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
809
    if (err) return self.respondError(500, err.stack || err)
810
    self.server.pushBlobs(self.blobsToPush, function (err) {
811
      if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
812
      self.respond(201)
813
      console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
814
    })
815
  })
816
}
817
818
Req.prototype.populatePackageJson = function (obj, cb) {
819
  var self = this
820
  var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
821
  var deps, bundledDeps, shasum
822
823
  // look for dependencies in links.
824
  // then fallback to getting it from the tarball blob
825
826
  pull(
827
    self.getMentionLinks(blobId),
828
    pull.drain(function (link) {
829
      if (link.dependencies) deps = link.dependencies
830
      if (link.shasum) shasum = link.shasum
831
      bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
832
      // how to handle multiple assignments of dependencies to a package?
833
    }, function (err) {
834
      if (err) return cb(new Error(err.stack || err))
835
      if (deps && (shasum || !self.needShasum)) {
836
        // assume that the dependencies in the links to the blob are
837
        // correct.
838
        obj.dependencies = deps
839
        obj.bundledDependencies = bundledDeps
840
        if (shasum) obj.dist.shasum = obj._shasum = shasum
841
        cb(null, obj)
842
      } else {
843
        // get dependencies from the tarball
844
        getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
845
          if (err) return cb(err)
846
          pkg.dist = obj.dist
847
          pkg.dist.shasum = pkg._shasum
848
          pkg.author = pkg.author || obj.author
849
          pkg.version = pkg.version || obj.version
850
          pkg.name = pkg.name || obj.name
851
          cb(null, pkg)
852
        })
853
      }
854
    })
855
  )
856
}
857
858
function getPackageJsonFromTarballBlob(sbot, id, cb) {
859
  var self = this
860
  getBlob(sbot, id, function (err, readBlob) {
861
    if (err) return cb(err)
862
    cb = once(cb)
863
    var extract = tar.extract()
864
    var pkg, shasum
865
    extract.on('entry', function (header, stream, next) {
866
      if (/^[^\/]*\/package\.json$/.test(header.name)) {
867
        pull(toPull.source(stream), pull.collect(function (err, bufs) {
868
          if (err) return cb(err)
869
          try { pkg = JSON.parse(Buffer.concat(bufs)) }
870
          catch(e) { return cb(e) }
871
          next()
872
        }))
873
      } else {
874
        stream.on('end', next)
875
        stream.resume()
876
      }
877
    })
878
    extract.on('finish', function () {
879
      pkg._shasum = shasum
880
      cb(null, pkg)
881
    })
882
    pull(
883
      readBlob,
884
      hash('sha1', 'hex', function (err, sum) {
885
        if (err) return cb(err)
886
        shasum = sum
887
      }),
888
      toPull(zlib.createGunzip()),
889
      toPull(extract)
890
    )
891
  })
892
}
893