git ssb

3+

cel / ssb-npm-registry



Tree: 56fcc7a692d12d4ad0c189fd1c538a1cdc8192c7

Files: 56fcc7a692d12d4ad0c189fd1c538a1cdc8192c7 / index.js

32470 bytesRaw
1var http = require('http')
2var os = require('os')
3var path = require('path')
4var fs = require('fs')
5var crypto = require('crypto')
6var pkg = require('./package')
7var semver = require('semver')
8var toPull = require('stream-to-pull-stream')
9var pull = require('pull-stream')
10var tar = require('tar-stream')
11var zlib = require('zlib')
12var cat = require('pull-cat')
13var hash = require('pull-hash')
14var multicb = require('multicb')
15var memo = require('asyncmemo')
16var lru = require('hashlru')
17
18function escapeHTML(str) {
19 return String(str)
20 .replace(/</g, '&lt;')
21 .replace(/>/g, '&gt;')
22}
23
24function onceify(fn, self) {
25 var cbs = [], err, data
26 return function (cb) {
27 if (fn) {
28 cbs.push(cb)
29 fn.call(self, function (_err, _data) {
30 err = _err, data = _data
31 var _cbs = cbs
32 cbs = null
33 while (_cbs.length) _cbs.shift()(err, data)
34 })
35 fn = null
36 } else if (cbs) {
37 cbs.push(cb)
38 } else {
39 cb(err, data)
40 }
41 }
42}
43
44function once(cb) {
45 var done
46 return function (err, result) {
47 if (done) {
48 if (err) console.trace(err)
49 } else {
50 done = true
51 cb(err, result)
52 }
53 }
54}
55
56function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57 // convert a package-lock.json file into data for serving as an npm registry
58 var hasNonBlobUrl = false
59 var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60 var pkgs = {}
61 var queue = [pkgLock, pkgLock.name]
62 while (queue.length) {
63 var dep = queue.shift(), name = queue.shift()
64 if (name) {
65 var pkg = pkgs[name] || (pkgs[name] = {
66 _id: name,
67 name: name,
68 versions: {}
69 })
70 if (dep.version && dep.integrity && dep.resolved) {
71 if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72 pkg.versions[dep.version] = {
73 name: name,
74 version: dep.version,
75 dist: {
76 integrity: dep.integrity,
77 tarball: dep.resolved
78 }
79 }
80 }
81 }
82 if (dep.dependencies) for (var depName in dep.dependencies) {
83 queue.push(dep.dependencies[depName], depName)
84 }
85 }
86 pkgs._hasNonBlobUrl = hasNonBlobUrl
87 return pkgs
88}
89
90function npmLogin(registryAddress, cb) {
91 var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
92 var filename = path.join(os.homedir(), '.npmrc')
93 fs.readFile(filename, 'utf8', function (err, data) {
94 if (err && err.code === 'ENOENT') data = ''
95 else if (err) return cb(new Error(err.stack))
96 var lines = data ? data.split('\n') : []
97 if (lines.indexOf(tokenLine) > -1) return cb()
98 var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
99 var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
100 fs.appendFile(filename, line, cb)
101 })
102}
103
104function formatHost(host) {
105 return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
106}
107
108exports.name = 'npm-registry'
109exports.version = '1.0.0'
110exports.manifest = {
111 getAddress: 'async'
112}
113exports.init = function (sbot, config) {
114 var conf = config.npm || {}
115 var port = conf.port || 8043
116 var host = conf.host || null
117 var autoAuth = conf.autoAuth !== false
118
119 var server = http.createServer(exports.respond(sbot, config))
120 var getAddress = onceify(function (cb) {
121 server.on('error', cb)
122 server.listen(port, host, function () {
123 server.removeListener('error', cb)
124 var regHost = formatHost(host || 'localhost')
125 var regPort = this.address().port
126 var regUrl = 'http://' + regHost + ':' + regPort + '/'
127 if (autoAuth) npmLogin(regUrl, next)
128 else next()
129 function next(err) {
130 cb(err, regUrl)
131 }
132 })
133 sbot.on('close', function () {
134 server.close()
135 })
136 })
137
138 getAddress(function (err, addr) {
139 if (err) return console.error(err)
140 console.log('[npm-registry] Listening on ' + addr)
141 })
142
143 return {
144 getAddress: getAddress
145 }
146}
147
148exports.respond = function (sbot, config) {
149 var reg = new SsbNpmRegistryServer(sbot, config)
150 return function (req, res) {
151 new Req(reg, req, res).serve()
152 }
153}
154
155function publishMsg(sbot, value, cb) {
156 var gotExpectedPrevious = false
157 sbot.publish(value, function next(err, msg) {
158 if (err && /^expected previous:/.test(err.message)) {
159 // retry once on this error
160 if (gotExpectedPrevious) return cb(err)
161 gotExpectedPrevious = true
162 return sbot.publish(value, next)
163 }
164 cb(err, msg)
165 })
166}
167
168function getDependencyBranches(sbot, id, cb) {
169 // get ids of heads of tree of dependencyBranch message links to include all
170 // the dependencies of the given tarball id.
171 var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
172 var msgs = {}
173 var branched = {}
174 var blobs = {}
175
176 function addPkgById(id, cb) {
177 if (blobs[id]) return cb()
178 blobs[id] = true
179 getPackageJsonCached(id, function (err, pkg) {
180 if (err) return cb(err)
181 var done = multicb()
182 for (var name in pkg.dependencies || {}) {
183 addPkgBySpec(name, pkg.dependencies[name], done())
184 }
185 done(cb)
186 })
187 }
188
189 function addPkgBySpec(name, spec, cb) {
190 var done = multicb()
191 pull(
192 getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
193 pull.map(function (mention) {
194 addPkgById(mention.link, done())
195 return packageLinks(sbot, mention.author, mention.link, name, spec)
196 }),
197 pull.flatten(),
198 pull.drain(function (msg) {
199 var c = msg && msg.value && msg.value.content
200 if (!c) return
201 msgs[msg.key] = msg.value
202 if (Array.isArray(c.dependencyBranch)) {
203 for (var k = 0; k < c.dependencyBranch.length; k++) {
204 branched[c.dependencyBranch[k]] = true
205 }
206 }
207 }, function (err) {
208 if (err) return cb(err)
209 done(cb)
210 })
211 )
212 }
213
214 addPkgById(id, function (err) {
215 if (err) return cb(err)
216 var ids = []
217 for (var key in msgs) {
218 if (!branched[key]) ids.push(key)
219 }
220 cb(null, ids)
221 })
222}
223
224function packageLinks(sbot, feed, id, name, spec) {
225 function matches(mention) {
226 var data = mention
227 && mention.link === id
228 && mention.name
229 && decodeName(mention.name)
230 return data
231 && data.name === name
232 && (spec ? semver.satisfies(data.version, spec) : true)
233 }
234 return pull(
235 sbot.links({
236 source: feed,
237 dest: id,
238 rel: 'mentions',
239 values: true,
240 }),
241 pull.filter(function (msg) {
242 var c = msg && msg.value && msg.value.content
243 return c && Array.isArray(c.mentions) && c.mentions.some(matches)
244 })
245 )
246}
247
248function getVersionBranches(sbot, link, cb) {
249 var data = decodeName(link.name)
250 var msgs = {}, branched = {}
251 pull(
252 getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
253 pull.map(function (mention) {
254 return packageLinks(sbot, mention.author, mention.link, data.name)
255 }),
256 pull.flatten(),
257 pull.drain(function (msg) {
258 var c = msg && msg.value && msg.value.content
259 if (!c) return
260 msgs[msg.key] = msg.value
261 if (Array.isArray(c.versionBranch)) {
262 for (var k = 0; k < c.versionBranch.length; k++) {
263 branched[c.versionBranch[k]] = true
264 }
265 }
266 }, function (err) {
267 if (err) return cb(err)
268 var ids = []
269 for (var key in msgs) {
270 if (!branched[key]) ids.push(key)
271 }
272 cb(null, ids)
273 })
274 )
275}
276
277// For each dependency that is not a bundledDependency, get message ids for
278// that dependency name + version.
279function publishSingleMention(sbot, mention, cb) {
280 if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
281 // Calculate dependencyBranch and versionBranch message ids.
282 var value = {
283 type: 'npm-packages',
284 mentions: [mention]
285 }
286 var done = multicb({pluck: 1, spread: true})
287 getDependencyBranches(sbot, mention.link, done())
288 getVersionBranches(sbot, mention, done())
289 done(function (err, dependencyBranches, versionBranches) {
290 if (err) return cb(err)
291 value.dependencyBranch = dependencyBranches || undefined
292 value.versionBranch = versionBranches || undefined
293 publishMsg(sbot, value, cb)
294 })
295}
296
297function publishMentions(sbot, mentions, cb) {
298 // console.error("publishing %s mentions", mentions.length)
299 if (mentions.length === 0) return cb(new Error('Empty mentions list'))
300 // if it is just one mention, fetch and add useful metadata
301 if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
302 publishMsg(sbot, {
303 type: 'npm-packages',
304 mentions: mentions,
305 }, cb)
306}
307
308exports.publishPkgMentions = function (sbot, mentions, cb) {
309 // try to fit the mentions into as few messages as possible,
310 // while fitting under the message size limit.
311 var msgs = []
312 ;(function next(i, chunks) {
313 if (i >= mentions.length) return cb(null, msgs)
314 var chunkLen = Math.ceil(mentions.length / chunks)
315 publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
316 if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
317 if (err && msgs.length) return onPartialPublish(err)
318 if (err) return cb(err)
319 msgs.push(msg)
320 next(i + chunkLen, chunks)
321 })
322 })(0, 1)
323 function onPartialPublish(err) {
324 var remaining = mentions.length - i
325 return cb(new Error('Published messages ' +
326 msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
327 'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
328 }
329}
330
331exports.expandPkgMentions = function (sbot, mentions, props, cb) {
332 cb = once(cb)
333 var waiting = 0
334 var expandedMentions = mentions.map(function (link) {
335 var id = link && link.link
336 if (!id) return link
337 waiting++
338 var newLink = {}
339 for (var k in link) newLink[k] = link[k]
340 getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
341 if (err) return cb(err)
342 for (var k in props) newLink[k] = pkg[k]
343 if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
344 if (!--waiting) next()
345 })
346 return newLink
347 })
348 if (!waiting) next()
349 function next() {
350 cb(null, expandedMentions)
351 }
352}
353
354function SsbNpmRegistryServer(sbot, config) {
355 this.sbot = sbot
356 this.config = config
357 this.npmConfig = config.npm || {}
358 this.host = this.npmConfig.host || 'localhost'
359 this.fetchAll = this.npmConfig.fetchAll
360 this.needShasum = this.npmConfig.needShasum
361 this.wsPort = config.ws && Number(config.ws.port) || '8989'
362 this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
363 + this.wsPort + '/blobs/get/'
364 this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
365 this.getMsg = memo({cache: lru(100)}, this.getMsg)
366}
367
368SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
369SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
370
371SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
372 var self = this
373 if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
374 ;(function next(i) {
375 if (i >= ids.length) return cb()
376 self.sbot.blobs.push(ids[i], function (err) {
377 if (err) return cb(err)
378 next(i+1)
379 })
380 })(0)
381}
382
383function getBlob(sbot, id, cb) {
384 var blobs = sbot.blobs
385 blobs.size(id, function (err, size) {
386 if (typeof size === 'number') cb(null, blobs.get(id))
387 else blobs.want(id, function (err, got) {
388 if (err) cb(err)
389 else if (!got) cb('missing blob ' + id)
390 else cb(null, blobs.get(id))
391 })
392 })
393}
394
395SsbNpmRegistryServer.prototype.blobDist = function (id) {
396 var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
397 if (!m) throw new Error('bad blob id: ' + id)
398 return {
399 integrity: m[2] + '-' + m[1],
400 tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
401 }
402}
403
404function getMentions(links2, name) {
405 return links2.read({
406 query: [
407 {$filter: {rel: ['mentions', name, {$gt: true}]}},
408 {$filter: {dest: {$prefix: '&'}}},
409 {$map: {
410 name: ['rel', 1],
411 size: ['rel', 2],
412 link: 'dest',
413 author: 'source',
414 ts: 'ts'
415 }}
416 ]
417 })
418}
419
420SsbNpmRegistryServer.prototype.getMentions = function (name) {
421 if (!this.sbot.links2) return pull.empty()
422 return getMentions(this.sbot.links2, name)
423}
424
425SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
426 var self = this
427 var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
428 var ids = {}
429 var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
430 fs.readdir(prebuildsDir, function (err, filenames) {
431 if (err) return cb(new Error(err.stack || err))
432 ;(function next(i) {
433 if (i >= filenames.length) return cb(null, ids)
434 var m = nameRegex.exec(filenames[i])
435 if (!m) return next(i+1)
436 var name = m[1]
437 fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
438 if (err) return cb(new Error(err.stack || err))
439 self.sbot.blobs.add(function (err, id) {
440 if (err) return cb(new Error(err.stack || err))
441 ids[name] = id
442 next(i+1)
443 })(pull.once(data))
444 })
445 })(0)
446 })
447}
448
449SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
450 var self = this
451 if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
452
453 self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
454 if (err) return cb(new Error(err.stack || err))
455 var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
456 if (pkgs._hasNonBlobUrl) {
457 console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
458 }
459
460 if (!sbotPkgLock.name) console.trace('missing pkg lock name')
461 if (!sbotPkgLock.version) console.trace('missing pkg lock version')
462
463 var waiting = 2
464
465 self.sbot.blobs.add(function (err, id) {
466 if (err) return next(new Error(err.stack || err))
467 var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
468 var versions = pkg.versions || (pkg.versions = {})
469 pkg.versions[sbotPkgLock.version] = {
470 name: sbotPkgLock.name,
471 version: sbotPkgLock.version,
472 dist: self.blobDist(id)
473 }
474 var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
475 distTags.latest = sbotPkgLock.version
476 next()
477 })(self.sbot.bootstrap.pack())
478
479 var prebuilds
480 self.getLocalPrebuildsLinks(function (err, _prebuilds) {
481 if (err) return next(err)
482 prebuilds = _prebuilds
483 next()
484 })
485
486 function next(err) {
487 if (err) return waiting = 0, cb(err)
488 if (--waiting) return
489 fs.readFile(path.join(__dirname, 'bootstrap.js'), {
490 encoding: 'utf8'
491 }, function (err, bootstrapScript) {
492 if (err) return cb(err)
493 var script = bootstrapScript + '\n' +
494 'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
495 'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
496
497 self.sbot.blobs.add(function (err, id) {
498 if (err) return cb(new Error(err.stack || err))
499 var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
500 if (!m) return cb(new Error('bad blob id: ' + id))
501 cb(null, {
502 name: sbotPkgLock.name,
503 blob: id,
504 hashType: m[2],
505 hashBuf: Buffer.from(m[1], 'base64'),
506 })
507 })(pull.once(script))
508 })
509 }
510 })
511}
512
513SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
514 if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
515 else this.sbot.get(id, function (err, value) {
516 if (err) return cb(err)
517 cb(null, {key: id, value: value})
518 })
519}
520
521SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
522 var self = this
523 var stack = heads.slice()
524 var seen = {}
525 return function (abort, cb) {
526 if (abort) return cb(abort)
527 for (var id; stack.length && seen[id = stack.pop()];);
528 if (!id) return cb(true)
529 seen[id] = true
530 // TODO: use DFS
531 self.getMsg(id, function (err, msg) {
532 if (err) return cb(new Error(err.stack || err))
533 var c = msg && msg.value && msg.value.content
534 var links = c && c[prop]
535 if (Array.isArray(links)) {
536 stack.push.apply(stack, links)
537 } else if (links) {
538 stack.push(links)
539 }
540 cb(null, msg)
541 })
542 }
543}
544
545function Req(server, req, res) {
546 this.server = server
547 this.req = req
548 this.res = res
549 this.blobsToPush = []
550 this.fetchAll = server.fetchAll != null ? server.fetchAll : true
551 var ua = this.req.headers['user-agent']
552 var m = /\bnpm\/([0-9]*)/.exec(ua)
553 var npmVersion = m && m[1]
554 this.needShasum = server.needShasum != null ? server.needShasum :
555 (npmVersion && npmVersion < 5)
556}
557
558Req.prototype.serve = function () {
559 console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
560 this.res.setTimeout(0)
561 var pathname = this.req.url.replace(/\?.*/, '')
562 var m
563 if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)(\/.*)$/.exec(pathname))) {
564 try {
565 this.headMsgId = decodeURIComponent(m[2])
566 this.headMsgPlus = !!m[1]
567 // ^ means also include packages published after the head message id
568 } catch(e) {
569 return this.respondError(400, e.stack || e)
570 }
571 pathname = m[3]
572 }
573 if (pathname === '/') return this.serveHome()
574 if (pathname === '/bootstrap') return this.serveBootstrap()
575 if (pathname === '/-/whoami') return this.serveWhoami()
576 if (pathname === '/-/ping') return this.respond(200, true)
577 if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
578 if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
579 if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
580 return this.respond(404)
581}
582
583Req.prototype.respond = function (status, message) {
584 this.res.writeHead(status, {'content-type': 'application/json'})
585 this.res.end(message && JSON.stringify(message, 0, 2))
586}
587
588Req.prototype.respondError = function (status, message) {
589 this.respond(status, {error: message})
590}
591
592var bootstrapName = 'ssb-npm-bootstrap'
593
594Req.prototype.serveHome = function () {
595 var self = this
596 self.res.writeHead(200, {'content-type': 'text/html'})
597 var port = 8044
598 self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
599 '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
600 '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
601 '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
602 '</body></html>')
603}
604
605Req.prototype.serveBootstrap = function () {
606 var self = this
607 self.server.getBootstrapInfo(function (err, info) {
608 if (err) return self.respondError(err.stack || err)
609 var pkgNameText = info.name
610 var pkgTmpText = '/tmp/' + bootstrapName + '.js'
611 var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
612 var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
613 var blobsHostname = httpHost + ':' + self.server.wsPort
614 var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
615 var pkgHashText = info.hashBuf.toString('hex')
616 var hashCmd = info.hashType + 'sum'
617
618 var script =
619 'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
620 'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
621 'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
622 'npm install -g ' + info.name + ' &&\n' +
623 'sbot server'
624
625 self.res.writeHead(200, {'content-type': 'text/plain'})
626 self.res.end(script)
627 })
628}
629
630Req.prototype.serveWhoami = function () {
631 var self = this
632 self.server.sbot.whoami(function (err, feed) {
633 if (err) return self.respondError(err.stack || err)
634 self.respond(200, {username: feed.id})
635 })
636}
637
638Req.prototype.serveUser1 = function () {
639 this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
640}
641
642function decodeName(name) {
643 var parts = String(name).replace(/\.tgz$/, '').split(':')
644 return {
645 name: parts[1],
646 version: parts[2],
647 distTag: parts[3],
648 }
649}
650
651Req.prototype.getMsgMentions = function (name) {
652 return pull(
653 this.server.streamTree([this.headMsgId], 'dependencyBranch'),
654 // decryption could be done here
655 pull.map(function (msg) {
656 var c = msg.value && msg.value.content
657 if (!c.mentions || !Array.isArray(c.mentions)) return []
658 return c.mentions.map(function (mention) {
659 return {
660 name: mention.name,
661 size: mention.size,
662 link: mention.link,
663 author: msg.value.author,
664 ts: msg.value.timestamp,
665 }
666 })
667 }),
668 pull.flatten(),
669 pull.filter(typeof name === 'string' ? function (link) {
670 return link.name === name
671 } : name && name.$prefix ? function (link) {
672 return link.name.substr(0, name.$prefix.length) === name.$prefix
673 } : function () {
674 throw new TypeError('unsupported name filter')
675 })
676 )
677}
678
679Req.prototype.getMentions = function (name) {
680 var useMsgMentions = this.headMsgId
681 var useServerMentions = !this.headMsgId || this.headMsgPlus
682 if (useServerMentions && !this.server.sbot.links2) {
683 return this.headMsgPlus
684 ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
685 : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
686 }
687 return cat([
688 useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
689 useServerMentions ? this.server.getMentions(name) : pull.empty()
690 ])
691}
692
693Req.prototype.getMentionLinks = function (blobId) {
694 return pull(
695 this.headMsgId
696 ? this.server.streamTree([this.headMsgId], 'dependencyBranch')
697 : this.server.sbot.links({
698 dest: blobId,
699 rel: 'mentions',
700 values: true,
701 }),
702 // decryption could be done here
703 pull.map(function (msg) {
704 var c = msg.value && msg.value.content
705 return c && Array.isArray(c.mentions) || []
706 }),
707 pull.flatten(),
708 pull.filter(function (link) {
709 return link && link.link === blobId
710 })
711 )
712}
713
714Req.prototype.servePkg = function (pathname) {
715 var self = this
716 var parts = pathname.split('/')
717 var pkgName = parts.shift().replace(/%2f/i, '/')
718 if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
719 var spec = parts.shift()
720 if (spec) try { spec = decodeURIComponent(spec) } finally {}
721 if (parts.length > 0) return this.respondError(404)
722 if (self.req.method === 'PUT') return self.publishPkg(pkgName)
723 var obj = {
724 _id: pkgName,
725 name: pkgName,
726 'dist-tags': {},
727 versions: {}
728 }
729 var distTags = {/* <tag>: {version, ts}*/}
730 pull(
731 self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
732 pull.drain(function (mention) {
733 var data = decodeName(mention.name)
734 if (!data.version) return
735 if (data.distTag) {
736 var tag = distTags[data.distTag]
737 if (!tag || mention.ts > tag.ts) {
738 /* TODO: sort by causal order (versionBranch links) instead of just
739 * by timestamps */
740 distTags[data.distTag] = {ts: mention.ts, version: data.version}
741 }
742 }
743 obj.versions[data.version] = {
744 author: {
745 url: mention.author
746 },
747 name: pkgName,
748 version: data.version,
749 dist: self.server.blobDist(mention.link)
750 }
751 }, function (err) {
752 if (err) return self.respondError(500, err.stack || err)
753 for (var tag in distTags) {
754 obj['dist-tags'][tag] = distTags[tag].version
755 }
756 if (spec) resolveSpec()
757 else if (self.fetchAll) resolveAll()
758 else resolved()
759 })
760 )
761 function resolveSpec() {
762 var version = obj['dist-tags'][spec]
763 || semver.maxSatisfying(Object.keys(obj.versions), spec)
764 obj = obj.versions[version]
765 if (!obj) return self.respondError(404, 'version not found: ' + spec)
766 self.populatePackageJson(obj, function (err, pkg) {
767 if (err) return resolved(err)
768 obj = pkg || obj
769 resolved()
770 })
771 }
772 function resolveVersion(version, cb) {
773 self.populatePackageJson(obj.versions[version], function (err, pkg) {
774 if (err) return cb(err)
775 if (pkg) obj.versions[version] = pkg
776 cb()
777 })
778 }
779 function resolveAll() {
780 var done = multicb()
781 for (var version in obj.versions) {
782 resolveVersion(version, done())
783 }
784 done(resolved)
785 }
786 function resolved(err) {
787 if (err) return self.respondError(500, err.stack || err)
788 self.respond(200, obj)
789 }
790}
791
792Req.prototype.servePrebuild = function (name) {
793 var self = this
794 var getMention = self.getMentions('prebuild:' + name)
795 var blobsByAuthor = {/* <author>: BlobId */}
796 getMention(null, function next(err, link) {
797 if (err === true) return done()
798 if (err) return self.respondError(500, err.stack || err)
799 blobsByAuthor[link.author] = link.link
800 getMention(null, next)
801 })
802 function done() {
803 var authorsByLink = {/* <BlobId>: [FeedId...] */}
804 var blobId
805 for (var feed in blobsByAuthor) {
806 var blob = blobId = blobsByAuthor[feed]
807 var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
808 feeds.push(feed)
809 }
810 switch (Object.keys(authorsByLink).length) {
811 case 0:
812 return self.respondError(404, 'Not Found')
813 case 1:
814 self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
815 return self.res.end()
816 default:
817 return self.respond(300, {choices: authorsByLink})
818 }
819 }
820}
821
822var localhosts = {
823 '::1': true,
824 '127.0.0.1': true,
825 '::ffff:127.0.0.1': true,
826}
827
828Req.prototype.publishPkg = function (pkgName) {
829 var self = this
830 var remoteAddress = self.req.socket.remoteAddress
831 if (!(remoteAddress in localhosts)) {
832 return self.respondError(403, 'You may not publish as this user.')
833 }
834
835 var chunks = []
836 self.req.on('data', function (data) {
837 chunks.push(data)
838 })
839 self.req.on('end', function () {
840 var data
841 try {
842 data = JSON.parse(Buffer.concat(chunks))
843 } catch(e) {
844 return self.respondError(400, e.stack)
845 }
846 return self.publishPkg2(pkgName, data || {})
847 })
848}
849
850Req.prototype.publishPkg2 = function (name, data) {
851 var self = this
852 if (data.users) console.trace('[npm-registry] users property is not supported')
853 var attachments = data._attachments || {}
854 var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
855 var done = multicb()
856 function addAttachmentAsBlob(filename, cb) {
857 var data = attachments[filename].data
858 var tarball = new Buffer(data, 'base64')
859 var length = attachments[filename].length
860 if (length && length !== tarball.length) return self.respondError(400,
861 'Length mismatch for attachment \'' + filename + '\'')
862 self.server.sbot.blobs.add(function (err, id) {
863 if (err) return cb(err)
864 self.blobsToPush.push(id)
865 var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
866 links[filename] = {link: id, size: tarball.length, shasum: shasum}
867 cb()
868 })(pull.once(tarball))
869 }
870 for (var filename in attachments) {
871 addAttachmentAsBlob(filename, done())
872 }
873 done(function (err) {
874 if (err) return self.respondError(500, err.stack || err)
875 try {
876 self.publishPkg3(name, data, links)
877 } catch(e) {
878 self.respondError(500, e.stack || e)
879 }
880 })
881}
882
883Req.prototype.publishPkg3 = function (name, data, links) {
884 var self = this
885 var versions = data.versions || {}
886 var linksByVersion = {/* <version>: link */}
887
888 // associate tarball blobs with versions
889 for (var version in versions) {
890 var pkg = versions[version]
891 if (!pkg) return self.respondError(400, 'Bad package object')
892 if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
893 if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
894 if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
895 var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
896 if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
897 var filename = m[1]
898 var link = links[filename]
899 if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
900 // TODO?: try to find missing tarball mentioned in other messages
901 if (pkg.version && pkg.version !== version)
902 return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
903 linksByVersion[version] = link
904 link.version = version
905 link.dependencies = pkg.dependencies || {}
906 link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
907 }
908
909 // associate blobs with dist-tags
910 var tags = data['dist-tags'] || {}
911 for (var tag in tags) {
912 var version = tags[tag]
913 var link = linksByVersion[version]
914 if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
915 // TODO?: support setting dist-tag without version,
916 // by looking up a tarball blob for the version
917 link.tag = tag
918 }
919
920 // compute blob links to publish
921 var mentions = []
922 for (var filename in links) {
923 var link = links[filename] || {}
924 if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
925 mentions.push({
926 name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
927 link: link.link,
928 size: link.size,
929 shasum: link.shasum,
930 dependencies: link.dependencies,
931 bundledDependencies: link.bundledDependencies,
932 })
933 }
934 return self.publishPkgs(mentions)
935}
936
937Req.prototype.publishPkgs = function (mentions) {
938 var self = this
939 exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
940 if (err) return self.respondError(500, err.stack || err)
941 self.server.pushBlobs(self.blobsToPush, function (err) {
942 if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
943 self.respond(201)
944 console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
945 })
946 })
947}
948
949Req.prototype.populatePackageJson = function (obj, cb) {
950 var self = this
951 var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
952 var deps, bundledDeps, shasum
953
954 // look for dependencies in links.
955 // then fallback to getting it from the tarball blob
956
957 pull(
958 self.getMentionLinks(blobId),
959 pull.drain(function (link) {
960 if (link.dependencies) deps = link.dependencies
961 if (link.shasum) shasum = link.shasum
962 bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
963 // how to handle multiple assignments of dependencies to a package?
964 }, function (err) {
965 if (err) return cb(new Error(err.stack || err))
966 if (deps && (shasum || !self.needShasum)) {
967 // assume that the dependencies in the links to the blob are
968 // correct.
969 obj.dependencies = deps
970 obj.bundledDependencies = bundledDeps
971 if (shasum) obj.dist.shasum = obj._shasum = shasum
972 cb(null, obj)
973 } else {
974 // get dependencies from the tarball
975 getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
976 if (err) return cb(err)
977 pkg.dist = obj.dist
978 pkg.dist.shasum = pkg._shasum
979 pkg.author = pkg.author || obj.author
980 pkg.version = pkg.version || obj.version
981 pkg.name = pkg.name || obj.name
982 cb(null, pkg)
983 })
984 }
985 })
986 )
987}
988
989function getPackageJsonFromTarballBlob(sbot, id, cb) {
990 var self = this
991 getBlob(sbot, id, function (err, readBlob) {
992 if (err) return cb(err)
993 cb = once(cb)
994 var extract = tar.extract()
995 var pkg, shasum
996 extract.on('entry', function (header, stream, next) {
997 if (/^[^\/]*\/package\.json$/.test(header.name)) {
998 pull(toPull.source(stream), pull.collect(function (err, bufs) {
999 if (err) return cb(err)
1000 try { pkg = JSON.parse(Buffer.concat(bufs)) }
1001 catch(e) { return cb(e) }
1002 next()
1003 }))
1004 } else {
1005 stream.on('end', next)
1006 stream.resume()
1007 }
1008 })
1009 extract.on('finish', function () {
1010 pkg._shasum = shasum
1011 cb(null, pkg)
1012 })
1013 pull(
1014 readBlob,
1015 hash('sha1', 'hex', function (err, sum) {
1016 if (err) return cb(err)
1017 shasum = sum
1018 }),
1019 toPull(zlib.createGunzip()),
1020 toPull(extract)
1021 )
1022 })
1023}
1024

Built with git-ssb-web