git ssb

3+

cel / ssb-npm-registry



Tree: aaf01da7e2cbb2e5f5dc2c3afdc312df17a9aef4

Files: aaf01da7e2cbb2e5f5dc2c3afdc312df17a9aef4 / index.js

32565 bytesRaw
1var http = require('http')
2var os = require('os')
3var path = require('path')
4var fs = require('fs')
5var crypto = require('crypto')
6var pkg = require('./package')
7var semver = require('semver')
8var toPull = require('stream-to-pull-stream')
9var pull = require('pull-stream')
10var tar = require('tar-stream')
11var zlib = require('zlib')
12var cat = require('pull-cat')
13var hash = require('pull-hash')
14var multicb = require('multicb')
15var memo = require('asyncmemo')
16var lru = require('hashlru')
17
18function escapeHTML(str) {
19 return String(str)
20 .replace(/</g, '&lt;')
21 .replace(/>/g, '&gt;')
22}
23
24function idToHex(id) {
25 var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
26 return new Buffer(b64, 'base64').toString('hex')
27}
28
29function onceify(fn, self) {
30 var cbs = [], err, data
31 return function (cb) {
32 if (fn) {
33 cbs.push(cb)
34 fn.call(self, function (_err, _data) {
35 err = _err, data = _data
36 var _cbs = cbs
37 cbs = null
38 while (_cbs.length) _cbs.shift()(err, data)
39 })
40 fn = null
41 } else if (cbs) {
42 cbs.push(cb)
43 } else {
44 cb(err, data)
45 }
46 }
47}
48
49function once(cb) {
50 var done
51 return function (err, result) {
52 if (done) {
53 if (err) console.trace(err)
54 } else {
55 done = true
56 cb(err, result)
57 }
58 }
59}
60
61function npmLogin(registryAddress, cb) {
62 var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
63 var filename = path.join(os.homedir(), '.npmrc')
64 fs.readFile(filename, 'utf8', function (err, data) {
65 if (err && err.code === 'ENOENT') data = ''
66 else if (err) return cb(new Error(err.stack))
67 var lines = data ? data.split('\n') : []
68 if (lines.indexOf(tokenLine) > -1) return cb()
69 var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
70 var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
71 fs.appendFile(filename, line, cb)
72 })
73}
74
75function formatHost(host) {
76 return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
77}
78
79exports.name = 'npm-registry'
80exports.version = '1.0.0'
81exports.manifest = {
82 getAddress: 'async'
83}
84exports.init = function (sbot, config) {
85 var conf = config.npm || {}
86 var port = conf.port || 8043
87 var host = conf.host || null
88 var autoAuth = conf.autoAuth !== false
89
90 var server = http.createServer(exports.respond(sbot, config))
91 var getAddress = onceify(function (cb) {
92 server.on('error', cb)
93 server.listen(port, host, function () {
94 server.removeListener('error', cb)
95 var regHost = formatHost(host || 'localhost')
96 var regPort = this.address().port
97 var regUrl = 'http://' + regHost + ':' + regPort + '/'
98 if (autoAuth) npmLogin(regUrl, next)
99 else next()
100 function next(err) {
101 cb(err, regUrl)
102 }
103 })
104 sbot.on('close', function () {
105 server.close()
106 })
107 })
108
109 getAddress(function (err, addr) {
110 if (err) return console.error(err)
111 console.log('[npm-registry] Listening on ' + addr)
112 })
113
114 return {
115 getAddress: getAddress
116 }
117}
118
119exports.respond = function (sbot, config) {
120 var reg = new SsbNpmRegistryServer(sbot, config)
121 return function (req, res) {
122 new Req(reg, req, res).serve()
123 }
124}
125
126function publishMsg(sbot, value, cb) {
127 var gotExpectedPrevious = false
128 sbot.publish(value, function next(err, msg) {
129 if (err && /^expected previous:/.test(err.message)) {
130 // retry once on this error
131 if (gotExpectedPrevious) return cb(err)
132 gotExpectedPrevious = true
133 return sbot.publish(value, next)
134 }
135 cb(err, msg)
136 })
137}
138
139function getDependencyBranches(sbot, link, cb) {
140 var id = link.link
141
142 // get ids of heads of tree of dependencyBranch message links to include all
143 // the dependencies of the given tarball id.
144 var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
145 var msgs = {}
146 var branched = {}
147 var blobs = {}
148
149 function addPkgById(id, cb) {
150 if (blobs[id]) return cb()
151 blobs[id] = true
152 getPackageJsonCached(id, function (err, pkg) {
153 if (err) return cb(err)
154 var done = multicb()
155 for (var name in pkg.dependencies || {}) {
156 addPkgBySpec(name, pkg.dependencies[name], done())
157 }
158 done(cb)
159 })
160 }
161
162 function addPkgBySpec(name, spec, cb) {
163 var versions = {}
164 var distTags = {}
165 pull(
166 getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
167 pull.filter(mentionMatches(null, name, spec)),
168 pull.map(function (mention) {
169 // query to get the messages since links2 does not include the msg id
170 return sbot.links({
171 source: mention.author,
172 dest: mention.link,
173 rel: 'mentions',
174 values: true,
175 })
176 }),
177 pull.flatten(),
178 pull.drain(function (msg) {
179 var c = msg && msg.value && msg.value.content
180 if (!c || !Array.isArray(c.mentions)) return
181 c.mentions.forEach(function (link) {
182 var data = link && link.name && decodeName(link.name)
183 if (!data || data.name !== name) return
184 versions[data.version] = {msg: msg, mention: link, mentionData: data}
185 if (data.distTag) distTags[data.distTag] = data.version
186 })
187 }, function (err) {
188 if (err) return cb(err)
189 var version = distTags[spec]
190 || semver.maxSatisfying(Object.keys(versions), spec)
191 var item = versions[version]
192 if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
193 addDepBranchMsg(item.msg)
194 // console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
195 addPkgById(item.mention.link, cb)
196 })
197 )
198 }
199
200 function addDepBranchMsg(msg) {
201 var c = msg && msg.value && msg.value.content
202 if (!c) return
203 if (msgs[msg.key]) return
204 msgs[msg.key] = msg.value
205 if (Array.isArray(c.dependencyBranch)) {
206 for (var k = 0; k < c.dependencyBranch.length; k++) {
207 branched[c.dependencyBranch[k]] = true
208 }
209 }
210 if (Array.isArray(c.versionBranch)) {
211 for (var k = 0; k < c.versionBranch.length; k++) {
212 branched[c.versionBranch[k]] = true
213 }
214 }
215 }
216
217 function addPkgsByName(name, cb) {
218 var data = decodeName(name)
219 pull(
220 getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
221 pull.map(function (mention) {
222 return packageLinks(sbot, mention.author, mention.link, data.name)
223 }),
224 pull.flatten(),
225 pull.drain(addDepBranchMsg, cb)
226 )
227 }
228
229 var done = multicb()
230 addPkgsByName(link.name, done())
231 addPkgById(id, done())
232 done(function (err) {
233 if (err) return cb(err)
234 var ids = []
235 for (var key in msgs) {
236 if (!branched[key]) ids.push(key)
237 }
238 cb(null, ids)
239 })
240}
241
242function packageLinks(sbot, feed, id, name, spec) {
243 var matches = mentionMatches(id, name, spec)
244 return pull(
245 sbot.links({
246 source: feed,
247 dest: id,
248 rel: 'mentions',
249 values: true,
250 }),
251 pull.filter(function (msg) {
252 var c = msg && msg.value && msg.value.content
253 return c && Array.isArray(c.mentions) && c.mentions.some(matches)
254 })
255 )
256}
257
258function getVersionBranches(sbot, link, cb) {
259}
260
261// For each dependency that is not a bundledDependency, get message ids for
262// that dependency name + version.
263function publishSingleMention(sbot, mention, cb) {
264 if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
265 // Calculate dependencyBranch message ids.
266 var value = {
267 type: 'npm-packages',
268 mentions: [mention]
269 }
270 var done = multicb({pluck: 1, spread: true})
271 getDependencyBranches(sbot, mention, function (err, branches) {
272 if (err) return cb(err)
273 value.dependencyBranch = branches || undefined
274 publishMsg(sbot, value, cb)
275 })
276}
277
278function publishMentions(sbot, mentions, cb) {
279 // console.error("publishing %s mentions", mentions.length)
280 if (mentions.length === 0) return cb(new Error('Empty mentions list'))
281 // if it is just one mention, fetch and add useful metadata
282 if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
283 publishMsg(sbot, {
284 type: 'npm-packages',
285 mentions: mentions,
286 }, cb)
287}
288
289exports.publishPkgMentions = function (sbot, mentions, cb) {
290 // try to fit the mentions into as few messages as possible,
291 // while fitting under the message size limit.
292 var msgs = []
293 ;(function next(i, chunks) {
294 if (i >= mentions.length) return cb(null, msgs)
295 var chunkLen = Math.ceil(mentions.length / chunks)
296 publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
297 if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
298 if (err && msgs.length) return onPartialPublish(err)
299 if (err) return cb(err)
300 msgs.push(msg)
301 next(i + chunkLen, chunks)
302 })
303 })(0, 1)
304 function onPartialPublish(err) {
305 var remaining = mentions.length - i
306 return cb(new Error('Published messages ' +
307 msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
308 'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
309 }
310}
311
312exports.expandPkgMentions = function (sbot, mentions, props, cb) {
313 cb = once(cb)
314 var waiting = 0
315 var expandedMentions = mentions.map(function (link) {
316 var id = link && link.link
317 if (!id) return link
318 waiting++
319 var newLink = {}
320 for (var k in link) newLink[k] = link[k]
321 getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
322 if (err) return cb(err)
323 for (var k in props) newLink[k] = pkg[k]
324 if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
325 if (!--waiting) next()
326 })
327 return newLink
328 })
329 if (!waiting) next()
330 function next() {
331 cb(null, expandedMentions)
332 }
333}
334
335function SsbNpmRegistryServer(sbot, config) {
336 this.sbot = sbot
337 this.config = config
338 this.npmConfig = config.npm || {}
339 this.host = this.npmConfig.host || 'localhost'
340 this.fetchAll = this.npmConfig.fetchAll
341 this.needShasum = this.npmConfig.needShasum
342 this.wsPort = config.ws && Number(config.ws.port) || '8989'
343 this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
344 + this.wsPort + '/blobs/get/'
345 this.getMsg = memo({cache: lru(100)}, this.getMsg)
346}
347
348SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
349SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
350
351SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
352 var self = this
353 if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
354 ;(function next(i) {
355 if (i >= ids.length) return cb()
356 self.sbot.blobs.push(ids[i], function (err) {
357 if (err) return cb(err)
358 next(i+1)
359 })
360 })(0)
361}
362
363function getBlob(sbot, id, cb) {
364 var blobs = sbot.blobs
365 blobs.size(id, function (err, size) {
366 if (typeof size === 'number') cb(null, blobs.get(id))
367 else blobs.want(id, function (err, got) {
368 if (err) cb(err)
369 else if (!got) cb('missing blob ' + id)
370 else cb(null, blobs.get(id))
371 })
372 })
373}
374
375SsbNpmRegistryServer.prototype.blobDist = function (id) {
376 var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
377 if (!m) throw new Error('bad blob id: ' + id)
378 return {
379 integrity: m[2] + '-' + m[1],
380 tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
381 }
382}
383
384function getMentions(links2, name) {
385 return links2.read({
386 query: [
387 {$filter: {rel: ['mentions', name, {$gt: true}]}},
388 {$filter: {dest: {$prefix: '&'}}},
389 {$map: {
390 name: ['rel', 1],
391 size: ['rel', 2],
392 link: 'dest',
393 author: 'source',
394 ts: 'ts'
395 }}
396 ]
397 })
398}
399
400function mentionMatches(id, name, spec) {
401 return function (mention) {
402 var data = mention
403 && (!id || id === mention.link)
404 && mention.name
405 && decodeName(mention.name)
406 return data
407 && data.name === name
408 && (spec ? semver.satisfies(data.version, spec) : true)
409 }
410}
411
412SsbNpmRegistryServer.prototype.getMentions = function (name) {
413 if (!this.sbot.links2) return pull.empty()
414 return getMentions(this.sbot.links2, name)
415}
416
417SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
418 if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
419 else this.sbot.get(id, function (err, value) {
420 if (err) return cb(err)
421 cb(null, {key: id, value: value})
422 })
423}
424
425SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
426 var self = this
427 var stack = heads.slice()
428 var seen = {}
429 return function (abort, cb) {
430 if (abort) return cb(abort)
431 for (var id; stack.length && seen[id = stack.pop()];);
432 if (!id) return cb(true)
433 seen[id] = true
434 // TODO: use DFS
435 self.getMsg(id, function (err, msg) {
436 if (err) return cb(new Error(err.stack || err))
437 var c = msg && msg.value && msg.value.content
438 var links = c && c[prop]
439 if (Array.isArray(links)) {
440 stack.push.apply(stack, links)
441 } else if (links) {
442 stack.push(links)
443 }
444 cb(null, msg)
445 })
446 }
447}
448
449function Req(server, req, res) {
450 this.server = server
451 this.req = req
452 this.res = res
453 this.blobsToPush = []
454 this.fetchAll = server.fetchAll != null ? server.fetchAll : true
455 var ua = this.req.headers['user-agent']
456 var m = /\bnpm\/([0-9]*)/.exec(ua)
457 var npmVersion = m && m[1]
458 this.needShasum = server.needShasum != null ? server.needShasum :
459 (npmVersion && npmVersion < 5)
460}
461
462Req.prototype.serve = function () {
463 console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
464 this.res.setTimeout(0)
465 var pathname = this.req.url.replace(/\?.*/, '')
466 var m
467 if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
468 try {
469 this.headMsgIds = decodeURIComponent(m[2]).split(',')
470 this.headMsgPlus = !!m[1]
471 // ^ means also include packages published after the head message id
472 } catch(e) {
473 return this.respondError(400, e.stack || e)
474 }
475 pathname = m[3]
476 }
477 if (pathname === '/') return this.serveHome()
478 if (pathname === '/bootstrap') return this.serveBootstrap()
479 if (pathname === '/robots.txt') return this.serveRobots()
480 if (pathname === '/-/whoami') return this.serveWhoami()
481 if (pathname === '/-/ping') return this.respond(200, true)
482 if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
483 if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
484 if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
485 return this.respond(404)
486}
487
488Req.prototype.respond = function (status, message) {
489 this.res.writeHead(status, {'content-type': 'application/json'})
490 this.res.end(message && JSON.stringify(message, 0, 2))
491}
492
493Req.prototype.respondError = function (status, message) {
494 this.respond(status, {error: message})
495}
496
497Req.prototype.respondErrorStr = function (status, err) {
498 this.res.writeHead(status, {'content-type': 'text/plain'})
499 this.res.end(err.stack || err)
500}
501
502Req.prototype.serveHome = function () {
503 var self = this
504 self.res.writeHead(200, {'content-type': 'text/html'})
505 var port = 8044
506 self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
507 '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
508 '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
509 '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
510 '</body></html>')
511}
512
513Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
514 var self = this
515 pull(
516 self.server.sbot.links({
517 source: feedId,
518 dest: blobId,
519 rel: 'mentions',
520 values: true,
521 }),
522 pull.filter(function (msg) {
523 var c = msg && msg.value && msg.value.content
524 return c.type === 'npm-packages'
525 }),
526 pull.collect(function (err, msgs) {
527 if (err) return cb(err)
528 if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
529 if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
530 // TODO: make a smarter decision about which message id to use
531 cb(null, msgs.pop().key)
532 })
533 )
534}
535
536Req.prototype.resolvePkg = function (pkgSpec, cb) {
537 var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
538 if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
539 var self = this
540 var pkgName = m[1]
541 var spec = m[2] || '*'
542 var versions = {}
543 var distTags = {}
544 pull(
545 self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
546 pull.drain(function (mention) {
547 var data = decodeName(mention.name)
548 if (!data.version) return
549 if (data.distTag) {
550 distTags[data.distTag] = data.version
551 }
552 versions[data.version] = {
553 author: mention.author,
554 name: pkgName,
555 version: data.version,
556 blobId: mention.link
557 }
558 }, function (err) {
559 if (err) return cb(err)
560 var version = distTags[spec]
561 || semver.maxSatisfying(Object.keys(versions), spec)
562 var item = versions[version]
563 if (!item) return cb(new Error('Version not found: ' + name + '@' + spec))
564 self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
565 if (err) return cb(err)
566 item.msgId = id
567 cb(null, item)
568 })
569 })
570 )
571}
572
573Req.prototype.resolvePkgs = function (specs, cb) {
574 var done = multicb({pluck: 1})
575 var self = this
576 specs.forEach(function (spec) {
577 self.resolvePkg(spec, done())
578 })
579 done(cb)
580}
581
582Req.prototype.serveBootstrap = function () {
583 var self = this
584 var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
585 var ssbNpmRegistryName = require('./package.json').name
586 var ssbNpmRegistryVersion = require('./package.json').version
587 var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
588 var done = multicb({pluck: 1, spread: true})
589 self.resolvePkg(ssbNpmRegistrySpec, done())
590 self.resolvePkgs(pkgs, done())
591 done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
592 if (err) return self.respondErrorStr(500, err.stack || err)
593 if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
594 var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
595 var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
596 var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
597 var globalPkgs = pkgsInfo.map(function (info) {
598 return info.name + '@' + info.version
599 }).join(' ')
600 var npmCmd = 'npm install -g ' + globalPkgs
601
602 var pkgTmpText = '/tmp/' + ssbNpmRegistryName + '.tar.gz'
603 var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
604 var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
605 var wsLink = 'http://' + httpHost + ':' + self.server.wsPort + '/'
606 var tarballLink = wsLink + 'blobs/get/' + ssbNpmRegistryBlobId
607
608 var script =
609 'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
610 'echo ' + ssbNpmRegistryBlobHex + ' ' + pkgTmpText + ' | sha256sum -c &&\n' +
611 'mkdir -p ~/.ssb/node_modules && cd ~/.ssb/node_modules &&\n' +
612 'tar xzf ' + pkgTmpText + ' &&\n' +
613 'mv package ' + ssbNpmRegistryName + ' &&\n' +
614 ssbNpmRegistryName + '/bootstrap/bin.js --ws-url ' + wsLink + ' \\\n' +
615 pkgMsgs.map(function (id) {
616 return ' --branch ' + id + ' \\\n'
617 }).join('') +
618 ' -- ' + npmCmd + ' &&\n' +
619 'sbot server'
620
621 self.res.writeHead(200, {'Content-type': 'text/plain'})
622 self.res.end(script)
623 })
624}
625
626Req.prototype.serveRobots = function () {
627 this.res.writeHead(200, {'Content-type': 'text/plain'})
628 this.res.end('User-agent: *\nDisallow: /\n')
629}
630
631Req.prototype.serveWhoami = function () {
632 var self = this
633 self.server.sbot.whoami(function (err, feed) {
634 if (err) return self.respondError(500, err.stack || err)
635 self.respond(200, {username: feed.id})
636 })
637}
638
639Req.prototype.serveUser1 = function () {
640 this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
641}
642
643function decodeName(name) {
644 var parts = String(name).replace(/\.tgz$/, '').split(':')
645 return {
646 name: parts[1],
647 version: parts[2],
648 distTag: parts[3],
649 }
650}
651
652Req.prototype.getMsgMentions = function (name) {
653 return pull(
654 this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
655 // decryption could be done here
656 pull.map(function (msg) {
657 var c = msg.value && msg.value.content
658 if (!c.mentions || !Array.isArray(c.mentions)) return []
659 return c.mentions.map(function (mention) {
660 return {
661 name: mention.name,
662 size: mention.size,
663 link: mention.link,
664 author: msg.value.author,
665 ts: msg.value.timestamp,
666 }
667 })
668 }),
669 pull.flatten(),
670 pull.filter(typeof name === 'string' ? function (link) {
671 return link.name === name
672 } : name && name.$prefix ? function (link) {
673 return link.name.substr(0, name.$prefix.length) === name.$prefix
674 } : function () {
675 throw new TypeError('unsupported name filter')
676 })
677 )
678}
679
680Req.prototype.getMentions = function (name) {
681 var useMsgMentions = this.headMsgIds
682 var useServerMentions = !this.headMsgIds || this.headMsgPlus
683 if (useServerMentions && !this.server.sbot.links2) {
684 return this.headMsgPlus
685 ? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
686 : pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
687 }
688 return cat([
689 useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
690 useServerMentions ? this.server.getMentions(name) : pull.empty()
691 ])
692}
693
694Req.prototype.getMentionLinks = function (blobId) {
695 return pull(
696 this.headMsgIds
697 ? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
698 : this.server.sbot.links({
699 dest: blobId,
700 rel: 'mentions',
701 values: true,
702 }),
703 // decryption could be done here
704 pull.map(function (msg) {
705 var c = msg.value && msg.value.content
706 return c && Array.isArray(c.mentions) && c.mentions || []
707 }),
708 pull.flatten(),
709 pull.filter(function (link) {
710 return link && link.link === blobId
711 })
712 )
713}
714
715Req.prototype.servePkg = function (pathname) {
716 var self = this
717 var parts = pathname.split('/')
718 var pkgName = parts.shift().replace(/%2f/i, '/')
719 if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
720 var spec = parts.shift()
721 if (spec) try { spec = decodeURIComponent(spec) } finally {}
722 if (parts.length > 0) return this.respondError(404)
723 if (self.req.method === 'PUT') return self.publishPkg(pkgName)
724 var obj = {
725 _id: pkgName,
726 name: pkgName,
727 'dist-tags': {},
728 versions: {}
729 }
730 var distTags = {/* <tag>: {version, ts}*/}
731 pull(
732 self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
733 pull.drain(function (mention) {
734 var data = decodeName(mention.name)
735 if (!data.version) return
736 if (data.distTag) {
737 var tag = distTags[data.distTag]
738 if (!tag || mention.ts > tag.ts) {
739 /* TODO: sort by causal order instead of only by timestamps */
740 distTags[data.distTag] = {ts: mention.ts, version: data.version}
741 }
742 }
743 obj.versions[data.version] = {
744 author: {
745 url: mention.author
746 },
747 name: pkgName,
748 version: data.version,
749 dist: self.server.blobDist(mention.link)
750 }
751 }, function (err) {
752 if (err) return self.respondError(500, err.stack || err)
753 for (var tag in distTags) {
754 obj['dist-tags'][tag] = distTags[tag].version
755 }
756 if (spec) resolveSpec()
757 else if (self.fetchAll) resolveAll()
758 else resolved()
759 })
760 )
761 function resolveSpec() {
762 var version = obj['dist-tags'][spec]
763 || semver.maxSatisfying(Object.keys(obj.versions), spec)
764 obj = obj.versions[version]
765 if (!obj) return self.respondError(404, 'version not found: ' + spec)
766 self.populatePackageJson(obj, function (err, pkg) {
767 if (err) return resolved(err)
768 obj = pkg || obj
769 resolved()
770 })
771 }
772 function resolveVersion(version, cb) {
773 self.populatePackageJson(obj.versions[version], function (err, pkg) {
774 if (err) return cb(err)
775 if (pkg) obj.versions[version] = pkg
776 cb()
777 })
778 }
779 function resolveAll() {
780 var done = multicb()
781 for (var version in obj.versions) {
782 resolveVersion(version, done())
783 }
784 done(resolved)
785 }
786 function resolved(err) {
787 if (err) return self.respondError(500, err.stack || err)
788 self.respond(200, obj)
789 }
790}
791
792Req.prototype.servePrebuild = function (name) {
793 var self = this
794 var getMention = self.getMentions('prebuild:' + name)
795 var blobsByAuthor = {/* <author>: BlobId */}
796 getMention(null, function next(err, link) {
797 if (err === true) return done()
798 if (err) return self.respondError(500, err.stack || err)
799 blobsByAuthor[link.author] = link.link
800 getMention(null, next)
801 })
802 function done() {
803 var authorsByLink = {/* <BlobId>: [FeedId...] */}
804 var blobId
805 for (var feed in blobsByAuthor) {
806 var blob = blobId = blobsByAuthor[feed]
807 var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
808 feeds.push(feed)
809 }
810 switch (Object.keys(authorsByLink).length) {
811 case 0:
812 return self.respondError(404, 'Not Found')
813 case 1:
814 self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId})
815 return self.res.end()
816 default:
817 return self.respond(300, {choices: authorsByLink})
818 }
819 }
820}
821
822var localhosts = {
823 '::1': true,
824 '127.0.0.1': true,
825 '::ffff:127.0.0.1': true,
826}
827
828Req.prototype.publishPkg = function (pkgName) {
829 var self = this
830 var remoteAddress = self.req.socket.remoteAddress
831 if (!(remoteAddress in localhosts)) {
832 return self.respondError(403, 'You may not publish as this user.')
833 }
834
835 var chunks = []
836 self.req.on('data', function (data) {
837 chunks.push(data)
838 })
839 self.req.on('end', function () {
840 var data
841 try {
842 data = JSON.parse(Buffer.concat(chunks))
843 } catch(e) {
844 return self.respondError(400, e.stack)
845 }
846 return self.publishPkg2(pkgName, data || {})
847 })
848}
849
850Req.prototype.publishPkg2 = function (name, data) {
851 var self = this
852 if (data.users) console.trace('[npm-registry] users property is not supported')
853 var attachments = data._attachments || {}
854 var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
855 var done = multicb()
856 function addAttachmentAsBlob(filename, cb) {
857 var data = attachments[filename].data
858 var tarball = new Buffer(data, 'base64')
859 var length = attachments[filename].length
860 if (length && length !== tarball.length) return self.respondError(400,
861 'Length mismatch for attachment \'' + filename + '\'')
862 self.server.sbot.blobs.add(function (err, id) {
863 if (err) return cb(err)
864 self.blobsToPush.push(id)
865 var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
866 links[filename] = {link: id, size: tarball.length, shasum: shasum}
867 cb()
868 })(pull.once(tarball))
869 }
870 for (var filename in attachments) {
871 addAttachmentAsBlob(filename, done())
872 }
873 done(function (err) {
874 if (err) return self.respondError(500, err.stack || err)
875 try {
876 self.publishPkg3(name, data, links)
877 } catch(e) {
878 self.respondError(500, e.stack || e)
879 }
880 })
881}
882
883Req.prototype.publishPkg3 = function (name, data, links) {
884 var self = this
885 var versions = data.versions || {}
886 var linksByVersion = {/* <version>: link */}
887
888 // associate tarball blobs with versions
889 for (var version in versions) {
890 var pkg = versions[version]
891 if (!pkg) return self.respondError(400, 'Bad package object')
892 if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
893 if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
894 if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
895 var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
896 if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
897 var filename = m[1]
898 var link = links[filename]
899 if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
900 // TODO?: try to find missing tarball mentioned in other messages
901 if (pkg.version && pkg.version !== version)
902 return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
903 linksByVersion[version] = link
904 link.version = version
905 link.dependencies = pkg.dependencies || {}
906 link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
907 }
908
909 // associate blobs with dist-tags
910 var tags = data['dist-tags'] || {}
911 for (var tag in tags) {
912 var version = tags[tag]
913 var link = linksByVersion[version]
914 if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
915 // TODO?: support setting dist-tag without version,
916 // by looking up a tarball blob for the version
917 link.tag = tag
918 }
919
920 // compute blob links to publish
921 var mentions = []
922 for (var filename in links) {
923 var link = links[filename] || {}
924 if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
925 mentions.push({
926 name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
927 link: link.link,
928 size: link.size,
929 shasum: link.shasum,
930 dependencies: link.dependencies,
931 bundledDependencies: link.bundledDependencies,
932 })
933 }
934 return self.publishPkgs(mentions)
935}
936
937Req.prototype.publishPkgs = function (mentions) {
938 var self = this
939 exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
940 if (err) return self.respondError(500, err.stack || err)
941 self.server.pushBlobs(self.blobsToPush, function (err) {
942 if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
943 self.respond(201)
944 console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
945 })
946 })
947}
948
949Req.prototype.populatePackageJson = function (obj, cb) {
950 var self = this
951 var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
952 var deps, bundledDeps, shasum
953
954 // look for dependencies in links.
955 // then fallback to getting it from the tarball blob
956
957 pull(
958 self.getMentionLinks(blobId),
959 pull.drain(function (link) {
960 if (link.dependencies) deps = link.dependencies
961 if (link.shasum) shasum = link.shasum
962 bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
963 // how to handle multiple assignments of dependencies to a package?
964 }, function (err) {
965 if (err) return cb(new Error(err.stack || err))
966 if (deps && (shasum || !self.needShasum)) {
967 // assume that the dependencies in the links to the blob are
968 // correct.
969 obj.dependencies = deps
970 obj.bundledDependencies = bundledDeps
971 if (shasum) obj.dist.shasum = obj._shasum = shasum
972 cb(null, obj)
973 } else {
974 // get dependencies from the tarball
975 getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
976 if (err) return cb(err)
977 pkg.dist = obj.dist
978 pkg.dist.shasum = pkg._shasum
979 pkg.author = pkg.author || obj.author
980 pkg.version = pkg.version || obj.version
981 pkg.name = pkg.name || obj.name
982 cb(null, pkg)
983 })
984 }
985 })
986 )
987}
988
989function getPackageJsonFromTarballBlob(sbot, id, cb) {
990 var self = this
991 getBlob(sbot, id, function (err, readBlob) {
992 if (err) return cb(err)
993 cb = once(cb)
994 var extract = tar.extract()
995 var pkg, shasum
996 extract.on('entry', function (header, stream, next) {
997 if (/^[^\/]*\/package\.json$/.test(header.name)) {
998 pull(toPull.source(stream), pull.collect(function (err, bufs) {
999 if (err) return cb(err)
1000 try { pkg = JSON.parse(Buffer.concat(bufs)) }
1001 catch(e) { return cb(e) }
1002 next()
1003 }))
1004 } else {
1005 stream.on('end', next)
1006 stream.resume()
1007 }
1008 })
1009 extract.on('finish', function () {
1010 pkg._shasum = shasum
1011 cb(null, pkg)
1012 })
1013 pull(
1014 readBlob,
1015 hash('sha1', 'hex', function (err, sum) {
1016 if (err) return cb(err)
1017 shasum = sum
1018 }),
1019 toPull(zlib.createGunzip()),
1020 toPull(extract)
1021 )
1022 })
1023}
1024

Built with git-ssb-web