git ssb

3+

cel / ssb-npm-registry



Tree: 45e34d5ff0c939447f0f24c04303dfc1663e3421

Files: 45e34d5ff0c939447f0f24c04303dfc1663e3421 / index.js

17087 bytesRaw
1var http = require('http')
2var os = require('os')
3var path = require('path')
4var fs = require('fs')
5var crypto = require('crypto')
6var pkg = require('./package')
7
8function pullOnce(data) {
9 var ended
10 return function (abort, cb) {
11 if (ended || (ended = abort)) return cb(ended)
12 ended = true
13 cb(null, data)
14 }
15}
16
17function escapeHTML(str) {
18 return String(str)
19 .replace(/</g, '&lt;')
20 .replace(/>/g, '&gt;')
21}
22
23function onceify(fn, self) {
24 var cbs = [], err, data
25 return function (cb) {
26 if (fn) {
27 cbs.push(cb)
28 fn.call(self, function (_err, _data) {
29 err = _err, data = _data
30 var _cbs = cbs
31 cbs = null
32 while (_cbs.length) _cbs.shift()(err, data)
33 })
34 fn = null
35 } else if (cbs) {
36 cbs.push(cb)
37 } else {
38 cb(err, data)
39 }
40 }
41}
42
43function pkgLockToRegistryPkgs(pkgLock, wsPort) {
44 // convert a package-lock.json file into data for serving as an npm registry
45 var hasNonBlobUrl = false
46 var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
47 var pkgs = {}
48 var queue = [pkgLock, pkgLock.name]
49 while (queue.length) {
50 var dep = queue.shift(), name = queue.shift()
51 if (name) {
52 var pkg = pkgs[name] || (pkgs[name] = {
53 _id: name,
54 name: name,
55 versions: {}
56 })
57 if (dep.version && dep.integrity && dep.resolved) {
58 if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
59 pkg.versions[dep.version] = {
60 dist: {
61 integrity: dep.integrity,
62 tarball: dep.resolved
63 }
64 }
65 }
66 }
67 if (dep.dependencies) for (var depName in dep.dependencies) {
68 queue.push(dep.dependencies[depName], depName)
69 }
70 }
71 pkgs._hasNonBlobUrl = hasNonBlobUrl
72 return pkgs
73}
74
75function npmLogin(registryAddress, cb) {
76 var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
77 var filename = path.join(os.homedir(), '.npmrc')
78 fs.readFile(filename, 'utf8', function (err, data) {
79 if (err && err.code === 'ENOENT') data = ''
80 else if (err) return cb(new Error(err.stack))
81 var lines = data ? data.split('\n') : []
82 if (lines.indexOf(tokenLine) > -1) return cb()
83 var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
84 var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
85 fs.appendFile(filename, line, cb)
86 })
87}
88
89function formatHost(host) {
90 return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
91}
92
93exports.name = 'npm-registry'
94exports.version = '1.0.0'
95exports.manifest = {
96 getAddress: 'async'
97}
98exports.init = function (sbot, config) {
99 var port = config.npm ? config.npm.port : 8043
100 var host = config.npm && config.npm.host || null
101 var autoAuth = config.npm && config.npm.autoAuth !== false
102 var getAddressCbs = []
103
104 var server = http.createServer(exports.respond(sbot, config))
105 var getAddress = onceify(function (cb) {
106 server.on('error', cb)
107 server.listen(port, host, function () {
108 server.removeListener('error', cb)
109 var regHost = formatHost(host || 'localhost')
110 var regUrl = 'http://' + regHost + ':' + this.address().port + '/'
111 if (autoAuth) npmLogin(regUrl, next)
112 else next()
113 function next(err) {
114 cb(err, regUrl)
115 }
116 })
117 sbot.on('close', function () {
118 server.close()
119 })
120 })
121
122 getAddress(function (err, addr) {
123 if (err) return console.error(err)
124 console.log('[npm-registry] Listening on ' + addr)
125 })
126
127 return {
128 getAddress: getAddress
129 }
130}
131
132exports.respond = function (sbot, config) {
133 var reg = new SsbNpmRegistryServer(sbot, config)
134 return function (req, res) {
135 new Req(reg, req, res).serve()
136 }
137}
138
139function publishMsg(sbot, value, cb) {
140 var gotExpectedPrevious = false
141 sbot.publish(value, function next(err, msg) {
142 if (err && /^expected previous:/.test(err.message)) {
143 // retry once on this error
144 if (gotExpectedPrevious) return cb(err)
145 gotExpectedPrevious = true
146 return sbot.publish(value, next)
147 }
148 cb(err, msg)
149 })
150}
151
152function publishMentions(sbot, mentions, cb) {
153 // console.error("publishing %s mentions", mentions.length)
154 if (mentions.length === 0) return cb(new Error('Empty mentions list'))
155 publishMsg(sbot, {
156 type: 'npm-packages',
157 mentions: mentions,
158 }, cb)
159}
160
161exports.publishPkgMentions = function (sbot, mentions, cb) {
162 // try to fit the mentions into as few messages as possible,
163 // while fitting under the message size limit.
164 var msgs = []
165 ;(function next(i, chunks) {
166 if (i >= mentions.length) return cb(null, msgs)
167 var chunkLen = Math.ceil(mentions.length / chunks)
168 publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
169 if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
170 if (err && msgs.length) return onPartialPublish(err)
171 if (err) return cb(err)
172 msgs.push(msg)
173 next(i + chunkLen, chunks)
174 })
175 })(0, 1)
176 function onPartialPublish(err) {
177 var remaining = mentions.length - i
178 return cb(new Error('Published messages ' +
179 msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
180 'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
181 }
182}
183
184function SsbNpmRegistryServer(sbot, config) {
185 this.sbot = sbot
186 this.config = config
187 this.links2 = sbot.links2
188 if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin')
189 this.wsPort = config.ws && Number(config.ws.port) || '8989'
190 this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
191 + this.wsPort + '/blobs/get/'
192 this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
193}
194
195SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
196SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
197
198SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
199 var self = this
200 if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
201 ;(function next(i) {
202 if (i >= ids.length) return cb()
203 self.sbot.blobs.push(ids[i], function (err) {
204 if (err) return cb(err)
205 next(i+1)
206 })
207 })(0)
208}
209
210SsbNpmRegistryServer.prototype.blobDist = function (id) {
211 var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
212 if (!m) throw new Error('bad blob id: ' + id)
213 return {
214 integrity: m[2] + '-' + m[1],
215 tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id
216 }
217}
218
219SsbNpmRegistryServer.prototype.getMentions = function (name) {
220 return this.links2.read({
221 query: [
222 {$filter: {rel: ['mentions', name]}},
223 {$filter: {dest: {$prefix: '&'}}},
224 {$map: {
225 name: ['rel', 1],
226 size: ['rel', 2],
227 link: 'dest',
228 author: 'source',
229 ts: 'ts'
230 }}
231 ]
232 })
233}
234
235SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
236 var self = this
237 if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
238
239 self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
240 if (err) return cb(new Error(err.stack || err))
241 var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
242 if (pkgs._hasNonBlobUrl) {
243 console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
244 }
245
246 if (!sbotPkgLock.name) console.trace('missing pkg lock name')
247 if (!sbotPkgLock.version) console.trace('missing pkg lock version')
248
249 self.sbot.blobs.add(function (err, id) {
250 if (err) return cb(new Error(err.stack || err))
251 var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
252 var versions = pkg.versions || (pkg.versions = {})
253 pkg.versions[sbotPkgLock.version] = {
254 dist: self.blobDist(id)
255 }
256 var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
257 distTags.latest = sbotPkgLock.version
258 next()
259 })(self.sbot.bootstrap.pack())
260
261 function next() {
262 fs.readFile(path.join(__dirname, 'bootstrap.js'), {
263 encoding: 'utf8'
264 }, function (err, bootstrapScript) {
265 if (err) return cb(err)
266 var script = bootstrapScript + '\n' +
267 'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2)
268
269 self.sbot.blobs.add(function (err, id) {
270 if (err) return cb(new Error(err.stack || err))
271 var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
272 if (!m) return cb(new Error('bad blob id: ' + id))
273 cb(null, {
274 name: sbotPkgLock.name,
275 blob: id,
276 hashType: m[2],
277 hashBuf: Buffer.from(m[1], 'base64'),
278 })
279 })(pullOnce(script))
280 })
281 }
282 })
283}
284
285SsbNpmRegistryServer.prototype.getBootstrapScriptHash = function (cb) {
286 var hasher = crypto.createHash('sha256')
287 hasher.update(data)
288 var hash = hasher.digest()
289 getBootstrapScriptHash = function (cb) {
290 return cb(null, hash)
291 }
292 getBootstrapScriptHash(cb)
293}
294
295function Req(server, req, res) {
296 this.server = server
297 this.req = req
298 this.res = res
299 this.blobsToPush = []
300}
301
302Req.prototype.serve = function () {
303 console.log(this.req.method, this.req.url, this.req.socket.remoteAddress)
304 var pathname = this.req.url.replace(/\?.*/, '')
305 if (pathname === '/') return this.serveHome()
306 if (pathname === '/bootstrap') return this.serveBootstrap()
307 if (pathname === '/-/whoami') return this.serveWhoami()
308 if (pathname === '/-/ping') return this.respond(200, true)
309 if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
310 if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
311 return this.respond(404)
312}
313
314Req.prototype.respond = function (status, message) {
315 this.res.writeHead(status, {'content-type': 'application/json'})
316 this.res.end(message && JSON.stringify(message))
317}
318
319Req.prototype.respondError = function (status, message) {
320 this.respond(status, {error: message})
321}
322
323var bootstrapName = 'ssb-npm-bootstrap'
324
325Req.prototype.serveHome = function () {
326 var self = this
327 self.res.writeHead(200, {'content-type': 'text/html'})
328 var port = 8044
329 self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
330 '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
331 '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
332 '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
333 '</body></html>')
334}
335
336Req.prototype.serveBootstrap = function () {
337 var self = this
338 self.server.getBootstrapInfo(function (err, info) {
339 if (err) return this.respondError(err.stack || err)
340 var pkgNameText = info.name
341 var pkgTmpText = '/tmp/' + bootstrapName + '.js'
342 var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
343 var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
344 var blobsHostname = httpHost + ':' + self.server.wsPort
345 var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
346 var pkgHashText = info.hashBuf.toString('hex')
347 var hashCmd = info.hashType + 'sum'
348
349 var script =
350 'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
351 'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
352 'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
353 'npm install -g ' + info.name + ' &&\n' +
354 'sbot server'
355
356 self.res.writeHead(200, {'content-type': 'text/plain'})
357 self.res.end(script)
358 })
359}
360
361Req.prototype.serveWhoami = function () {
362 var self = this
363 self.server.sbot.whoami(function (err, feed) {
364 if (err) return self.respondError(err.stack || err)
365 self.respond(200, {username: feed.id})
366 })
367}
368
369Req.prototype.serveUser1 = function () {
370 this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
371}
372
373function decodeName(name) {
374 var parts = name.replace(/\.tgz$/, '').split(':')
375 return {
376 name: parts[1],
377 version: parts[2],
378 distTag: parts[3],
379 }
380}
381
382Req.prototype.servePkg = function (pathname) {
383 var self = this
384 var parts = pathname.split('/')
385 var pkgName = parts.shift()
386 if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
387 if (parts.length > 0) return this.respondError(404)
388 if (self.req.method === 'PUT') return self.publishPkg(pkgName)
389 var obj = {
390 _id: pkgName,
391 name: pkgName,
392 'dist-tags': {},
393 versions: {}
394 }
395 var oldest, newest
396 var getMention = self.server.getMentions({$prefix: 'npm:' + pkgName + ':'})
397 getMention(null, function next(err, mention) {
398 if (err === true) return self.respond(200, obj)
399 if (err) return self.respondError(500, err.stack || err)
400 var data = decodeName(mention.name)
401 if (!data.version) return
402 if (data.distTag) obj['dist-tags'][data.distTag] = data.version
403 obj.versions[data.version] = {
404 author: {
405 url: mention.author
406 },
407 dist: self.server.blobDist(mention.link)
408 }
409 getMention(null, next)
410 })
411}
412
413var localhosts = {
414 '::1': true,
415 '127.0.0.1': true,
416 '::ffff:127.0.0.1': true,
417}
418
419Req.prototype.publishPkg = function (pkgName) {
420 var self = this
421 var remoteAddress = self.req.socket.remoteAddress
422 if (!(remoteAddress in localhosts)) {
423 return self.respondError(403, 'You may not publish as this user.')
424 }
425
426 var chunks = []
427 self.req.on('data', function (data) {
428 chunks.push(data)
429 })
430 self.req.on('end', function () {
431 var data
432 try {
433 data = JSON.parse(Buffer.concat(chunks))
434 } catch(e) {
435 return self.respondError(400, e.stack)
436 }
437 return self.publishPkg2(pkgName, data || {})
438 })
439}
440
441Req.prototype.publishPkg2 = function (name, data) {
442 var self = this
443 if (data.users) console.trace('[npm-registry] users property is not supported')
444 var attachments = data._attachments || {}
445 var links = {/* <name>-<version>.tgz: {link: <BlobId>, size: number} */}
446 var waiting = 0
447 Object.keys(attachments).forEach(function (filename) {
448 waiting++
449 var tarball = new Buffer(attachments[filename].data, 'base64')
450 var length = attachments[filename].length
451 if (length && length !== tarball.length) return self.respondError(400,
452 'Length mismatch for attachment \'' + filename + '\'')
453 self.server.sbot.blobs.add(function (err, id) {
454 if (err) return self.respondError(500,
455 'Adding attachment \'' + filename + '\' as blob failed')
456 self.blobsToPush.push(id)
457 links[filename] = {link: id, size: tarball.length}
458 if (!--waiting) next()
459 })(pullOnce(tarball))
460 })
461 function next() {
462 try {
463 self.publishPkg3(name, data, links)
464 } catch(e) {
465 self.respondError(500, e.stack || e)
466 }
467 }
468}
469
470Req.prototype.publishPkg3 = function (name, data, links) {
471 var self = this
472 var versions = data.versions || {}
473 var linksByVersion = {/* <version>: link */}
474
475 // associate tarball blobs with versions
476 for (var version in versions) {
477 var pkg = versions[version]
478 if (!pkg) return self.respondError(400, 'Bad package object')
479 if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
480 if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
481 if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
482 var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
483 if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
484 var filename = m[1]
485 var link = links[filename]
486 if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
487 // TODO?: try to find missing tarball mentioned in other messages
488 if (pkg.version && pkg.version !== version)
489 return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
490 linksByVersion[version] = link
491 link.version = version
492 }
493
494 // associate blobs with dist-tags
495 var tags = data['dist-tags'] || {}
496 for (var tag in tags) {
497 var version = tags[tag]
498 var link = linksByVersion[version]
499 if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
500 // TODO?: support setting dist-tag without version,
501 // by looking up a tarball blob for the version
502 link.tag = tag
503 }
504
505 // compute blob links to publish
506 var mentions = []
507 for (var filename in links) {
508 var link = links[filename] || {}
509 if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
510 mentions.push({
511 name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
512 link: link.link,
513 size: link.size,
514 })
515 }
516 return self.publishPkgs(mentions)
517}
518
519Req.prototype.publishPkgs = function (mentions) {
520 var self = this
521 exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
522 if (err) self.respondError(500, err.stack || err)
523 self.server.pushBlobs(self.blobsToPush, function (err) {
524 if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
525 self.respond(201)
526 console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
527 })
528 })
529}
530

Built with git-ssb-web