git ssb

3+

cel / ssb-npm-registry



Commit 688be4c9d6200dd66d47d9087952c52b4a9de824

Remove scuttlebot bootstrap feature

cel committed on 6/30/2018, 8:57:15 PM
Parent: 59ec2a7aa684fe61657b02c1c3ed14b657b73896

Files changed

README.mdchanged
index.jschanged
bootstrap.jsdeleted
README.mdView
@@ -124,33 +124,8 @@
124124 If `sbot` does not have the `ssb-links` scuttlebot plugin, requests to the registry must use message scope and not the caret, since `ssb-links` is required otherwise.
125125
126126 [`ssb-ooo`]: %AlNuRW5JCvClmCcdnHVt5vMHegdn43dFw0kCBI1c+CI=.sha256
127127
128-## Bootstrapping Scuttlebot
129-
130-This plugin includes a script for securely bootstrapping an npm-installation of scuttlebot from your machine to a peer's machine.
131-
132-Steps for bootstrapping:
133-
134-- Find your local IP.
135-- In your web browser, go to your IP, port 8043 (or other port, if you set
136- `config.npm.port`).
137-- Click the "bootstrap" link.
138-- Send that URL to your peer.
139-- Check that your peer sees the same hash on the bootstrap page as you do.
140-- While you are still online, have your peer run the script on that page.
141-- When your peer's sbot is running, verify that they have the same hash at
142- their `http://localhost:8043/bootstrap` page.
143-- Proceed with gossip/pub onboarding.
144-
145-Note about prebuilds: the bootstrap npm registry server script serves prebuilds
146-as blobs from your npm prebuilds cache directory at `~/.npm/_prebuilds/`.
147-Therefore, if you use the bootstrap script to install scuttlebot to a machine
148-with a different architecture or node API, it might not fetch all of the host's
149-prebuilds. In this case, the new node may have to compile modules, and it may
150-result in a different bootstrap hash from the node that served the bootstrap
151-sript.
152-
153128 ## License
154129
155130 Copyright (C) 2017 Secure Scuttlebutt Consortium
156131
index.jsView
@@ -52,42 +52,8 @@
5252 }
5353 }
5454 }
5555
56-function pkgLockToRegistryPkgs(pkgLock, wsPort) {
57- // convert a package-lock.json file into data for serving as an npm registry
58- var hasNonBlobUrl = false
59- var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&')
60- var pkgs = {}
61- var queue = [pkgLock, pkgLock.name]
62- while (queue.length) {
63- var dep = queue.shift(), name = queue.shift()
64- if (name) {
65- var pkg = pkgs[name] || (pkgs[name] = {
66- _id: name,
67- name: name,
68- versions: {}
69- })
70- if (dep.version && dep.integrity && dep.resolved) {
71- if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true
72- pkg.versions[dep.version] = {
73- name: name,
74- version: dep.version,
75- dist: {
76- integrity: dep.integrity,
77- tarball: dep.resolved
78- }
79- }
80- }
81- }
82- if (dep.dependencies) for (var depName in dep.dependencies) {
83- queue.push(dep.dependencies[depName], depName)
84- }
85- }
86- pkgs._hasNonBlobUrl = hasNonBlobUrl
87- return pkgs
88-}
89-
9056 function npmLogin(registryAddress, cb) {
9157 var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
9258 var filename = path.join(os.homedir(), '.npmrc')
9359 fs.readFile(filename, 'utf8', function (err, data) {
@@ -371,9 +337,8 @@
371337 this.needShasum = this.npmConfig.needShasum
372338 this.wsPort = config.ws && Number(config.ws.port) || '8989'
373339 this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':'
374340 + this.wsPort + '/blobs/get/'
375- this.getBootstrapInfo = onceify(this.getBootstrapInfo, this)
376341 this.getMsg = memo({cache: lru(100)}, this.getMsg)
377342 }
378343
379344 SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
@@ -444,96 +409,8 @@
444409 if (!this.sbot.links2) return pull.empty()
445410 return getMentions(this.sbot.links2, name)
446411 }
447412
448-SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) {
449- var self = this
450- var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds')
451- var ids = {}
452- var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$')
453- fs.readdir(prebuildsDir, function (err, filenames) {
454- if (err) return cb(new Error(err.stack || err))
455- ;(function next(i) {
456- if (i >= filenames.length) return cb(null, ids)
457- var m = nameRegex.exec(filenames[i])
458- if (!m) return next(i+1)
459- var name = m[1]
460- fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) {
461- if (err) return cb(new Error(err.stack || err))
462- self.sbot.blobs.add(function (err, id) {
463- if (err) return cb(new Error(err.stack || err))
464- ids[name] = id
465- next(i+1)
466- })(pull.once(data))
467- })
468- })(0)
469- })
470-}
471-
472-SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) {
473- var self = this
474- if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin'))
475-
476- self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) {
477- if (err) return cb(new Error(err.stack || err))
478- var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort)
479- if (pkgs._hasNonBlobUrl) {
480- console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.')
481- }
482-
483- if (!sbotPkgLock.name) console.trace('missing pkg lock name')
484- if (!sbotPkgLock.version) console.trace('missing pkg lock version')
485-
486- var waiting = 2
487-
488- self.sbot.blobs.add(function (err, id) {
489- if (err) return next(new Error(err.stack || err))
490- var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {})
491- var versions = pkg.versions || (pkg.versions = {})
492- pkg.versions[sbotPkgLock.version] = {
493- name: sbotPkgLock.name,
494- version: sbotPkgLock.version,
495- dist: self.blobDist(id)
496- }
497- var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {})
498- distTags.latest = sbotPkgLock.version
499- next()
500- })(self.sbot.bootstrap.pack())
501-
502- var prebuilds
503- self.getLocalPrebuildsLinks(function (err, _prebuilds) {
504- if (err) return next(err)
505- prebuilds = _prebuilds
506- next()
507- })
508-
509- function next(err) {
510- if (err) return waiting = 0, cb(err)
511- if (--waiting) return
512- fs.readFile(path.join(__dirname, 'bootstrap.js'), {
513- encoding: 'utf8'
514- }, function (err, bootstrapScript) {
515- if (err) return cb(err)
516- var script = bootstrapScript + '\n' +
517- 'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' +
518- 'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2)
519-
520- self.sbot.blobs.add(function (err, id) {
521- if (err) return cb(new Error(err.stack || err))
522- var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
523- if (!m) return cb(new Error('bad blob id: ' + id))
524- cb(null, {
525- name: sbotPkgLock.name,
526- blob: id,
527- hashType: m[2],
528- hashBuf: Buffer.from(m[1], 'base64'),
529- })
530- })(pull.once(script))
531- })
532- }
533- })
534-}
535-
536413 SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
537414 if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
538415 else this.sbot.get(id, function (err, value) {
539416 if (err) return cb(err)
@@ -593,9 +470,8 @@
593470 }
594471 pathname = m[3]
595472 }
596473 if (pathname === '/') return this.serveHome()
597- if (pathname === '/bootstrap') return this.serveBootstrap()
598474 if (pathname === '/-/whoami') return this.serveWhoami()
599475 if (pathname === '/-/ping') return this.respond(200, true)
600476 if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
601477 if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
@@ -611,46 +487,18 @@
611487 Req.prototype.respondError = function (status, message) {
612488 this.respond(status, {error: message})
613489 }
614490
615-var bootstrapName = 'ssb-npm-bootstrap'
616-
617491 Req.prototype.serveHome = function () {
618492 var self = this
619493 self.res.writeHead(200, {'content-type': 'text/html'})
620494 var port = 8044
621495 self.res.end('<!doctype html><html><head><meta charset=utf-8>' +
622496 '<title>' + escapeHTML(pkg.name) + '</title></head><body>' +
623497 '<h1>' + escapeHTML(pkg.name) + '</h1>\n' +
624- '<p><a href="/bootstrap">Bootstrap</a></p>\n' +
625498 '</body></html>')
626499 }
627500
628-Req.prototype.serveBootstrap = function () {
629- var self = this
630- self.server.getBootstrapInfo(function (err, info) {
631- if (err) return self.respondError(err.stack || err)
632- var pkgNameText = info.name
633- var pkgTmpText = '/tmp/' + bootstrapName + '.js'
634- var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress
635- var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
636- var blobsHostname = httpHost + ':' + self.server.wsPort
637- var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob
638- var pkgHashText = info.hashBuf.toString('hex')
639- var hashCmd = info.hashType + 'sum'
640-
641- var script =
642- 'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' +
643- 'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' +
644- 'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' +
645- 'npm install -g ' + info.name + ' &&\n' +
646- 'sbot server'
647-
648- self.res.writeHead(200, {'content-type': 'text/plain'})
649- self.res.end(script)
650- })
651-}
652-
653501 Req.prototype.serveWhoami = function () {
654502 var self = this
655503 self.server.sbot.whoami(function (err, feed) {
656504 if (err) return self.respondError(err.stack || err)
bootstrap.jsView
@@ -1,238 +1,0 @@
1-var http = require('http')
2-var os = require('os')
3-var fs = require('fs')
4-var path = require('path')
5-var URL = require('url')
6-var http = require('http')
7-var https = require('https')
8-var crypto = require('crypto')
9-var Transform = require('stream').Transform
10-var proc = require('child_process')
11-
12-function usage(code) {
13- console.error('Usage: ' + process.argv[0] + ' ' + process.argv[1] + '\n' +
14- ' [--help] [--verbose]\n' +
15- ' [--blobs-host <host>] [--blobs-port <port>]\n' +
16- ' [--registry-host <host>] [--registry-port <port>]\n' +
17- ' [--blobs-remote <remote_address>]\n' +
18- ' [--] [<cmd> <args...>]')
19- process.exit(code)
20-}
21-
22-var ssbAppname = process.env.ssb_appname || 'ssb'
23-var ssbPath = process.env.ssb_path || path.join(os.homedir(), '.' + ssbAppname)
24-var blobsPath = path.join(ssbPath, 'blobs')
25-var blobsTmpPath = path.join(blobsPath, 'tmp')
26-
27-var numTmpBlobs = 0
28-var remotes = []
29-
30-function blobFilename(buf) {
31- var str = buf.toString('hex')
32- return path.join(blobsPath, 'sha256', str.slice(0, 2), str.slice(2))
33-}
34-
35-function formatHttpAddr(addr) {
36- return 'http://' + (typeof addr === 'string' ? 'unix:' + addr
37- : addr.family === 'IPv6' ? '[' + addr.address + ']:' + addr.port
38- : addr.address + ':' + addr.port)
39-}
40-
41-function serveBlobs(opts, req, res) {
42- var p = URL.parse(req.url)
43- if (p.pathname === '/') return serveStatus(res, 204)
44- var m = /^\/blobs\/get\/(&([A-Za-z0-9\/+]{43}=)\.sha256)$/.exec(p.pathname)
45- if (m) return serveBlobsGet(req, res, opts.remote, m[1], new Buffer(m[2], 'base64'))
46- return serveStatus(res, 404, 'Not Found')
47-}
48-
49-function serveRegistry(opts, req, res) {
50- var p = URL.parse(req.url)
51- var pathname = req.url.replace(/\?.*/, '')
52- if (pathname === '/') return serveStatus(res, 204)
53- if (pathname === '/-/ping') return serveStatus(res, 200, null, '"pong"')
54- if (/^\/-\//.test(pathname)) return serveStatus(404)
55- return servePkg(opts.pkgs, req, res, pathname.substr(1))
56-}
57-
58-function serveStatus(res, code, message, body) {
59- res.writeHead(code, message)
60- res.end(body || message)
61-}
62-
63-function serveStop(opts, req, res) {
64- var addr = req.socket.remoteAddress
65- if (addr !== '::1' && addr !== '127.0.0.1' && addr !== '::ffff:127.0.0.1') {
66- return serveStatus(res, 403)
67- }
68- serveStatus(res, 200)
69- opts.registryServer.close()
70- opts.blobsServer.close()
71-}
72-
73-function serveBlobsGet(req, res, remote, id, hash) {
74- var filename = blobFilename(hash)
75- getAddBlob(remote, id, hash, filename, function (err, stream) {
76- if (err) return serveStatus(res, 500, null, err.message)
77- if (!stream) return serveStatus(res, 404, 'Blob Not Found')
78- res.writeHead(200)
79- stream.pipe(res)
80- })
81-}
82-
83-function getAddBlob(remote, id, hash, filename, cb) {
84- fs.access(filename, fs.constants.R_OK, function (err) {
85- if (err && err.code === 'ENOENT') {
86- return fetchAddBlob(remote, id, hash, filename, cb)
87- }
88- if (err) return cb(err)
89- cb(null, fs.createReadStream(filename))
90- })
91-}
92-
93-function getRemoteBlob(remote, id, cb) {
94- if (/^https:\/\//.test(remote)) return https.get(remote + id, cb)
95- if (/^http:\/\//.test(remote)) return https.get(remote + id, cb)
96- return http.get('http://' + remote + '/blobs/get/' + id, cb)
97-}
98-
99-function mkdirp(dir, cb) {
100- fs.stat(dir, function (err, stats) {
101- if (!err) return cb()
102- fs.mkdir(dir, function (err) {
103- if (!err) return cb()
104- mkdirp(path.dirname(dir), function (err) {
105- if (err) return cb(err)
106- fs.mkdir(dir, cb)
107- })
108- })
109- })
110-}
111-
112-function rename(src, dest, cb) {
113- mkdirp(path.dirname(dest), function (err) {
114- if (err) return cb(err)
115- fs.rename(src, dest, cb)
116- })
117-}
118-
119-function fetchAddBlob(remote, id, hash, filename, cb) {
120- var req = getRemoteBlob(remote, id, function (res) {
121- req.removeListener('error', cb)
122- if (res.statusCode !== 200) return cb(new Error(res.statusMessage))
123- mkdirp(blobsTmpPath, function (err) {
124- if (err) return res.destroy(), cb(err)
125- var blobTmpPath = path.join(blobsTmpPath, Date.now() + '-' + numTmpBlobs++)
126- fs.open(blobTmpPath, 'w+', function (err, fd) {
127- if (err) return res.destroy(), cb(err)
128- var writeStream = fs.createWriteStream(null, {
129- fd: fd, flags: 'w+', autoClose: false})
130- var hasher = crypto.createHash('sha256')
131- var hashThrough = new Transform({
132- transform: function (data, encoding, cb) {
133- hasher.update(data)
134- cb(null, data)
135- }
136- })
137- res.pipe(hashThrough).pipe(writeStream, {end: false})
138- res.on('error', function (err) {
139- writeStream.end(function (err1) {
140- fs.unlink(blobTmpPath, function (err2) {
141- cb(err || err1 || err2)
142- })
143- })
144- })
145- hashThrough.on('end', function () {
146- var receivedHash = hasher.digest()
147- if (hash.compare(receivedHash)) {
148- writeStream.end(function (err) {
149- fs.unlink(blobTmpPath, function (err1) {
150- cb(err || err1)
151- })
152- })
153- } else {
154- res.unpipe(hashThrough)
155- rename(blobTmpPath, filename, function (err) {
156- if (err) return console.error(err)
157- cb(null, fs.createReadStream(null, {fd: fd, start: 0}))
158- })
159- }
160- })
161- })
162- })
163- })
164- req.on('error', cb)
165-}
166-
167-function servePkg(pkgs, req, res, pathname) {
168- var self = this
169- var parts = pathname.split('/')
170- var pkgName = parts.shift()
171- if (parts[0] === '-rev') return serveStatus(res, 501, 'Unpublish is not supported')
172- if (parts.length > 0) return serveStatus(res, 404)
173- var pkg = pkgs[pkgName] || {}
174- serveStatus(res, 200, null, JSON.stringify(pkg, 0, 2))
175-}
176-
177-function startServers(opts, cb) {
178- var waiting = 2
179- opts.blobsServer = http.createServer(serveBlobs.bind(null, opts))
180- .listen(opts.blobsPort, opts.blobsHost, function () {
181- if (opts.verbose) console.error('blobs listening on ' + formatHttpAddr(this.address()))
182- if (!--waiting) cb()
183- })
184- opts.registryServer = http.createServer(serveRegistry.bind(null, opts))
185- .listen(opts.registryPort, opts.registryHost, function () {
186- if (opts.verbose) console.error('registry listening on ' + formatHttpAddr(this.address()))
187- if (!--waiting) cb()
188- })
189-}
190-
191-process.nextTick(function () {
192- var args = process.argv.slice(2)
193- var opts = {
194- pkgs: exports.pkgs || {},
195- blobsPort: 8989,
196- registryPort: 8043,
197- }
198- var cmd
199-
200- getopts: while (args.length) {
201- var arg = args.shift()
202- switch (arg) {
203- case '--help': return usage(0)
204- case '--verbose': opts.verbose = true; break
205- case '--blobs-host': opts.blobsHost = args.shift(); break
206- case '--blobs-port': opts.blobsPort = args.shift(); break
207- case '--registry-port': opts.registryPort = args.shift(); break
208- case '--registry-host': opts.registryHost = args.shift(); break
209- case '--blobs-remote': opts.remote = args.shift(); break
210- case '--': cmd = args.shift(); break getopts
211- default: if (/^--/.test(arg)) return usage(1); cmd = arg; break getopts
212- }
213- }
214-
215- startServers(opts, function (err) {
216- if (err) throw err
217- if (cmd) {
218- var regHost = opts.registryHost || 'localhost'
219- var regHostname = regHost + ':' + opts.registryPort
220- if (cmd === 'npm' || cmd === 'npx') {
221- args.unshift('--registry=http://' + regHostname + '/')
222- args.unshift('--//' + regHostname + '/:_authToken=1')
223- args.unshift('--download=http://' + regHostname + '/-/prebuild/{name}-v{version}-{runtime}-v{abi}-{platform}{libc}-{arch}.tar.gz')
224- }
225- var child = proc.spawn(cmd, args, {
226- stdio: 'inherit'
227- })
228- child.on('exit', process.exit)
229- process.on('SIGINT', function () { child.kill('SIGINT') })
230- process.on('SIGTERM', function () { child.kill('SIGTERM') })
231- process.on('uncaughtException', function (e) {
232- console.error(e)
233- child.kill('SIGKILL')
234- process.exit(1)
235- })
236- }
237- })
238-})

Built with git-ssb-web