var http = require('http') var os = require('os') var path = require('path') var fs = require('fs') var crypto = require('crypto') var pkg = require('./package') function pullOnce(data) { var ended return function (abort, cb) { if (ended || (ended = abort)) return cb(ended) ended = true cb(null, data) } } function escapeHTML(str) { return String(str) .replace(//g, '>') } function onceify(fn, self) { var cbs = [], err, data return function (cb) { if (fn) { cbs.push(cb) fn.call(self, function (_err, _data) { err = _err, data = _data var _cbs = cbs cbs = null while (_cbs.length) _cbs.shift()(err, data) }) fn = null } else if (cbs) { cbs.push(cb) } else { cb(err, data) } } } function pkgLockToRegistryPkgs(pkgLock, wsPort) { // convert a package-lock.json file into data for serving as an npm registry var hasNonBlobUrl = false var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&') var pkgs = {} var queue = [pkgLock, pkgLock.name] while (queue.length) { var dep = queue.shift(), name = queue.shift() if (name) { var pkg = pkgs[name] || (pkgs[name] = { _id: name, name: name, versions: {} }) if (dep.version && dep.integrity && dep.resolved) { if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true pkg.versions[dep.version] = { dist: { integrity: dep.integrity, tarball: dep.resolved } } } } if (dep.dependencies) for (var depName in dep.dependencies) { queue.push(dep.dependencies[depName], depName) } } pkgs._hasNonBlobUrl = hasNonBlobUrl return pkgs } function npmLogin(registryAddress, cb) { var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1' var filename = path.join(os.homedir(), '.npmrc') fs.readFile(filename, 'utf8', function (err, data) { if (err && err.code === 'ENOENT') data = '' else if (err) return cb(new Error(err.stack)) var lines = data ? data.split('\n') : [] if (lines.indexOf(tokenLine) > -1) return cb() var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '') var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine fs.appendFile(filename, line, cb) }) } function formatHost(host) { return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host } exports.name = 'npm-registry' exports.version = '1.0.0' exports.manifest = { getAddress: 'async' } exports.init = function (sbot, config) { var conf = config.npm || {} var port = conf.port || 8043 var host = conf.host || null var autoAuth = conf.autoAuth !== false var server = http.createServer(exports.respond(sbot, config)) var getAddress = onceify(function (cb) { server.on('error', cb) server.listen(port, host, function () { server.removeListener('error', cb) var regHost = formatHost(host || 'localhost') var regPort = this.address().port var regUrl = 'http://' + regHost + ':' + regPort + '/' if (autoAuth) npmLogin(regUrl, next) else next() function next(err) { cb(err, regUrl) } }) sbot.on('close', function () { server.close() }) }) getAddress(function (err, addr) { if (err) return console.error(err) console.log('[npm-registry] Listening on ' + addr) }) return { getAddress: getAddress } } exports.respond = function (sbot, config) { var reg = new SsbNpmRegistryServer(sbot, config) return function (req, res) { new Req(reg, req, res).serve() } } function publishMsg(sbot, value, cb) { var gotExpectedPrevious = false sbot.publish(value, function next(err, msg) { if (err && /^expected previous:/.test(err.message)) { // retry once on this error if (gotExpectedPrevious) return cb(err) gotExpectedPrevious = true return sbot.publish(value, next) } cb(err, msg) }) } function publishMentions(sbot, mentions, cb) { // console.error("publishing %s mentions", mentions.length) if (mentions.length === 0) return cb(new Error('Empty mentions list')) publishMsg(sbot, { type: 'npm-packages', mentions: mentions, }, cb) } exports.publishPkgMentions = function (sbot, mentions, cb) { // try to fit the mentions into as few messages as possible, // while fitting under the message size limit. var msgs = [] ;(function next(i, chunks) { if (i >= mentions.length) return cb(null, msgs) var chunkLen = Math.ceil(mentions.length / chunks) publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) { if (err && /must not be large/.test(err.message)) return next(i, chunks + 1) if (err && msgs.length) return onPartialPublish(err) if (err) return cb(err) msgs.push(msg) next(i + chunkLen, chunks) }) })(0, 1) function onPartialPublish(err) { var remaining = mentions.length - i return cb(new Error('Published messages ' + msgs.map(function (msg) { return msg.key }).join(', ') + ' ' + 'but failed to publish remaining ' + remaining + ': ' + (err.stack || err))) } } function SsbNpmRegistryServer(sbot, config) { this.sbot = sbot this.config = config this.npmConfig = config.npm || {} this.host = this.npmConfig.host || 'localhost' this.links2 = sbot.links2 if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin') this.wsPort = config.ws && Number(config.ws.port) || '8989' this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':' + this.wsPort + '/blobs/get/' this.getBootstrapInfo = onceify(this.getBootstrapInfo, this) } SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype) SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) { var self = this if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push')) ;(function next(i) { if (i >= ids.length) return cb() self.sbot.blobs.push(ids[i], function (err) { if (err) return cb(err) next(i+1) }) })(0) } SsbNpmRegistryServer.prototype.blobDist = function (id) { var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id) if (!m) throw new Error('bad blob id: ' + id) return { integrity: m[2] + '-' + m[1], tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id } } SsbNpmRegistryServer.prototype.getMentions = function (name) { return this.links2.read({ query: [ {$filter: {rel: ['mentions', name, {$gt: true}]}}, {$filter: {dest: {$prefix: '&'}}}, {$map: { name: ['rel', 1], size: ['rel', 2], link: 'dest', author: 'source', ts: 'ts' }} ] }) } SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) { var self = this var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds') var ids = {} var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$') fs.readdir(prebuildsDir, function (err, filenames) { if (err) return cb(new Error(err.stack || err)) ;(function next(i) { if (i >= filenames.length) return cb(null, ids) var m = nameRegex.exec(filenames[i]) if (!m) return next(i+1) var name = m[1] fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) { if (err) return cb(new Error(err.stack || err)) self.sbot.blobs.add(function (err, id) { if (err) return cb(new Error(err.stack || err)) ids[name] = id next(i+1) })(pullOnce(data)) }) })(0) }) } SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) { var self = this if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin')) self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) { if (err) return cb(new Error(err.stack || err)) var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort) if (pkgs._hasNonBlobUrl) { console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.') } if (!sbotPkgLock.name) console.trace('missing pkg lock name') if (!sbotPkgLock.version) console.trace('missing pkg lock version') var waiting = 2 self.sbot.blobs.add(function (err, id) { if (err) return next(new Error(err.stack || err)) var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {}) var versions = pkg.versions || (pkg.versions = {}) pkg.versions[sbotPkgLock.version] = { dist: self.blobDist(id) } var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {}) distTags.latest = sbotPkgLock.version next() })(self.sbot.bootstrap.pack()) var prebuilds self.getLocalPrebuildsLinks(function (err, _prebuilds) { if (err) return next(err) prebuilds = _prebuilds next() }) function next(err) { if (err) return waiting = 0, cb(err) if (--waiting) return fs.readFile(path.join(__dirname, 'bootstrap.js'), { encoding: 'utf8' }, function (err, bootstrapScript) { if (err) return cb(err) var script = bootstrapScript + '\n' + 'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' + 'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2) self.sbot.blobs.add(function (err, id) { if (err) return cb(new Error(err.stack || err)) var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id) if (!m) return cb(new Error('bad blob id: ' + id)) cb(null, { name: sbotPkgLock.name, blob: id, hashType: m[2], hashBuf: Buffer.from(m[1], 'base64'), }) })(pullOnce(script)) }) } }) } function Req(server, req, res) { this.server = server this.req = req this.res = res this.blobsToPush = [] } Req.prototype.serve = function () { // console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, '')) var pathname = this.req.url.replace(/\?.*/, '') var m if (pathname === '/') return this.serveHome() if (pathname === '/bootstrap') return this.serveBootstrap() if (pathname === '/-/whoami') return this.serveWhoami() if (pathname === '/-/ping') return this.respond(200, true) if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1() if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1]) if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1)) return this.respond(404) } Req.prototype.respond = function (status, message) { this.res.writeHead(status, {'content-type': 'application/json'}) this.res.end(message && JSON.stringify(message, 0, 2)) } Req.prototype.respondError = function (status, message) { this.respond(status, {error: message}) } var bootstrapName = 'ssb-npm-bootstrap' Req.prototype.serveHome = function () { var self = this self.res.writeHead(200, {'content-type': 'text/html'}) var port = 8044 self.res.end('
' + '