var http = require('http') var os = require('os') var path = require('path') var fs = require('fs') var crypto = require('crypto') var pkg = require('./package') var semver = require('semver') var toPull = require('stream-to-pull-stream') var pull = require('pull-stream') var tar = require('tar-stream') var zlib = require('zlib') var cat = require('pull-cat') var hash = require('pull-hash') var multicb = require('multicb') var memo = require('asyncmemo') var lru = require('hashlru') function escapeHTML(str) { return String(str) .replace(//g, '>') } function onceify(fn, self) { var cbs = [], err, data return function (cb) { if (fn) { cbs.push(cb) fn.call(self, function (_err, _data) { err = _err, data = _data var _cbs = cbs cbs = null while (_cbs.length) _cbs.shift()(err, data) }) fn = null } else if (cbs) { cbs.push(cb) } else { cb(err, data) } } } function once(cb) { var done return function (err, result) { if (done) { if (err) console.trace(err) } else { done = true cb(err, result) } } } function pkgLockToRegistryPkgs(pkgLock, wsPort) { // convert a package-lock.json file into data for serving as an npm registry var hasNonBlobUrl = false var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&') var pkgs = {} var queue = [pkgLock, pkgLock.name] while (queue.length) { var dep = queue.shift(), name = queue.shift() if (name) { var pkg = pkgs[name] || (pkgs[name] = { _id: name, name: name, versions: {} }) if (dep.version && dep.integrity && dep.resolved) { if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true pkg.versions[dep.version] = { name: name, version: dep.version, dist: { integrity: dep.integrity, tarball: dep.resolved } } } } if (dep.dependencies) for (var depName in dep.dependencies) { queue.push(dep.dependencies[depName], depName) } } pkgs._hasNonBlobUrl = hasNonBlobUrl return pkgs } function npmLogin(registryAddress, cb) { var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1' var filename = path.join(os.homedir(), '.npmrc') fs.readFile(filename, 'utf8', function (err, data) { if (err && err.code === 'ENOENT') data = '' else if (err) return cb(new Error(err.stack)) var lines = data ? data.split('\n') : [] if (lines.indexOf(tokenLine) > -1) return cb() var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '') var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine fs.appendFile(filename, line, cb) }) } function formatHost(host) { return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host } exports.name = 'npm-registry' exports.version = '1.0.0' exports.manifest = { getAddress: 'async' } exports.init = function (sbot, config) { var conf = config.npm || {} var port = conf.port || 8043 var host = conf.host || null var autoAuth = conf.autoAuth !== false var server = http.createServer(exports.respond(sbot, config)) var getAddress = onceify(function (cb) { server.on('error', cb) server.listen(port, host, function () { server.removeListener('error', cb) var regHost = formatHost(host || 'localhost') var regPort = this.address().port var regUrl = 'http://' + regHost + ':' + regPort + '/' if (autoAuth) npmLogin(regUrl, next) else next() function next(err) { cb(err, regUrl) } }) sbot.on('close', function () { server.close() }) }) getAddress(function (err, addr) { if (err) return console.error(err) console.log('[npm-registry] Listening on ' + addr) }) return { getAddress: getAddress } } exports.respond = function (sbot, config) { var reg = new SsbNpmRegistryServer(sbot, config) return function (req, res) { new Req(reg, req, res).serve() } } function publishMsg(sbot, value, cb) { var gotExpectedPrevious = false sbot.publish(value, function next(err, msg) { if (err && /^expected previous:/.test(err.message)) { // retry once on this error if (gotExpectedPrevious) return cb(err) gotExpectedPrevious = true return sbot.publish(value, next) } cb(err, msg) }) } function getDependencyBranches(sbot, id, cb) { // get ids of heads of tree of dependencyBranch message links to include all // the dependencies of the given tarball id. var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot) var msgs = {} var branched = {} var blobs = {} function addPkgById(id, cb) { if (blobs[id]) return cb() blobs[id] = true getPackageJsonCached(id, function (err, pkg) { if (err) return cb(err) var done = multicb() for (var name in pkg.dependencies || {}) { addPkgBySpec(name, pkg.dependencies[name], done()) } done(cb) }) } function addPkgBySpec(name, spec, cb) { var done = multicb() pull( getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}), pull.map(function (mention) { addPkgById(mention.link, done()) return packageLinks(sbot, mention.author, mention.link, name, spec) }), pull.flatten(), pull.drain(function (msg) { var c = msg && msg.value && msg.value.content if (!c) return msgs[msg.key] = msg.value if (Array.isArray(c.dependencyBranch)) { for (var k = 0; k < c.dependencyBranch.length; k++) { branched[c.dependencyBranch[k]] = true } } }, function (err) { if (err) return cb(err) done(cb) }) ) } addPkgById(id, function (err) { if (err) return cb(err) var ids = [] for (var key in msgs) { if (!branched[key]) ids.push(key) } cb(null, ids) }) } function packageLinks(sbot, feed, id, name, spec) { function matches(mention) { var data = mention && mention.link === id && decodeName(mention.name) return data && data.name === name && (spec ? semver.satisfies(data.version, spec) : true) } return pull( sbot.links({ source: feed, dest: id, rel: 'mentions', values: true, }), pull.filter(function (msg) { var c = msg && msg.value && msg.value.content return c && Array.isArray(c.mentions) && c.mentions.some(matches) }) ) } function getVersionBranches(sbot, link, cb) { var data = decodeName(link.name) var msgs = {}, branched = {} pull( getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}), pull.map(function (mention) { return packageLinks(sbot, mention.author, mention.link, data.name) }), pull.flatten(), pull.drain(function (msg) { var c = msg && msg.value && msg.value.content if (!c) return msgs[msg.key] = msg.value if (Array.isArray(c.versionBranch)) { for (var k = 0; k < c.versionBranch.length; k++) { branched[c.versionBranch[k]] = true } } }, function (err) { if (err) return cb(err) var ids = [] for (var key in msgs) { if (!branched[key]) ids.push(key) } cb(null, ids) }) ) } // For each dependency that is not a bundledDependency, get message ids for // that dependency name + version. function publishSingleMention(sbot, mention, cb) { // Calculate dependencyBranch and versionBranch message ids. var value = { type: 'npm-packages', mentions: [mention] } var done = multicb({pluck: 1, spread: true}) getDependencyBranches(sbot, mention.link, done()) getVersionBranches(sbot, mention, done()) done(function (err, dependencyBranches, versionBranches) { if (err) return cb(err) value.dependencyBranch = dependencyBranches || undefined value.versionBranch = versionBranches || undefined publishMsg(sbot, value, cb) }) } function publishMentions(sbot, mentions, cb) { // console.error("publishing %s mentions", mentions.length) if (mentions.length === 0) return cb(new Error('Empty mentions list')) // if it is just one mention, fetch and add useful metadata if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb) publishMsg(sbot, { type: 'npm-packages', mentions: mentions, }, cb) } exports.publishPkgMentions = function (sbot, mentions, cb) { // try to fit the mentions into as few messages as possible, // while fitting under the message size limit. var msgs = [] ;(function next(i, chunks) { if (i >= mentions.length) return cb(null, msgs) var chunkLen = Math.ceil(mentions.length / chunks) publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) { if (err && /must not be large/.test(err.message)) return next(i, chunks + 1) if (err && msgs.length) return onPartialPublish(err) if (err) return cb(err) msgs.push(msg) next(i + chunkLen, chunks) }) })(0, 1) function onPartialPublish(err) { var remaining = mentions.length - i return cb(new Error('Published messages ' + msgs.map(function (msg) { return msg.key }).join(', ') + ' ' + 'but failed to publish remaining ' + remaining + ': ' + (err.stack || err))) } } exports.expandPkgMentions = function (sbot, mentions, props, cb) { cb = once(cb) var waiting = 0 var expandedMentions = mentions.map(function (link) { var id = link && link.link if (!id) return link waiting++ var newLink = {} for (var k in link) newLink[k] = link[k] getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) { if (err) return cb(err) for (var k in props) newLink[k] = pkg[k] if (!--waiting) next() }) return newLink }) if (!waiting) next() function next() { cb(null, expandedMentions) } } function SsbNpmRegistryServer(sbot, config) { this.sbot = sbot this.config = config this.npmConfig = config.npm || {} this.host = this.npmConfig.host || 'localhost' this.fetchAll = this.npmConfig.fetchAll this.links2 = sbot.links2 if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin') this.wsPort = config.ws && Number(config.ws.port) || '8989' this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':' + this.wsPort + '/blobs/get/' this.getBootstrapInfo = onceify(this.getBootstrapInfo, this) this.getMsg = memo({cache: lru(100)}, this.getMsg) } SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype) SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) { var self = this if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push')) ;(function next(i) { if (i >= ids.length) return cb() self.sbot.blobs.push(ids[i], function (err) { if (err) return cb(err) next(i+1) }) })(0) } function getBlob(sbot, id, cb) { var blobs = sbot.blobs blobs.size(id, function (err, size) { if (typeof size === 'number') cb(null, blobs.get(id)) else blobs.want(id, function (err, got) { if (err) cb(err) else if (!got) cb('missing blob ' + id) else cb(null, blobs.get(id)) }) }) } SsbNpmRegistryServer.prototype.blobDist = function (id) { var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id) if (!m) throw new Error('bad blob id: ' + id) return { integrity: m[2] + '-' + m[1], tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id } } function getMentions(links2, name) { return links2.read({ query: [ {$filter: {rel: ['mentions', name, {$gt: true}]}}, {$filter: {dest: {$prefix: '&'}}}, {$map: { name: ['rel', 1], size: ['rel', 2], link: 'dest', author: 'source', ts: 'ts' }} ] }) } SsbNpmRegistryServer.prototype.getMentions = function (name) { return getMentions(this.links2, name) } SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) { var self = this var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds') var ids = {} var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$') fs.readdir(prebuildsDir, function (err, filenames) { if (err) return cb(new Error(err.stack || err)) ;(function next(i) { if (i >= filenames.length) return cb(null, ids) var m = nameRegex.exec(filenames[i]) if (!m) return next(i+1) var name = m[1] fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) { if (err) return cb(new Error(err.stack || err)) self.sbot.blobs.add(function (err, id) { if (err) return cb(new Error(err.stack || err)) ids[name] = id next(i+1) })(pull.once(data)) }) })(0) }) } SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) { var self = this if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin')) self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) { if (err) return cb(new Error(err.stack || err)) var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort) if (pkgs._hasNonBlobUrl) { console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.') } if (!sbotPkgLock.name) console.trace('missing pkg lock name') if (!sbotPkgLock.version) console.trace('missing pkg lock version') var waiting = 2 self.sbot.blobs.add(function (err, id) { if (err) return next(new Error(err.stack || err)) var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {}) var versions = pkg.versions || (pkg.versions = {}) pkg.versions[sbotPkgLock.version] = { name: sbotPkgLock.name, version: sbotPkgLock.version, dist: self.blobDist(id) } var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {}) distTags.latest = sbotPkgLock.version next() })(self.sbot.bootstrap.pack()) var prebuilds self.getLocalPrebuildsLinks(function (err, _prebuilds) { if (err) return next(err) prebuilds = _prebuilds next() }) function next(err) { if (err) return waiting = 0, cb(err) if (--waiting) return fs.readFile(path.join(__dirname, 'bootstrap.js'), { encoding: 'utf8' }, function (err, bootstrapScript) { if (err) return cb(err) var script = bootstrapScript + '\n' + 'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' + 'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2) self.sbot.blobs.add(function (err, id) { if (err) return cb(new Error(err.stack || err)) var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id) if (!m) return cb(new Error('bad blob id: ' + id)) cb(null, { name: sbotPkgLock.name, blob: id, hashType: m[2], hashBuf: Buffer.from(m[1], 'base64'), }) })(pull.once(script)) }) } }) } SsbNpmRegistryServer.prototype.getMsg = function (id, cb) { if (this.sbot.ooo) return this.sbot.ooo.get(id, cb) else this.sbot.get(id, function (err, value) { if (err) return cb(err) cb(null, {key: id, value: value}) }) } SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) { var self = this var stack = heads.slice() var seen = {} return function (abort, cb) { if (abort) return cb(abort) for (var id; stack.length && seen[id = stack.pop()];); if (!id) return cb(true) seen[id] = true // TODO: use DFS self.getMsg(id, function (err, msg) { if (err) return cb(new Error(err.stack || err)) var c = msg && msg.value && msg.value.content var links = c && c[prop] if (Array.isArray(links)) { stack.push.apply(stack, links) } else if (links) { stack.push(links) } cb(null, msg) }) } } function Req(server, req, res) { this.server = server this.req = req this.res = res this.blobsToPush = [] this.fetchAll = server.fetchAll != null ? server.fetchAll : true } Req.prototype.serve = function () { console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, '')) var pathname = this.req.url.replace(/\?.*/, '') var m if ((m = /^\/(%25.*sha256)(\/.*)$/.exec(pathname))) { try { this.headMsgId = decodeURIComponent(m[1]) } catch(e) { return this.respondError(400, e.stack || e) } pathname = m[2] } if (pathname === '/') return this.serveHome() if (pathname === '/bootstrap') return this.serveBootstrap() if (pathname === '/-/whoami') return this.serveWhoami() if (pathname === '/-/ping') return this.respond(200, true) if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1() if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1]) if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1)) return this.respond(404) } Req.prototype.respond = function (status, message) { this.res.writeHead(status, {'content-type': 'application/json'}) this.res.end(message && JSON.stringify(message, 0, 2)) } Req.prototype.respondError = function (status, message) { this.respond(status, {error: message}) } var bootstrapName = 'ssb-npm-bootstrap' Req.prototype.serveHome = function () { var self = this self.res.writeHead(200, {'content-type': 'text/html'}) var port = 8044 self.res.end('
' + '