var http = require('http') var os = require('os') var path = require('path') var fs = require('fs') var crypto = require('crypto') var pkg = require('./package') function pullOnce(data) { var ended return function (abort, cb) { if (ended || (ended = abort)) return cb(ended) ended = true cb(null, data) } } function escapeHTML(str) { return String(str) .replace(//g, '>') } function onceify(fn, self) { var cbs = [], err, data return function (cb) { if (fn) { cbs.push(cb) fn.call(self, function (_err, _data) { err = _err, data = _data var _cbs = cbs cbs = null while (_cbs.length) _cbs.shift()(err, data) }) fn = null } else if (cbs) { cbs.push(cb) } else { cb(err, data) } } } function pkgLockToRegistryPkgs(pkgLock, wsPort) { // convert a package-lock.json file into data for serving as an npm registry var hasNonBlobUrl = false var blobUrlRegex = new RegExp('^http://localhost:' + wsPort + '/blobs/get/&') var pkgs = {} var queue = [pkgLock, pkgLock.name] while (queue.length) { var dep = queue.shift(), name = queue.shift() if (name) { var pkg = pkgs[name] || (pkgs[name] = { _id: name, name: name, versions: {} }) if (dep.version && dep.integrity && dep.resolved) { if (!hasNonBlobUrl && !blobUrlRegex.test(dep.resolved)) hasNonBlobUrl = true pkg.versions[dep.version] = { dist: { integrity: dep.integrity, tarball: dep.resolved } } } } if (dep.dependencies) for (var depName in dep.dependencies) { queue.push(dep.dependencies[depName], depName) } } pkgs._hasNonBlobUrl = hasNonBlobUrl return pkgs } function npmLogin(registryAddress, cb) { var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1' var filename = path.join(os.homedir(), '.npmrc') fs.readFile(filename, 'utf8', function (err, data) { if (err && err.code === 'ENOENT') data = '' else if (err) return cb(new Error(err.stack)) var lines = data ? data.split('\n') : [] if (lines.indexOf(tokenLine) > -1) return cb() var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '') var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine fs.appendFile(filename, line, cb) }) } function formatHost(host) { return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host } exports.name = 'npm-registry' exports.version = '1.0.0' exports.manifest = { getAddress: 'async' } exports.init = function (sbot, config) { var conf = config.npm || {} var port = conf.port || 8043 var host = conf.host || null var autoAuth = conf.autoAuth !== false var server = http.createServer(exports.respond(sbot, config)) var getAddress = onceify(function (cb) { server.on('error', cb) server.listen(port, host, function () { server.removeListener('error', cb) var regHost = formatHost(host || 'localhost') var regPort = this.address().port var regUrl = 'http://' + regHost + ':' + regPort + '/' if (autoAuth) npmLogin(regUrl, next) else next() function next(err) { cb(err, regUrl) } }) sbot.on('close', function () { server.close() }) }) getAddress(function (err, addr) { if (err) return console.error(err) console.log('[npm-registry] Listening on ' + addr) }) return { getAddress: getAddress } } exports.respond = function (sbot, config) { var reg = new SsbNpmRegistryServer(sbot, config) return function (req, res) { new Req(reg, req, res).serve() } } function publishMsg(sbot, value, cb) { var gotExpectedPrevious = false sbot.publish(value, function next(err, msg) { if (err && /^expected previous:/.test(err.message)) { // retry once on this error if (gotExpectedPrevious) return cb(err) gotExpectedPrevious = true return sbot.publish(value, next) } cb(err, msg) }) } function publishMentions(sbot, mentions, cb) { // console.error("publishing %s mentions", mentions.length) if (mentions.length === 0) return cb(new Error('Empty mentions list')) publishMsg(sbot, { type: 'npm-packages', mentions: mentions, }, cb) } exports.publishPkgMentions = function (sbot, mentions, cb) { // try to fit the mentions into as few messages as possible, // while fitting under the message size limit. var msgs = [] ;(function next(i, chunks) { if (i >= mentions.length) return cb(null, msgs) var chunkLen = Math.ceil(mentions.length / chunks) publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) { if (err && /must not be large/.test(err.message)) return next(i, chunks + 1) if (err && msgs.length) return onPartialPublish(err) if (err) return cb(err) msgs.push(msg) next(i + chunkLen, chunks) }) })(0, 1) function onPartialPublish(err) { var remaining = mentions.length - i return cb(new Error('Published messages ' + msgs.map(function (msg) { return msg.key }).join(', ') + ' ' + 'but failed to publish remaining ' + remaining + ': ' + (err.stack || err))) } } function SsbNpmRegistryServer(sbot, config) { this.sbot = sbot this.config = config this.npmConfig = config.npm || {} this.host = this.npmConfig.host || 'localhost' this.links2 = sbot.links2 if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin') this.wsPort = config.ws && Number(config.ws.port) || '8989' this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':' + this.wsPort + '/blobs/get/' this.getBootstrapInfo = onceify(this.getBootstrapInfo, this) } SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype) SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) { var self = this if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push')) ;(function next(i) { if (i >= ids.length) return cb() self.sbot.blobs.push(ids[i], function (err) { if (err) return cb(err) next(i+1) }) })(0) } SsbNpmRegistryServer.prototype.blobDist = function (id) { var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id) if (!m) throw new Error('bad blob id: ' + id) return { integrity: m[2] + '-' + m[1], tarball: 'http://localhost:' + this.wsPort + '/blobs/get/' + id } } SsbNpmRegistryServer.prototype.getMentions = function (name) { return this.links2.read({ query: [ {$filter: {rel: ['mentions', name, {$gt: true}]}}, {$filter: {dest: {$prefix: '&'}}}, {$map: { name: ['rel', 1], size: ['rel', 2], link: 'dest', author: 'source', ts: 'ts' }} ] }) } SsbNpmRegistryServer.prototype.getLocalPrebuildsLinks = function (cb) { var self = this var prebuildsDir = path.join(os.homedir(), '.npm', '_prebuilds') var ids = {} var nameRegex = new RegExp('^http-' + self.host.replace(/\./g, '.') + '-(?:[0-9]+)-prebuild-(.*)$') fs.readdir(prebuildsDir, function (err, filenames) { if (err) return cb(new Error(err.stack || err)) ;(function next(i) { if (i >= filenames.length) return cb(null, ids) var m = nameRegex.exec(filenames[i]) if (!m) return next(i+1) var name = m[1] fs.readFile(path.join(prebuildsDir, filenames[i]), function (err, data) { if (err) return cb(new Error(err.stack || err)) self.sbot.blobs.add(function (err, id) { if (err) return cb(new Error(err.stack || err)) ids[name] = id next(i+1) })(pullOnce(data)) }) })(0) }) } SsbNpmRegistryServer.prototype.getBootstrapInfo = function (cb) { var self = this if (!self.sbot.bootstrap) return cb(new Error('missing sbot bootstrap plugin')) self.sbot.bootstrap.getPackageLock(function (err, sbotPkgLock) { if (err) return cb(new Error(err.stack || err)) var pkgs = pkgLockToRegistryPkgs(sbotPkgLock, self.wsPort) if (pkgs._hasNonBlobUrl) { console.error('[npm-registry] Warning: package-lock.json has non-blob URLs. Bootstrap installation may not be fully peer-to-peer.') } if (!sbotPkgLock.name) console.trace('missing pkg lock name') if (!sbotPkgLock.version) console.trace('missing pkg lock version') var waiting = 2 self.sbot.blobs.add(function (err, id) { if (err) return next(new Error(err.stack || err)) var pkg = pkgs[sbotPkgLock.name] || (pkgs[sbotPkgLock.name] = {}) var versions = pkg.versions || (pkg.versions = {}) pkg.versions[sbotPkgLock.version] = { dist: self.blobDist(id) } var distTags = pkg['dist-tags'] || (pkg['dist-tags'] = {}) distTags.latest = sbotPkgLock.version next() })(self.sbot.bootstrap.pack()) var prebuilds self.getLocalPrebuildsLinks(function (err, _prebuilds) { if (err) return next(err) prebuilds = _prebuilds next() }) function next(err) { if (err) return waiting = 0, cb(err) if (--waiting) return fs.readFile(path.join(__dirname, 'bootstrap.js'), { encoding: 'utf8' }, function (err, bootstrapScript) { if (err) return cb(err) var script = bootstrapScript + '\n' + 'exports.pkgs = ' + JSON.stringify(pkgs, 0, 2) + '\n' + 'exports.prebuilds = ' + JSON.stringify(prebuilds, 0, 2) self.sbot.blobs.add(function (err, id) { if (err) return cb(new Error(err.stack || err)) var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id) if (!m) return cb(new Error('bad blob id: ' + id)) cb(null, { name: sbotPkgLock.name, blob: id, hashType: m[2], hashBuf: Buffer.from(m[1], 'base64'), }) })(pullOnce(script)) }) } }) } function Req(server, req, res) { this.server = server this.req = req this.res = res this.blobsToPush = [] } Req.prototype.serve = function () { // console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, '')) var pathname = this.req.url.replace(/\?.*/, '') var m if (pathname === '/') return this.serveHome() if (pathname === '/bootstrap') return this.serveBootstrap() if (pathname === '/-/whoami') return this.serveWhoami() if (pathname === '/-/ping') return this.respond(200, true) if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1() if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1]) if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1)) return this.respond(404) } Req.prototype.respond = function (status, message) { this.res.writeHead(status, {'content-type': 'application/json'}) this.res.end(message && JSON.stringify(message, 0, 2)) } Req.prototype.respondError = function (status, message) { this.respond(status, {error: message}) } var bootstrapName = 'ssb-npm-bootstrap' Req.prototype.serveHome = function () { var self = this self.res.writeHead(200, {'content-type': 'text/html'}) var port = 8044 self.res.end('' + '' + escapeHTML(pkg.name) + '' + '

' + escapeHTML(pkg.name) + '

\n' + '

Bootstrap

\n' + '') } Req.prototype.serveBootstrap = function () { var self = this self.server.getBootstrapInfo(function (err, info) { if (err) return this.respondError(err.stack || err) var pkgNameText = info.name var pkgTmpText = '/tmp/' + bootstrapName + '.js' var host = String(self.req.headers.host).replace(/:[0-9]*$/, '') || self.req.socket.localAddress var httpHost = /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host var blobsHostname = httpHost + ':' + self.server.wsPort var tarballLink = 'http://' + blobsHostname + '/blobs/get/' + info.blob var pkgHashText = info.hashBuf.toString('hex') var hashCmd = info.hashType + 'sum' var script = 'wget \'' + tarballLink + '\' -O ' + pkgTmpText + ' &&\n' + 'echo ' + pkgHashText + ' ' + pkgTmpText + ' | ' + hashCmd + ' -c &&\n' + 'node ' + pkgTmpText + ' --blobs-remote ' + blobsHostname + ' -- ' + 'npm install -g ' + info.name + ' &&\n' + 'sbot server' self.res.writeHead(200, {'content-type': 'text/plain'}) self.res.end(script) }) } Req.prototype.serveWhoami = function () { var self = this self.server.sbot.whoami(function (err, feed) { if (err) return self.respondError(err.stack || err) self.respond(200, {username: feed.id}) }) } Req.prototype.serveUser1 = function () { this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'}) } function decodeName(name) { var parts = name.replace(/\.tgz$/, '').split(':') return { name: parts[1], version: parts[2], distTag: parts[3], } } Req.prototype.servePkg = function (pathname) { var self = this var parts = pathname.split('/') var pkgName = parts.shift() if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported') if (parts.length > 0) return this.respondError(404) if (self.req.method === 'PUT') return self.publishPkg(pkgName) var obj = { _id: pkgName, name: pkgName, 'dist-tags': {}, versions: {} } var oldest, newest var getMention = self.server.getMentions({$prefix: 'npm:' + pkgName + ':'}) getMention(null, function next(err, mention) { if (err === true) return self.respond(200, obj) if (err) return self.respondError(500, err.stack || err) var data = decodeName(mention.name) if (!data.version) return if (data.distTag) obj['dist-tags'][data.distTag] = data.version obj.versions[data.version] = { author: { url: mention.author }, dist: self.server.blobDist(mention.link) } getMention(null, next) }) } Req.prototype.servePrebuild = function (name) { var self = this var getMention = self.server.getMentions('prebuild:' + name) var blobsByAuthor = {/* : BlobId */} getMention(null, function next(err, link) { if (err === true) return done() if (err) return self.respondError(500, err.stack || err) blobsByAuthor[link.author] = link.link getMention(null, next) }) function done() { var authorsByLink = {/* : [FeedId...] */} var blobId for (var feed in blobsByAuthor) { var blob = blobId = blobsByAuthor[feed] var feeds = authorsByLink[blob] || (authorsByLink[blob] = []) feeds.push(feed) } switch (Object.keys(authorsByLink).length) { case 0: return self.respondError(404, 'Not Found') case 1: self.res.writeHead(303, {Location: self.server.blobsPrefix + blobId}) return self.res.end() default: return self.respond(300, {choices: authorsByLink}) } } } var localhosts = { '::1': true, '127.0.0.1': true, '::ffff:127.0.0.1': true, } Req.prototype.publishPkg = function (pkgName) { var self = this var remoteAddress = self.req.socket.remoteAddress if (!(remoteAddress in localhosts)) { return self.respondError(403, 'You may not publish as this user.') } var chunks = [] self.req.on('data', function (data) { chunks.push(data) }) self.req.on('end', function () { var data try { data = JSON.parse(Buffer.concat(chunks)) } catch(e) { return self.respondError(400, e.stack) } return self.publishPkg2(pkgName, data || {}) }) } Req.prototype.publishPkg2 = function (name, data) { var self = this if (data.users) console.trace('[npm-registry] users property is not supported') var attachments = data._attachments || {} var links = {/* -.tgz: {link: , size: number} */} var waiting = 0 Object.keys(attachments).forEach(function (filename) { waiting++ var tarball = new Buffer(attachments[filename].data, 'base64') var length = attachments[filename].length if (length && length !== tarball.length) return self.respondError(400, 'Length mismatch for attachment \'' + filename + '\'') self.server.sbot.blobs.add(function (err, id) { if (err) return self.respondError(500, 'Adding attachment \'' + filename + '\' as blob failed') self.blobsToPush.push(id) links[filename] = {link: id, size: tarball.length} if (!--waiting) next() })(pullOnce(tarball)) }) function next() { try { self.publishPkg3(name, data, links) } catch(e) { self.respondError(500, e.stack || e) } } } Req.prototype.publishPkg3 = function (name, data, links) { var self = this var versions = data.versions || {} var linksByVersion = {/* : link */} // associate tarball blobs with versions for (var version in versions) { var pkg = versions[version] if (!pkg) return self.respondError(400, 'Bad package object') if (!pkg.dist) return self.respondError(400, 'Missing package dist property') if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property') if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported') var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball) if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'') var filename = m[1] var link = links[filename] if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'') // TODO?: try to find missing tarball mentioned in other messages if (pkg.version && pkg.version !== version) return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version]) linksByVersion[version] = link link.version = version } // associate blobs with dist-tags var tags = data['dist-tags'] || {} for (var tag in tags) { var version = tags[tag] var link = linksByVersion[version] if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.') // TODO?: support setting dist-tag without version, // by looking up a tarball blob for the version link.tag = tag } // compute blob links to publish var mentions = [] for (var filename in links) { var link = links[filename] || {} if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata') mentions.push({ name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''), link: link.link, size: link.size, }) } return self.publishPkgs(mentions) } Req.prototype.publishPkgs = function (mentions) { var self = this exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) { if (err) self.respondError(500, err.stack || err) self.server.pushBlobs(self.blobsToPush, function (err) { if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err)) self.respond(201) console.log(msgs.map(function (msg) { return msg.key }).join('\n')) }) }) }