var http = require('http') var os = require('os') var path = require('path') var fs = require('fs') function pullOnce(data) { var ended return function (abort, cb) { if (ended || (ended = abort)) return cb(ended) ended = true cb(null, data) } } exports.name = 'npm-registry' exports.version = '1.0.0' exports.manifest = { getAddress: 'async' } exports.init = function (sbot, config) { var port = config.npm ? config.npm.port : 8043 var host = config.npm && config.npm.host || 'localhost' var registryAddress var getAddressCbs = [] var server = http.createServer(exports.respond(sbot, config)) server.listen(port, host, function () { registryAddress = 'http://' + host + ':' + this.address().port + '/' if (config.npm && config.npm.autoAuth !== false) login() console.log('[npm-registry] Listening on ' + registryAddress) while (getAddressCbs.length) getAddressCbs.shift()(null, registryAddress) }) sbot.on('close', function () { server.close() }) function login() { var filename = path.join(os.homedir(), '.npmrc') var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1' var lines = fs.readFileSync(filename, 'utf8').split('\n') if (lines.indexOf(tokenLine) === -1) { fs.appendFileSync(filename, (lines.pop() ? '' : '\n') + tokenLine) } } return { getAddress: function (cb) { if (registryAddress) cb(null, registryAddress) else getAddressCbs.push(cb) } } } exports.respond = function (sbot, config) { var reg = new SsbNpmRegistryServer(sbot, config) return function (req, res) { new Req(reg, req, res).serve() } } function publishMsg(sbot, value, cb) { var gotExpectedPrevious = false sbot.publish(value, function next(err, msg) { if (err && /^expected previous:/.test(err.message)) { // retry once on this error if (gotExpectedPrevious) return cb(err) gotExpectedPrevious = true return sbot.publish(value, next) } cb(err, msg) }) } function publishMentions(sbot, mentions, cb) { // console.error("publishing %s mentions", mentions.length) if (mentions.length === 0) return cb(new Error('Empty mentions list')) publishMsg(sbot, { type: 'npm-packages', mentions: mentions, }, cb) } exports.publishPkgMentions = function (sbot, mentions, cb) { // try to fit the mentions into as few messages as possible, // while fitting under the message size limit. var msgs = [] ;(function next(i, chunks) { if (i >= mentions.length) return cb(null, msgs) var chunkLen = Math.ceil(mentions.length / chunks) publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) { if (err && /must not be large/.test(err.message)) return next(i, chunks + 1) if (err && msgs.length) return onPartialPublish(err) if (err) return cb(err) msgs.push(msg) next(i + chunkLen, chunks) }) })(0, 1) function onPartialPublish(err) { var remaining = mentions.length - i return cb(new Error('Published messages ' + msgs.map(function (msg) { return msg.key }).join(', ') + ' ' + 'but failed to publish remaining ' + remaining + ': ' + (err.stack || err))) } } function SsbNpmRegistryServer(sbot, config) { this.sbot = sbot this.config = config this.links2 = sbot.links2 if (!this.links2) throw new Error('missing ssb-links2 scuttlebot plugin') this.blobsPrefix = 'http://' + (config.host || 'localhost') + ':' + (config.ws && config.ws.port || '8989') + '/blobs/get/' } SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype) SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) { var self = this if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push')) ;(function next(i) { if (i >= ids.length) return cb() self.sbot.blobs.push(ids[i], function (err) { if (err) return cb(err) next(i+1) }) })(0) } SsbNpmRegistryServer.prototype.blobUrl = function (id) { return this.blobsPrefix + id } SsbNpmRegistryServer.prototype.getMentions = function (name) { return this.links2.read({ query: [ {$filter: {rel: ['mentions', name]}}, {$filter: {dest: {$prefix: '&'}}}, {$map: { name: ['rel', 1], size: ['rel', 2], link: 'dest', author: 'source', ts: 'ts' }} ] }) } function Req(server, req, res) { this.server = server this.req = req this.res = res this.blobsToPush = [] } Req.prototype.serve = function () { // console.log(this.req.method, this.req.url) var pathname = this.req.url.replace(/\?.*/, '') if (pathname === '/-/whoami') return this.serveWhoami() if (pathname === '/-/ping') return this.respond(200, true) if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1)) return this.respond(404) } Req.prototype.respond = function (status, message) { this.res.writeHead(status, {'content-type': 'application/json'}) this.res.end(message && JSON.stringify(message)) } Req.prototype.respondError = function (status, message) { this.respond(status, {error: message}) } Req.prototype.serveWhoami = function () { var self = this self.server.sbot.whoami(function (err, feed) { if (err) return self.respondError(err.stack || err) self.respond(200, {username: feed.id}) }) } function decodeName(name) { var parts = name.replace(/\.tgz$/, '').split(':') return { name: parts[1], version: parts[2], distTag: parts[3], } } Req.prototype.servePkg = function (pathname) { var self = this var parts = pathname.split('/') var pkgName = parts.shift() if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported') if (parts.length > 0) return this.respondError(404) if (self.req.method === 'PUT') return self.publishPkg(pkgName) var obj = { _id: pkgName, name: pkgName, 'dist-tags': {}, versions: {} } var oldest, newest var getMention = self.server.getMentions({$prefix: 'npm:' + pkgName + ':'}) getMention(null, function next(err, mention) { if (err === true) return self.respond(200, obj) if (err) return self.respondError(500, err.stack || err) var data = decodeName(mention.name) if (!data.version) return var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(mention.link) if (!m) return if (data.distTag) obj['dist-tags'][data.distTag] = data.version obj.versions[data.version] = { author: { url: mention.author }, dist: { integrity: m[2] + '-' + m[1], tarball: self.server.blobUrl(mention.link) } } getMention(null, next) }) } Req.prototype.publishPkg = function (pkgName) { var chunks = [] var self = this self.req.on('data', function (data) { chunks.push(data) }) self.req.on('end', function () { var data try { data = JSON.parse(Buffer.concat(chunks)) } catch(e) { return self.respondError(400, e.stack) } return self.publishPkg2(pkgName, data || {}) }) } Req.prototype.publishPkg2 = function (name, data) { var self = this if (data.users) console.trace('[npm-registry users property is not supported') var attachments = data._attachments || {} var links = {/* -.tgz: {link: , size: number} */} var waiting = 0 Object.keys(attachments).forEach(function (filename) { waiting++ var tarball = new Buffer(attachments[filename].data, 'base64') var length = attachments[filename].length if (length && length !== tarball.length) return self.respondError(400, 'Length mismatch for attachment \'' + filename + '\'') self.server.sbot.blobs.add(function (err, id) { if (err) return self.respondError(500, 'Adding attachment \'' + filename + '\' as blob failed') self.blobsToPush.push(id) links[filename] = {link: id, size: tarball.length} if (!--waiting) next() })(pullOnce(tarball)) }) function next() { try { self.publishPkg3(name, data, links) } catch(e) { self.respondError(500, e.stack || e) } } } Req.prototype.publishPkg3 = function (name, data, links) { var self = this var versions = data.versions || {} var linksByVersion = {/* : link */} // associate tarball blobs with versions for (var version in versions) { var pkg = versions[version] if (!pkg) return self.respondError(400, 'Bad package object') if (!pkg.dist) return self.respondError(400, 'Missing package dist property') if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property') if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported') var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball) if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'') var filename = m[1] var link = links[filename] if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'') // TODO?: try to find missing tarball mentioned in other messages if (pkg.version && pkg.version !== version) return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version]) linksByVersion[version] = link link.version = version } // associate blobs with dist-tags var tags = data['dist-tags'] || {} for (var tag in tags) { var version = tags[tag] var link = linksByVersion[version] if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.') // TODO?: support setting dist-tag without version, // by looking up a tarball blob for the version link.tag = tag } // compute blob links to publish var mentions = [] for (var filename in links) { var link = links[filename] || {} if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata') mentions.push({ name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''), link: link.link, size: link.size, }) } return self.publishPkgs(mentions) } Req.prototype.publishPkgs = function (mentions) { var self = this exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) { if (err) self.respondError(500, err.stack || err) self.server.pushBlobs(self.blobsToPush, function (err) { if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err)) self.respond(201) console.log(msgs.map(function (msg) { return msg.key }).join('\n')) }) }) }