var http = require('http')
var os = require('os')
var path = require('path')
var fs = require('fs')
var crypto = require('crypto')
var pkg = require('./package')
var semver = require('semver')
var toPull = require('stream-to-pull-stream')
var pull = require('pull-stream')
var tar = require('tar-stream')
var zlib = require('zlib')
var cat = require('pull-cat')
var hash = require('pull-hash')
var multicb = require('multicb')
var memo = require('asyncmemo')
var lru = require('hashlru')
function escapeHTML(str) {
return String(str)
.replace(//g, '>')
}
function idToHex(id) {
var b64 = String(id).replace(/^[%#&]|\.[a-z0-9]*$/g, '')
return new Buffer(b64, 'base64').toString('hex')
}
function onceify(fn, self) {
var cbs = [], err, data
return function (cb) {
if (fn) {
cbs.push(cb)
fn.call(self, function (_err, _data) {
err = _err, data = _data
var _cbs = cbs
cbs = null
while (_cbs.length) _cbs.shift()(err, data)
})
fn = null
} else if (cbs) {
cbs.push(cb)
} else {
cb(err, data)
}
}
}
function once(cb) {
var done
return function (err, result) {
if (done) {
if (err) console.trace(err)
} else {
done = true
cb(err, result)
}
}
}
function npmLogin(registryAddress, cb) {
var tokenLine = registryAddress.replace(/^http:/, '') + ':_authToken=1'
var filename = path.join(os.homedir(), '.npmrc')
fs.readFile(filename, 'utf8', function (err, data) {
if (err && err.code === 'ENOENT') data = ''
else if (err) return cb(new Error(err.stack))
var lines = data ? data.split('\n') : []
if (lines.indexOf(tokenLine) > -1) return cb()
var trailingNewline = (lines.length === 0 || lines[lines.length-1] === '')
var line = trailingNewline ? tokenLine + '\n' : '\n' + tokenLine
fs.appendFile(filename, line, cb)
})
}
function formatHost(host) {
return /^[^\[]:.*:.*:/.test(host) ? '[' + host + ']' : host
}
exports.name = 'npm-registry'
exports.version = '1.0.0'
exports.manifest = {
getAddress: 'async'
}
exports.init = function (sbot, config) {
var conf = config.npm || {}
var port = conf.port || 8043
var host = conf.host || null
var autoAuth = conf.autoAuth !== false
var server = http.createServer(exports.respond(sbot, config))
var getAddress = onceify(function (cb) {
server.on('error', cb)
server.listen(port, host, function () {
server.removeListener('error', cb)
var regHost = formatHost(host || 'localhost')
var regPort = this.address().port
var regUrl = 'http://' + regHost + ':' + regPort + '/'
if (autoAuth) npmLogin(regUrl, next)
else next()
function next(err) {
cb(err, regUrl)
}
})
sbot.on('close', function () {
server.close()
})
})
getAddress(function (err, addr) {
if (err) return console.error('[npm-registry]', err.stack || err)
console.log('[npm-registry] Listening on ' + addr)
})
return {
getAddress: getAddress
}
}
exports.respond = function (sbot, config) {
var reg = new SsbNpmRegistryServer(sbot, config)
return function (req, res) {
new Req(reg, req, res).serve()
}
}
function publishMsg(sbot, value, cb) {
var gotExpectedPrevious = false
sbot.publish(value, function next(err, msg) {
if (err && /^expected previous:/.test(err.message)) {
// retry once on this error
if (gotExpectedPrevious) return cb(err)
gotExpectedPrevious = true
return sbot.publish(value, next)
}
cb(err, msg)
})
}
function getDependencyBranches(sbot, link, cb) {
var id = link.link
// get ids of heads of tree of dependencyBranch message links to include all
// the dependencies of the given tarball id.
var getPackageJsonCached = memo(getPackageJsonFromTarballBlob, sbot)
var msgs = {}
var branched = {}
var blobs = {}
function addPkgById(id, cb) {
if (blobs[id]) return cb()
blobs[id] = true
getPackageJsonCached(id, function (err, pkg) {
if (err) return cb(err)
var done = multicb()
for (var name in pkg.dependencies || {}) {
addPkgBySpec(name, pkg.dependencies[name], done())
}
done(cb)
})
}
function addPkgBySpec(name, spec, cb) {
var versions = {}
var distTags = {}
pull(
getMentions(sbot.links2, {$prefix: 'npm:' + name + ':'}),
pull.filter(mentionMatches(null, name, spec)),
pull.map(function (mention) {
// query to get the messages since links2 does not include the msg id
return sbot.links({
source: mention.author,
dest: mention.link,
rel: 'mentions',
values: true,
})
}),
pull.flatten(),
pull.drain(function (msg) {
var c = msg && msg.value && msg.value.content
if (!c || !Array.isArray(c.mentions)) return
c.mentions.forEach(function (link) {
var data = link && link.name && decodeName(link.name)
if (!data || data.name !== name) return
versions[data.version] = {msg: msg, mention: link, mentionData: data}
if (data.distTag) distTags[data.distTag] = data.version
})
}, function (err) {
if (err) return cb(err)
var version = distTags[spec]
|| semver.maxSatisfying(Object.keys(versions), spec)
var item = versions[version]
if (!item) return cb(new Error('Dependency version not found: ' + name + '@' + spec))
addDepBranchMsg(item.msg)
// console.log('add', item.msg.key, item.mentionData.name, item.mentionData.version)
addPkgById(item.mention.link, cb)
})
)
}
function addDepBranchMsg(msg) {
var c = msg && msg.value && msg.value.content
if (!c) return
if (msgs[msg.key]) return
msgs[msg.key] = msg.value
if (Array.isArray(c.dependencyBranch)) {
for (var k = 0; k < c.dependencyBranch.length; k++) {
branched[c.dependencyBranch[k]] = true
}
}
if (Array.isArray(c.versionBranch)) {
for (var k = 0; k < c.versionBranch.length; k++) {
branched[c.versionBranch[k]] = true
}
}
}
function addPkgsByName(name, cb) {
var data = decodeName(name)
pull(
getMentions(sbot.links2, {$prefix: 'npm:' + data.name + ':'}),
pull.map(function (mention) {
return packageLinks(sbot, mention.author, mention.link, data.name)
}),
pull.flatten(),
pull.drain(addDepBranchMsg, cb)
)
}
var done = multicb()
addPkgsByName(link.name, done())
addPkgById(id, done())
done(function (err) {
if (err) return cb(err)
var ids = []
for (var key in msgs) {
if (!branched[key]) ids.push(key)
}
cb(null, ids)
})
}
function packageLinks(sbot, feed, id, name, spec) {
var matches = mentionMatches(id, name, spec)
return pull(
sbot.links({
source: feed,
dest: id,
rel: 'mentions',
values: true,
}),
pull.filter(function (msg) {
var c = msg && msg.value && msg.value.content
return c && Array.isArray(c.mentions) && c.mentions.some(matches)
})
)
}
// For each dependency that is not a bundledDependency, get message ids for
// that dependency name + version.
function publishSingleMention(sbot, mention, cb) {
if (!sbot.links2) return cb(new Error('ssb-links scuttlebot plugin is required to publish ssb-npm packages'))
// Calculate dependencyBranch message ids.
var value = {
type: 'npm-packages',
mentions: [mention]
}
var done = multicb({pluck: 1, spread: true})
getDependencyBranches(sbot, mention, function (err, branches) {
if (err) return cb(err)
value.dependencyBranch = branches || undefined
publishMsg(sbot, value, cb)
})
}
function publishMentions(sbot, mentions, cb) {
// console.error("publishing %s mentions", mentions.length)
if (mentions.length === 0) return cb(new Error('Empty mentions list'))
// if it is just one mention, fetch and add useful metadata
if (mentions.length === 1) return publishSingleMention(sbot, mentions[0], cb)
publishMsg(sbot, {
type: 'npm-packages',
mentions: mentions,
}, cb)
}
exports.publishPkgMentions = function (sbot, mentions, cb) {
// try to fit the mentions into as few messages as possible,
// while fitting under the message size limit.
var msgs = []
;(function next(i, chunks) {
if (i >= mentions.length) return cb(null, msgs)
var chunkLen = Math.ceil(mentions.length / chunks)
publishMentions(sbot, mentions.slice(i, i + chunkLen), function (err, msg) {
if (err && /must not be large/.test(err.message)) return next(i, chunks + 1)
if (err && msgs.length) return onPartialPublish(err)
if (err) return cb(err)
msgs.push(msg)
next(i + chunkLen, chunks)
})
})(0, 1)
function onPartialPublish(err) {
var remaining = mentions.length - i
return cb(new Error('Published messages ' +
msgs.map(function (msg) { return msg.key }).join(', ') + ' ' +
'but failed to publish remaining ' + remaining + ': ' + (err.stack || err)))
}
}
exports.expandPkgMentions = function (sbot, mentions, props, cb) {
cb = once(cb)
var waiting = 0
var expandedMentions = mentions.map(function (link) {
var id = link && link.link
if (!id) return link
waiting++
var newLink = {}
for (var k in link) newLink[k] = link[k]
getPackageJsonFromTarballBlob(sbot, id, function (err, pkg) {
if (err) return cb(err)
for (var k in props) newLink[k] = pkg[k]
if (props.shasum && !pkg.shasum) newLink.shasum = pkg._shasum
if (!--waiting) next()
})
return newLink
})
if (!waiting) next()
function next() {
cb(null, expandedMentions)
}
}
function SsbNpmRegistryServer(sbot, config) {
this.sbot = sbot
this.config = config
this.npmConfig = config.npm || {}
this.host = this.npmConfig.host || 'localhost'
this.fetchAll = this.npmConfig.fetchAll
this.needShasum = this.npmConfig.needShasum
var wsPort = config.ws && Number(config.ws.port) || '8989'
this.wsLink = this.npmConfig.wsLink
|| ('http://' + (config.host || 'localhost') + ':' + wsPort)
this.getMsg = memo({cache: lru(100)}, this.getMsg)
}
SsbNpmRegistryServer.prototype = Object.create(http.Server.prototype)
SsbNpmRegistryServer.prototype.constructor = SsbNpmRegistryServer
SsbNpmRegistryServer.prototype.pushBlobs = function (ids, cb) {
var self = this
if (!self.sbot.blobs.push) return cb(new Error('missing blobs.push'))
;(function next(i) {
if (i >= ids.length) return cb()
self.sbot.blobs.push(ids[i], function (err) {
if (err) return cb(err)
next(i+1)
})
})(0)
}
function getBlob(sbot, id, cb) {
var blobs = sbot.blobs
blobs.size(id, function (err, size) {
if (err) return cb(err)
if (typeof size === 'number') cb(null, blobs.get(id), size)
else blobs.want(id, function (err, got) {
if (err) cb(err)
else if (!got) cb('missing blob ' + id)
else blobs.size(id, function (err, size) {
if (err) return cb(err)
cb(null, blobs.get(id), size)
})
})
})
}
SsbNpmRegistryServer.prototype.blobDist = function (id) {
var m = /^&([^.]+)\.([a-z0-9]+)$/.exec(id)
if (!m) throw new Error('bad blob id: ' + id)
return {
integrity: m[2] + '-' + m[1],
tarball: this.wsLink + '/blobs/get/' + id
}
}
function getMentions(links2, name) {
return links2.read({
query: [
{$filter: {rel: ['mentions', name, {$gt: true}]}},
{$filter: {dest: {$prefix: '&'}}},
{$map: {
name: ['rel', 1],
size: ['rel', 2],
link: 'dest',
author: 'source',
ts: 'ts'
}}
]
})
}
function mentionMatches(id, name, spec) {
return function (mention) {
var data = mention
&& (!id || id === mention.link)
&& mention.name
&& decodeName(mention.name)
return data
&& data.name === name
&& (spec ? semver.satisfies(data.version, spec) : true)
}
}
SsbNpmRegistryServer.prototype.getMentions = function (name) {
if (!this.sbot.links2) return pull.empty()
return getMentions(this.sbot.links2, name)
}
SsbNpmRegistryServer.prototype.getMsg = function (id, cb) {
if (this.sbot.ooo) return this.sbot.ooo.get(id, cb)
else this.sbot.get(id, function (err, value) {
if (err) return cb(err)
cb(null, {key: id, value: value})
})
}
SsbNpmRegistryServer.prototype.streamTree = function (heads, prop) {
var self = this
var stack = heads.slice()
var seen = {}
return function (abort, cb) {
if (abort) return cb(abort)
for (var id; stack.length && seen[id = stack.pop()];);
if (!id) return cb(true)
seen[id] = true
// TODO: use DFS
self.getMsg(id, function (err, msg) {
if (err) return cb(new Error(err.stack || err))
var c = msg && msg.value && msg.value.content
var links = c && c[prop]
if (Array.isArray(links)) {
stack.push.apply(stack, links)
} else if (links) {
stack.push(links)
}
cb(null, msg)
})
}
}
function Req(server, req, res) {
this.server = server
this.req = req
this.res = res
this.blobsToPush = []
this.fetchAll = server.fetchAll != null ? server.fetchAll : true
var ua = this.req.headers['user-agent']
var m = /\bnpm\/([0-9]*)/.exec(ua)
var npmVersion = m && m[1]
this.needShasum = server.needShasum != null ? server.needShasum :
(npmVersion && npmVersion < 5)
this.wsLink = this.server.wsLink
}
Req.prototype.serve = function () {
console.log(this.req.method, this.req.url, this.req.socket.remoteAddress.replace(/^::ffff:/, ''))
this.res.setTimeout(0)
var pathname = this.req.url.replace(/\?.*/, '')
var m
if ((m = /^\/(\^|%5[Ee])?(%25.*sha256)+(\/.*)$/.exec(pathname))) {
try {
this.headMsgIds = decodeURIComponent(m[2]).split(',')
this.headMsgPlus = !!m[1]
// ^ means also include packages published after the head message id
} catch(e) {
return this.respondError(400, e.stack || e)
}
pathname = m[3]
}
if (pathname === '/') return this.serveHome()
if (pathname === '/robots.txt') return this.serveRobots()
if (pathname === '/-/bootstrap') return this.serveBootstrap()
if (pathname === '/-/whoami') return this.serveWhoami()
if (pathname === '/-/ping') return this.respond(200, true)
if (pathname === '/-/user/org.couchdb.user:1') return this.serveUser1()
if (pathname.startsWith('/-/blobs/get/')) return this.serveBlob(pathname.substr(13))
if (pathname.startsWith('/-/msg/')) return this.serveMsg(pathname.substr(7))
if ((m = /^\/-\/prebuild\/(.*)$/.exec(pathname))) return this.servePrebuild(m[1])
if (!/^\/-\//.test(pathname)) return this.servePkg(pathname.substr(1))
return this.respond(404, {error: 'Not found'})
}
Req.prototype.respond = function (status, message) {
this.res.writeHead(status, {'content-type': 'application/json'})
this.res.end(message && JSON.stringify(message, 0, 2))
}
Req.prototype.respondError = function (status, message) {
this.respond(status, {error: message})
}
Req.prototype.respondErrorStr = function (status, err) {
this.res.writeHead(status, {'content-type': 'text/plain'})
this.res.end(err.stack || err)
}
Req.prototype.respondRaw = function (status, body) {
this.res.writeHead(status)
this.res.end(body)
}
Req.prototype.serveHome = function () {
var self = this
self.res.writeHead(200, {'content-type': 'text/html'})
var port = 8044
self.res.end('
' +
'' + escapeHTML(pkg.name) + '' +
'' + escapeHTML(pkg.name) + '
\n' +
'Bootstrap
\n' +
'')
}
Req.prototype.getMsgIdForBlobMention = function (blobId, feedId, cb) {
var self = this
pull(
self.server.sbot.links({
source: feedId,
dest: blobId,
rel: 'mentions',
values: true,
}),
pull.filter(function (msg) {
var c = msg && msg.value && msg.value.content
return c.type === 'npm-packages'
}),
pull.collect(function (err, msgs) {
if (err) return cb(err)
if (msgs.length === 0) return cb(new Error('Unable to find message id for mention ' + blobId + ' ' + feedId))
if (msgs.length > 1) console.warn('Warning: multiple messages mentioning blob id ' + blobId + ' ' + feedId)
// TODO: make a smarter decision about which message id to use
cb(null, msgs.pop().key)
})
)
}
Req.prototype.resolvePkg = function (pkgSpec, cb) {
var m = /(.[^@]*)(?:@(.*))?/.exec(pkgSpec)
if (!m) return cb(new Error('unable to parse spec: \'' + pkgSpec + '\''))
var self = this
var pkgName = m[1]
var spec = m[2] || '*'
var versions = {}
var distTags = {}
pull(
self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
pull.drain(function (mention) {
var data = decodeName(mention.name)
if (!data.version) return
if (data.distTag) {
distTags[data.distTag] = data.version
}
versions[data.version] = {
author: mention.author,
name: pkgName,
version: data.version,
blobId: mention.link
}
}, function (err) {
if (err) return cb(err)
var version = distTags[spec]
|| semver.maxSatisfying(Object.keys(versions), spec)
var item = versions[version]
if (!item) return cb(new Error('Version not found: ' + pkgName + '@' + spec))
self.getMsgIdForBlobMention(item.blobId, item.author, function (err, id) {
if (err) return cb(err)
item.msgId = id
cb(null, item)
})
})
)
}
Req.prototype.resolvePkgs = function (specs, cb) {
var done = multicb({pluck: 1})
var self = this
specs.forEach(function (spec) {
self.resolvePkg(spec, done())
})
done(cb)
}
Req.prototype.serveBootstrap = function () {
var self = this
var pkgs = self.server.npmConfig.defaultPkgs || ['scuttlebot', 'ssb-npm', 'git-ssb']
var ssbNpmRegistryName = require('./package.json').name
var ssbNpmRegistryVersion = require('./package.json').version
var ssbNpmRegistryNameVersion = ssbNpmRegistryName + '-' + ssbNpmRegistryVersion
var ssbNpmRegistrySpec = ssbNpmRegistryName + '@^' + ssbNpmRegistryVersion
var done = multicb({pluck: 1, spread: true})
self.resolvePkg(ssbNpmRegistrySpec, done())
self.resolvePkgs(pkgs, done())
done(function (err, ssbNpmRegistryPkgInfo, pkgsInfo) {
if (err) return self.respondErrorStr(500, err.stack || err)
if (!ssbNpmRegistryPkgInfo) return self.respondErrorStr(500, 'Missing ssb-npm-registry package')
var ssbNpmRegistryBlobId = ssbNpmRegistryPkgInfo.blobId
var ssbNpmRegistryBlobHex = idToHex(ssbNpmRegistryBlobId)
var pkgMsgs = pkgsInfo.map(function (info) { return info.msgId })
var globalPkgs = pkgsInfo.map(function (info) {
return info.name + '@' + info.version
}).join(' ')
var npmCmd = 'install -g ' + globalPkgs
var tmpDir = '/tmp/' + encodeURIComponent(ssbNpmRegistryNameVersion)
var tarballLink = self.wsLink + '/blobs/get/' + ssbNpmRegistryBlobId
var script =
'mkdir -p ' + tmpDir + ' && cd ' + tmpDir + ' &&\n' +
'wget -q \'' + tarballLink + '\' -O package.tgz &&\n' +
'echo \'' + ssbNpmRegistryBlobHex + ' package.tgz\' | sha256sum -c &&\n' +
'tar xzf package.tgz &&\n' +
'./*/bootstrap/bin.js --ws-url ' + self.wsLink + ' \\\n' +
pkgMsgs.map(function (id) {
return ' --branch ' + id + ' \\\n'
}).join('') +
' -- ' + npmCmd + ' &&\n' +
'sbot server'
self.res.writeHead(200, {'Content-type': 'text/plain'})
self.res.end(script)
})
}
Req.prototype.serveRobots = function () {
this.res.writeHead(200, {'Content-type': 'text/plain'})
this.res.end('User-agent: *\nDisallow: /\n')
}
Req.prototype.serveWhoami = function () {
var self = this
self.server.sbot.whoami(function (err, feed) {
if (err) return self.respondError(500, err.stack || err)
self.respond(200, {username: feed.id})
})
}
Req.prototype.serveUser1 = function () {
this.respond(this.req.method === 'PUT' ? 201 : 200, {token: '1'})
}
Req.prototype.serveBlob = function (id) {
var self = this
if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
getBlob(self.server.sbot, id, function (err, readBlob, size) {
if (err) {
if (/^invalid/.test(err.message)) return self.respondErrorStr(400, err.message)
else return self.respondErrorStr(500, err.message || err)
}
self.res.writeHead(200, {
'Cache-Control': 'public, max-age=315360000',
'Content-Length': size,
'etag': id
})
pull(
readBlob,
toPull(self.res, function (err) {
if (err) console.error('[npm-registry]', err)
})
)
})
}
Req.prototype.serveMsg = function (id) {
var self = this
try { id = decodeURIComponent(id) }
catch (e) {}
if (self.req.headers['if-none-match'] === id) return self.respondRaw(304)
self.server.sbot.get(id, function (err, value) {
if (err) return self.respondError(500, err.message || err)
var out = new Buffer(JSON.stringify({key: id, value: value}, null, 2), 'utf8')
self.res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=315360000',
'Content-Length': out.length,
'etag': id
})
self.res.end(out)
})
}
function decodeName(name) {
var parts = String(name).replace(/\.tgz$/, '').split(':')
return {
name: parts[1],
version: parts[2],
distTag: parts[3],
}
}
Req.prototype.getMsgMentions = function (name) {
return pull(
this.server.streamTree(this.headMsgIds, 'dependencyBranch'),
// decryption could be done here
pull.map(function (msg) {
var c = msg.value && msg.value.content
if (!c.mentions || !Array.isArray(c.mentions)) return []
return c.mentions.map(function (mention) {
return {
name: mention.name,
size: mention.size,
link: mention.link,
author: msg.value.author,
ts: msg.value.timestamp,
}
})
}),
pull.flatten(),
pull.filter(typeof name === 'string' ? function (link) {
return link.name === name
} : name && name.$prefix ? function (link) {
return link.name.substr(0, name.$prefix.length) === name.$prefix
} : function () {
throw new TypeError('unsupported name filter')
})
)
}
Req.prototype.getMentions = function (name) {
var useMsgMentions = this.headMsgIds
var useServerMentions = !this.headMsgIds || this.headMsgPlus
if (useServerMentions && !this.server.sbot.links2) {
return this.headMsgPlus
? pull.error(new Error('ssb-links scuttlebot plugin is needed for ^msgid queries'))
: pull.error(new Error('ssb-links scuttlebot plugin is needed for non-msgid queries'))
}
return cat([
useMsgMentions ? this.getMsgMentions(name) : pull.empty(),
useServerMentions ? this.server.getMentions(name) : pull.empty()
])
}
Req.prototype.getMentionLinks = function (blobId) {
return pull(
this.headMsgIds
? this.server.streamTree(this.headMsgIds, 'dependencyBranch')
: this.server.sbot.links({
dest: blobId,
rel: 'mentions',
values: true,
}),
// decryption could be done here
pull.map(function (msg) {
var c = msg.value && msg.value.content
return c && Array.isArray(c.mentions) && c.mentions || []
}),
pull.flatten(),
pull.filter(function (link) {
return link && link.link === blobId
})
)
}
Req.prototype.servePkg = function (pathname) {
var self = this
var parts = pathname.split('/')
var pkgName = parts.shift().replace(/%2f/i, '/')
if (parts[0] === '-rev') return this.respondError(501, 'Unpublish is not supported')
var spec = parts.shift()
if (spec) try { spec = decodeURIComponent(spec) } finally {}
if (parts.length > 0) return this.respondError(404)
if (self.req.method === 'PUT') return self.publishPkg(pkgName)
var obj = {
_id: pkgName,
name: pkgName,
'dist-tags': {},
versions: {}
}
var distTags = {/* : {version, ts}*/}
pull(
self.getMentions({$prefix: 'npm:' + pkgName + ':'}),
pull.drain(function (mention) {
var data = decodeName(mention.name)
if (!data.version) return
if (data.distTag) {
var tag = distTags[data.distTag]
if (!tag || mention.ts > tag.ts) {
/* TODO: sort by causal order instead of only by timestamps */
distTags[data.distTag] = {ts: mention.ts, version: data.version}
}
}
obj.versions[data.version] = {
author: {
url: mention.author
},
name: pkgName,
version: data.version,
dist: self.server.blobDist(mention.link)
}
}, function (err) {
if (err) return self.respondError(500, err.stack || err)
for (var tag in distTags) {
obj['dist-tags'][tag] = distTags[tag].version
}
if (spec) resolveSpec()
else if (self.fetchAll) resolveAll()
else resolved()
})
)
function resolveSpec() {
var version = obj['dist-tags'][spec]
|| semver.maxSatisfying(Object.keys(obj.versions), spec)
obj = obj.versions[version]
if (!obj) return self.respondError(404, 'version not found: ' + spec)
self.populatePackageJson(obj, function (err, pkg) {
if (err) return resolved(err)
obj = pkg || obj
resolved()
})
}
function resolveVersion(version, cb) {
self.populatePackageJson(obj.versions[version], function (err, pkg) {
if (err) return cb(err)
if (pkg) obj.versions[version] = pkg
cb()
})
}
function resolveAll() {
var done = multicb()
for (var version in obj.versions) {
resolveVersion(version, done())
}
done(resolved)
}
function resolved(err) {
if (err) return self.respondError(500, err.stack || err)
self.respond(200, obj)
}
}
Req.prototype.servePrebuild = function (name) {
var self = this
var getMention = self.getMentions('prebuild:' + name)
var blobsByAuthor = {/* : BlobId */}
pull(getMention, pull.drain(function (link) {
blobsByAuthor[link.author] = link.link
}, function (err) {
if (err) return self.respondError(500, err.stack || err)
var authorsByLink = {/* : [FeedId...] */}
var blobId
for (var feed in blobsByAuthor) {
var blob = blobId = blobsByAuthor[feed]
var feeds = authorsByLink[blob] || (authorsByLink[blob] = [])
feeds.push(feed)
}
switch (Object.keys(authorsByLink).length) {
case 0:
return self.respondError(404, 'Not Found')
case 1:
self.res.writeHead(303, {Location: self.wsLink + '/blobs/get/' + blobId})
return self.res.end()
default:
return self.respond(300, {choices: authorsByLink})
}
}))
}
var localhosts = {
'::1': true,
'127.0.0.1': true,
'::ffff:127.0.0.1': true,
}
Req.prototype.publishPkg = function (pkgName) {
var self = this
var remoteAddress = self.req.socket.remoteAddress
if (!(remoteAddress in localhosts)) {
return self.respondError(403, 'You may not publish as this user.')
}
var chunks = []
self.req.on('data', function (data) {
chunks.push(data)
})
self.req.on('end', function () {
var data
try {
data = JSON.parse(Buffer.concat(chunks))
} catch(e) {
return self.respondError(400, e.stack)
}
return self.publishPkg2(pkgName, data || {})
})
}
Req.prototype.publishPkg2 = function (name, data) {
var self = this
if (data.users) console.trace('[npm-registry] users property is not supported')
var attachments = data._attachments || {}
var links = {/* -.tgz: {link: , size: number} */}
var done = multicb()
function addAttachmentAsBlob(filename, cb) {
var data = attachments[filename].data
var tarball = new Buffer(data, 'base64')
var length = attachments[filename].length
if (length && length !== tarball.length) return self.respondError(400,
'Length mismatch for attachment \'' + filename + '\'')
self.server.sbot.blobs.add(function (err, id) {
if (err) return cb(err)
self.blobsToPush.push(id)
var shasum = crypto.createHash('sha1').update(tarball).digest('hex')
links[filename] = {link: id, size: tarball.length, shasum: shasum}
cb()
})(pull.once(tarball))
}
for (var filename in attachments) {
addAttachmentAsBlob(filename, done())
}
done(function (err) {
if (err) return self.respondError(500, err.stack || err)
try {
self.publishPkg3(name, data, links)
} catch(e) {
self.respondError(500, e.stack || e)
}
})
}
Req.prototype.publishPkg3 = function (name, data, links) {
var self = this
var versions = data.versions || {}
var linksByVersion = {/* : link */}
// associate tarball blobs with versions
for (var version in versions) {
var pkg = versions[version]
if (!pkg) return self.respondError(400, 'Bad package object')
if (!pkg.dist) return self.respondError(400, 'Missing package dist property')
if (!pkg.dist.tarball) return self.respondError(400, 'Missing dist.tarball property')
if (pkg.deprecated) return self.respondError(501, 'Deprecation is not supported')
var m = /\/-\/([^\/]+)$/.exec(pkg.dist.tarball)
if (!m) return self.respondError(400, 'Bad tarball URL \'' + pkg.dist.tarball + '\'')
var filename = m[1]
var link = links[filename]
if (!link) return self.respondError(501, 'Unable to find attachment \'' + filename + '\'')
// TODO?: try to find missing tarball mentioned in other messages
if (pkg.version && pkg.version !== version)
return self.respondError(400, 'Mismatched package version: ' + [pkg.version, version])
linksByVersion[version] = link
link.version = version
link.dependencies = pkg.dependencies || {}
link.bundledDependencies = pkg.bundledDependencies || pkg.bundleDependencies
}
// associate blobs with dist-tags
var tags = data['dist-tags'] || {}
for (var tag in tags) {
var version = tags[tag]
var link = linksByVersion[version]
if (!link) return self.respondError(501, 'Setting a dist-tag for a version not being published is not supported.')
// TODO?: support setting dist-tag without version,
// by looking up a tarball blob for the version
link.tag = tag
}
// compute blob links to publish
var mentions = []
for (var filename in links) {
var link = links[filename] || {}
if (!link.version) return self.respondError(400, 'Attachment ' + filename + ' was not linked to in the package metadata')
mentions.push({
name: 'npm:' + name + ':' + link.version + (link.tag ? ':' + link.tag : ''),
link: link.link,
size: link.size,
shasum: link.shasum,
dependencies: link.dependencies,
bundledDependencies: link.bundledDependencies,
})
}
return self.publishPkgs(mentions)
}
Req.prototype.publishPkgs = function (mentions) {
var self = this
exports.publishPkgMentions(self.server.sbot, mentions, function (err, msgs) {
if (err) return self.respondError(500, err.stack || err)
self.server.pushBlobs(self.blobsToPush, function (err) {
if (err) console.error('[npm-registry] Failed to push blob ' + id + ': ' + (err.stack || err))
self.respond(201)
console.log(msgs.map(function (msg) { return msg.key }).join('\n'))
})
})
}
Req.prototype.populatePackageJson = function (obj, cb) {
var self = this
var blobId = obj.dist.tarball.replace(/.*\/blobs\/get\//, '')
var deps, bundledDeps, shasum
// look for dependencies in links.
// then fallback to getting it from the tarball blob
pull(
self.getMentionLinks(blobId),
pull.drain(function (link) {
if (link.dependencies) deps = link.dependencies
if (link.shasum) shasum = link.shasum
bundledDeps = link.bundledDependencies || link.bundleDependencies || bundledDeps
// how to handle multiple assignments of dependencies to a package?
}, function (err) {
if (err) return cb(new Error(err.stack || err))
if (deps && (shasum || !self.needShasum)) {
// assume that the dependencies in the links to the blob are
// correct.
obj.dependencies = deps
obj.bundledDependencies = bundledDeps
if (shasum) obj.dist.shasum = obj._shasum = shasum
cb(null, obj)
} else {
// get dependencies from the tarball
getPackageJsonFromTarballBlob(self.server.sbot, blobId, function (err, pkg) {
if (err) return cb(err)
pkg.dist = obj.dist
pkg.dist.shasum = pkg._shasum
pkg.author = pkg.author || obj.author
pkg.version = pkg.version || obj.version
pkg.name = pkg.name || obj.name
cb(null, pkg)
})
}
})
)
}
function getPackageJsonFromTarballBlob(sbot, id, cb) {
var self = this
getBlob(sbot, id, function (err, readBlob) {
if (err) return cb(err)
cb = once(cb)
var extract = tar.extract()
var pkg, shasum
extract.on('entry', function (header, stream, next) {
if (/^[^\/]*\/package\.json$/.test(header.name)) {
pull(toPull.source(stream), pull.collect(function (err, bufs) {
if (err) return cb(err)
try { pkg = JSON.parse(Buffer.concat(bufs)) }
catch(e) { return cb(e) }
next()
}))
} else {
stream.on('end', next)
stream.resume()
}
})
extract.on('finish', function () {
pkg._shasum = shasum
cb(null, pkg)
})
pull(
readBlob,
hash('sha1', 'hex', function (err, sum) {
if (err) return cb(err)
shasum = sum
}),
toPull(zlib.createGunzip()),
toPull(extract)
)
})
}