var pull = require('pull-stream') var cat = require('pull-cat') var loop = require('looper') var multicb = require('multicb') var crypto = require('crypto') var skipFooter = require('pull-skip-footer') var packidx = require('pull-git-packidx-parser') var blockFilter = require('pull-block-filter') function packHeader(numObjects) { var header = new Buffer(12) header.write('PACK') header.writeUInt32BE(2, 4) header.writeUInt32BE(numObjects, 8) return header } function forEachAsync(arr, fn, cb) { var i = 0 loop(function (next) { if (i >= arr.length) return cb && cb() fn(arr[i++], function (err) { if (err) return cb && cb(err) next() }) }) } function compareByOffset(a, b) { return a.offset - b.offset } function dedupPacks(packs, cb) { var seen = {} var numObjects = 0 forEachAsync(packs, function (pack, cb) { return pull(pack.readIdx, packidx(function (err, idx) { if (err) return cb(err) var blocks = [] var lastBlock offset = 0 var objs = idx.objects.sort(compareByOffset) for (var i = 0; i < objs.length; i++) { var obj = objs[i] var id = obj.oid.toString('hex') if (seen[id]) continue seen[id] = true numObjects++ if (obj.offset > offset) { blocks.push(lastBlock = {skip: obj.offset - offset, length: 0}) offset = obj.offset } else if (obj.offset < offset) { return cb(new Error('bad offset')) } var len = obj.next ? obj.next.offset - obj.offset : Infinity lastBlock.length += len offset += len } pack.read = pull( pack.read, skipFooter(20), blockFilter(pull.values(blocks)) ) cb() })) }, function (err) { cb(err, numObjects) }) } function closePacks(packs, cb) { forEachAsync(packs, function (pack, cb) { pack.read(true, cb) }, cb) } module.exports = function concatPacks(packs) { /* packs: [{read: source, readIdx: source}] */ if (packs.length === 1) return packs[0].read var checksum = crypto.createHash('sha1') var packI = 0 var state = 'begin' return function next(end, cb) { switch (state) { case 'begin': if (end) return closePacks(cb) return dedupPacks(packs, function (err, numObjects) { if (err) return cb(err) var header = packHeader(numObjects) checksum.update(header) state = 'payload' cb(null, header) }) case 'payload': if (end) return closePacks(cb) if (packI >= packs.length) { state = 'end' return cb(null, checksum.digest()) } var pack = packs[packI] return pack.read(null, function (err, data) { if (err === true) { packI++ return next(null, cb) } if (err) return cb(err) checksum.update(data) cb(null, data) }) case 'end': return cb(true) } } }