git ssb

0+

cel / pull-git-remote-helper



Tree: 1c27109b9155b4dd90f5068566dfc45b92a17f82

Files: 1c27109b9155b4dd90f5068566dfc45b92a17f82 / lib / pack.js

10672 bytesRaw
1var buffered = require('pull-buffered')
2var pull = require('pull-stream')
3var toPull = require('stream-to-pull-stream')
4var pako = require('pako')
5var createHash = require('./util').createHash
6var cat = require('pull-cat')
7
8exports.decode = decodePack
9exports.encode = encodePack
10
11var PACK_VERSION = 2
12
13var objectTypes = [
14 'none', 'commit', 'tree', 'blob',
15 'tag', 'unused', 'ofs-delta', 'ref-delta'
16]
17var objectTypeNums = {
18 commit: 1,
19 tree: 2,
20 blob: 3,
21 tag: 4,
22 'ofs-delta': 6,
23 'ref-delta': 7
24}
25
26function error(cb) {
27 return function (err) {
28 cb(err || true)
29 }
30}
31
32function inflateBytes(read) {
33 var inflate = new pako.Inflate()
34 var ended, dataOut
35
36 inflate.onData = function (data) {
37 dataOut = new Buffer(data)
38 // console.error('inflated data', data.length)
39 }
40
41 inflate.onEnd = function (status) {
42 ended = (status === 0) ? true : new Error(inflate.strm.msg)
43 // console.error('inflated end', status, ended)
44 }
45
46 return function (abort, cb) {
47 if (ended) return cb(ended)
48 read(abort, function next(end, data) {
49 if (end === true) {
50 end = null
51 data = []
52 }
53 if (ended = end) return cb(end)
54 if (data.length > 1) return cb(new Error('got more than one byte'))
55 dataOut = null
56 inflate.push(data, end === true)
57 if (dataOut)
58 cb(null, dataOut)
59 else if (ended)
60 cb(ended)
61 else
62 // let the stack unwind
63 setImmediate(function () {
64 read(null, next)
65 })
66 })
67 }
68}
69
70function deflate(read) {
71 var def = new pako.Deflate()
72 var queue = []
73 var ended
74
75 def.onData = function (data) {
76 queue.push([null, new Buffer(data)])
77 }
78
79 def.onEnd = function (status) {
80 queue.push([(status === 0) ? true : new Error(def.strm.msg)])
81 }
82
83 return function readOut(abort, cb) {
84 if (ended)
85 cb(ended)
86 else if (queue.length)
87 cb.apply(this, queue.shift())
88 else
89 read(abort, function next(end, data) {
90 if (end === true) def.push([], true)
91 else if (end) return cb(end)
92 else def.push(data)
93 readOut(null, cb)
94 })
95 }
96}
97
98function decodePack(opts, repo, onEnd, read) {
99 if (read === undefined)
100 return decodePack.bind(this, opts, repo, onEnd)
101
102 var ended
103 var numObjects = -1
104 var checksum = createHash('sha1')
105 var b = buffered(read)
106 // TODO: optimize to pass through buffers to checksum
107 var readByte = checksum(b.chunks(1))
108 var readWord = checksum(b.chunks(4))
109 var readHash = checksum(b.chunks(20))
110 var readChecksum = b.chunks(20)
111 var expectChecksum = true
112
113 function readHeader(cb) {
114 readWord(null, function (end, header) {
115 if (ended = end) return cb(end)
116 if (!header.equals(header, new Buffer('PACK')))
117 read(new Error('Invalid packfile header'), error(cb))
118 else
119 readVersion(cb)
120 })
121 }
122
123 function readVersion(cb) {
124 readWord(null, function (end, word) {
125 if (ended = end) return cb(end)
126 var version = word.readUInt32BE()
127 if (version < 2 || version > 3)
128 read(new Error('Invalid packfile version ' + version), error(cb))
129 else
130 readNumObjects(cb)
131 })
132 }
133
134 function readNumObjects(cb) {
135 readWord(null, function (end, word) {
136 if (ended = end) return cb(end)
137 numObjects = word.readUInt32BE()
138 if (opts.verbosity >= 2)
139 console.error(numObjects + ' objects')
140 if (opts.onHeader)
141 opts.onHeader(numObjects)
142 readObject(null, cb)
143 })
144 }
145
146 function readTypedVarInt(cb) {
147 var type, value, shift
148 // https://codewords.recurse.com/images/three/varint.svg
149 readByte(null, function (end, buf) {
150 if (ended = end) return cb(end)
151 var firstByte = buf[0]
152 type = objectTypes[(firstByte >> 4) & 7]
153 value = firstByte & 15
154 // console.error('byte1', firstByte, firstByte.toString(2))
155 shift = 4
156 checkByte(firstByte)
157 })
158
159 function checkByte(byte) {
160 if (byte & 0x80)
161 readByte(null, gotByte)
162 else
163 cb(null, type, value)
164 }
165
166 function gotByte(end, buf) {
167 if (ended = end) return cb(end)
168 var byte = buf[0]
169 value += (byte & 0x7f) << shift
170 shift += 7
171 // console.error('byte', byte, byte.toString(2))
172 checkByte(byte)
173 }
174 }
175
176 function getObject(cb) {
177 readTypedVarInt(function (end, type, length) {
178 if (opts.verbosity >= 2)
179 console.error('read object header', end, type, length)
180 numObjects--
181 if (end === true && expectChecksum)
182 onEnd(new Error('Missing checksum'))
183 if (ended = end) return cb(end)
184 // TODO: verify that the inflated data is the correct length
185 if (type == 'ref-delta')
186 getObjectFromRefDelta(length, cb)
187 else
188 cb(null, {
189 type: type,
190 length: length,
191 read: inflateBytes(readByte)
192 })
193 })
194 }
195
196 // TODO: test with ref-delta objects in pack
197 function getObjectFromRefDelta(length, cb) {
198 readHash(null, function (end, sourceHash) {
199 if (end) return cb(end)
200 sourceHash = sourceHash.toString('hex')
201 var b = buffered(inflateBytes(readByte))
202 var readInflatedByte = b.chunks(1)
203 readVarInt(readInflatedByte, function (err, expectedSourceLength) {
204 if (err) return cb(err)
205 readVarInt(readInflatedByte, function (err, expectedTargetLength) {
206 if (err) return cb(err)
207 if (opts.verbosity >= 3)
208 console.error('getting object', sourceHash)
209 repo.getObject(sourceHash, function (err, sourceObject) {
210 if (opts.verbosity >= 3)
211 console.error('got object', sourceHash, sourceObject, err)
212 if (err) return cb(err)
213 if (sourceObject.length != expectedSourceLength)
214 cb(new Error('Incorrect source object size in ref delta'))
215 else
216 patchObject(opts, b, length, sourceObject,
217 expectedTargetLength, cb)
218 })
219 })
220 })
221 })
222 }
223
224 function readTrailer(cb) {
225 // read the checksum before it updates to include the trailer
226 var expected = checksum.digest()
227 readChecksum(null, function (end, value) {
228 cb(true)
229 if (end === true && expectChecksum)
230 onEnd(new Error('Missing checksum'))
231 if (!value.equals(expected)) {
232 onEnd(new Error('Checksum mismatch: ' +
233 expected.hexSlice() + ' != ' + value.hexSlice()))
234 } else {
235 if (opts.verbosity >= 3)
236 console.error('checksum ok', expected.hexSlice())
237 onEnd(null)
238 }
239 })
240 }
241
242 function readObject(abort, cb) {
243 if (ended) cb(ended)
244 else if (abort) read(abort, function (err) { cb(ended = err || abort) })
245 else if (numObjects < 0) readHeader(cb)
246 else if (numObjects > 0) getObject(cb)
247 else if (expectChecksum) readTrailer(cb)
248 }
249
250 return readObject
251}
252
253function readVarInt(readByte, cb) {
254 var value = 0, shift = 0
255 readByte(null, function gotByte(end, buf) {
256 if (ended = end) return cb(end)
257 var byte = buf[0]
258 value += (byte & 0x7f) << shift
259 shift += 7
260 if (byte & 0x80)
261 readByte(null, gotByte)
262 else
263 cb(null, value)
264 })
265}
266
267function patchObject(opts, deltaB, deltaLength, srcObject, targetLength, cb) {
268 var readByte = deltaB.chunks(1)
269 var srcBuf
270 var ended
271
272 if (opts.verbosity >= 2)
273 console.error('patching', srcObject.type, targetLength)
274 pull(
275 srcObject.read,
276 pull.collect(function (err, bufs) {
277 srcBuf = Buffer.concat(bufs, srcObject.length)
278 cb(null, {
279 type: srcObject.type,
280 length: targetLength,
281 read: read
282 })
283 })
284 )
285
286 function read(abort, cb) {
287 if (ended) return cb(ended)
288 readByte(null, function (end, dBuf) {
289 if (ended = end) return cb(end)
290 var cmd = dBuf[0]
291 if (cmd & 0x80)
292 // skip a variable amount and then pass through a variable amount
293 readOffsetSize(cmd, deltaB, function (err, offset, size) {
294 if (err) return earlyEnd(err)
295 var buf = srcBuf.slice(offset, offset + size)
296 cb(end, buf)
297 })
298 else if (cmd)
299 // insert `cmd` bytes from delta
300 deltaB.chunks(cmd)(null, cb)
301 else
302 cb(new Error("unexpected delta opcode 0"))
303 })
304
305 function earlyEnd(err) {
306 cb(err === true ? new Error('stream ended early') : err)
307 }
308 }
309}
310
311function readOffsetSize(cmd, b, readCb) {
312 var readByte = b.chunks(1)
313 var offset = 0, size = 0
314
315 function addByte(bit, outPos, cb) {
316 if (cmd & (1 << bit))
317 readByte(null, function (err, buf) {
318 if (err) readCb(err)
319 else cb(buf[0] << (outPos << 3))
320 })
321 else
322 cb(0)
323 }
324
325 addByte(0, 0, function (val) {
326 offset = val
327 addByte(1, 1, function (val) {
328 offset |= val
329 addByte(2, 2, function (val) {
330 offset |= val
331 addByte(3, 3, function (val) {
332 offset |= val
333 addSize()
334 })
335 })
336 })
337 })
338 function addSize() {
339 addByte(4, 0, function (val) {
340 size = val
341 addByte(5, 1, function (val) {
342 size |= val
343 addByte(6, 2, function (val) {
344 size |= val
345 readCb(null, offset, size || 0x10000)
346 })
347 })
348 })
349 }
350}
351
352function encodeTypedVarInt(typeStr, length, cb) {
353 var type = objectTypeNums[typeStr]
354 // console.error('TYPE', type, typeStr, 'len', length, typeof cb)
355 if (!type)
356 return cb(new Error("Bad object type " + typeStr))
357
358 var vals = []
359 var b = (type << 4) | (length & 15)
360 for (length >>= 4; length; length >>= 7) {
361 vals.push(b | 0x80)
362 b = length & 0x7f
363 }
364 vals.push(b)
365 /*
366 console.error('sending var int', vals, vals.map(function (n) {
367 return ('00000000' + Number(n).toString(2)).substr(-8)
368 }))
369 */
370 cb(null, new Buffer(vals))
371}
372
373function encodePack(opts, numObjects, readObject) {
374 if (readObject === undefined)
375 return encodePack.bind(this, numObjects)
376
377 var header = new Buffer(12)
378 header.write('PACK')
379 header.writeUInt32BE(PACK_VERSION, 4)
380 header.writeUInt32BE(numObjects, 8)
381 var checksum = createHash('sha1')
382 var readData
383
384 return cat([
385 checksum(cat([
386 pull.once(header),
387 encodeObject
388 ])),
389 checksum.readDigest
390 ])
391
392 function encodeObject(abort, cb) {
393 if (readData)
394 readData(abort, function (end, data) {
395 if (end === true)
396 readObject(abort, nextObject)
397 else
398 cb(end, data)
399 })
400 else
401 readObject(abort, nextObject)
402
403 function nextObject(end, object) {
404 if (end) return cb(end)
405 readData = deflate(object.read)
406 encodeTypedVarInt(object.type, object.length, cb)
407 }
408 }
409}
410

Built with git-ssb-web