git ssb

1
var buffered = require('pull-buffered')
2
var pull = require('pull-stream')
3
var toPull = require('stream-to-pull-stream')
4
var pako = require('pako')
5
var createHash = require('./util').createHash
6
var cat = require('pull-cat')
7
8
exports.decode = decodePack
9
exports.encode = encodePack
10
11
var PACK_VERSION = 2
12
13
var objectTypes = [
14
  'none', 'commit', 'tree', 'blob',
15
  'tag', 'unused', 'ofs-delta', 'ref-delta'
16
]
17
var objectTypeNums = {
18
  commit: 1,
19
  tree: 2,
20
  blob: 3,
21
  tag: 4
22
}
23
24
function error(cb) {
25
  return function (err) {
26
    cb(err || true)
27
  }
28
}
29
30
function inflateBytes(read) {
31
  var inflate = new pako.Inflate()
32
  var ended, dataOut
33
34
  inflate.onData = function (data) {
35
    dataOut = new Buffer(data)
36
    // console.error('inflated data', data.length)
37
  }
38
39
  inflate.onEnd = function (status) {
40
    ended = (status === 0) ? true : new Error(inflate.strm.msg)
41
    // console.error('inflated end', status, ended)
42
  }
43
44
  return function (abort, cb) {
45
    if (ended) return cb(ended)
46
    read(abort, function next(end, data) {
47
      if (end === true) {
48
        end = null
49
        data = []
50
      }
51
      if (ended = end) return cb(end)
52
      if (data.length > 1) return cb(new Error('got more than one byte'))
53
      dataOut = null
54
      inflate.push(data, end === true)
55
      if (dataOut)
56
        cb(null, dataOut)
57
      else if (ended)
58
        cb(ended)
59
      else
60
        read(null, next)
61
    })
62
  }
63
}
64
65
function deflate(read) {
66
  var def = new pako.Deflate()
67
  var queue = []
68
  var ended
69
70
  def.onData = function (data) {
71
    queue.push([null, new Buffer(data)])
72
  }
73
74
  def.onEnd = function (status) {
75
    queue.push([(status === 0) ? true : new Error(def.strm.msg)])
76
  }
77
78
  return function readOut(abort, cb) {
79
    if (ended)
80
      cb(ended)
81
    else if (queue.length)
82
      cb.apply(this, queue.shift())
83
    else
84
      read(abort, function next(end, data) {
85
        if (end === true) def.push([], true)
86
        else if (end) return cb(end)
87
        else def.push(data)
88
        readOut(null, cb)
89
      })
90
  }
91
}
92
93
function decodePack(onEnd, read) {
94
  if (read === undefined)
95
    return decodePack.bind(this, onEnd)
96
97
  var ended
98
  var numObjects = -1
99
  var checksum = createHash('sha1')
100
  var b = buffered(read)
101
  // TODO: optimize to pass through buffers to checksum
102
  var readByte = checksum(b.chunks(1))
103
  var readWord = checksum(b.chunks(4))
104
  var readChecksum = b.chunks(20)
105
  var expectChecksum = true
106
  var opts = {
107
    verbosity: 0
108
  }
109
110
  function readHeader(cb) {
111
    readWord(null, function (end, header) {
112
      if (ended = end) return cb(end)
113
      if (!header.equals(header, new Buffer('PACK')))
114
        read(new Error('Invalid packfile header'), error(cb))
115
      else
116
        readVersion(cb)
117
    })
118
  }
119
120
  function readVersion(cb) {
121
    readWord(null, function (end, word) {
122
      if (ended = end) return cb(end)
123
      var version = word.readUInt32BE()
124
      if (version < 2 || version > 3)
125
        read(new Error('Invalid packfile version ' + version), error(cb))
126
      else
127
        readNumObjects(cb)
128
    })
129
  }
130
131
  function readNumObjects(cb) {
132
    readWord(null, function (end, word) {
133
      if (ended = end) return cb(end)
134
      numObjects = word.readUInt32BE()
135
      if (opts.verbosity >= 1)
136
        console.error(numObjects + ' objects')
137
      readObject(null, cb)
138
    })
139
  }
140
141
  function readVarInt(cb) {
142
    var type, value, shift
143
    // https://codewords.recurse.com/images/three/varint.svg
144
    readByte(null, function (end, buf) {
145
      if (ended = end) return cb(end)
146
      var firstByte = buf[0]
147
      type = objectTypes[(firstByte >> 4) & 7]
148
      value = firstByte & 15
149
      // console.error('byte1', firstByte, firstByte.toString(2))
150
      shift = 4
151
      checkByte(firstByte)
152
    })
153
154
    function checkByte(byte) {
155
      if (byte & 0x80)
156
        readByte(null, gotByte)
157
      else
158
        cb(null, type, value)
159
    }
160
161
    function gotByte(end, buf) {
162
      if (ended = end) return cb(end)
163
      var byte = buf[0]
164
      value += (byte & 0x7f) << shift
165
      shift += 7
166
      // console.error('byte', byte, byte.toString(2))
167
      checkByte(byte)
168
    }
169
  }
170
171
  function getObject(cb) {
172
    readVarInt(function (end, type, length) {
173
      if (opts.verbosity >= 2)
174
        console.error('read object header', end, type, length)
175
      numObjects--
176
      if (end === true && expectChecksum)
177
        onEnd(new Error('Missing checksum'))
178
      if (ended = end) return cb(end)
179
      // TODO: verify that the inflated data is the correct length
180
      cb(null, {
181
        type: type,
182
        length: length,
183
        read: inflateBytes(readByte)
184
      })
185
    })
186
  }
187
188
  function readTrailer(cb) {
189
    // read the checksum before it updates to include the trailer
190
    var expected = checksum.digest()
191
    readChecksum(null, function (end, value) {
192
      cb(true)
193
      if (end === true && expectChecksum)
194
        onEnd(new Error('Missing checksum'))
195
      if (!value.equals(expected)) {
196
        onEnd(new Error('Checksum mismatch: ' +
197
          expected.hexSlice() + ' != ' + value.hexSlice()))
198
      } else {
199
        if (opts.verbosity >= 2)
200
          console.error('checksum ok', expected.hexSlice())
201
        onEnd(null)
202
      }
203
    })
204
  }
205
206
  function readObject(abort, cb) {
207
    if (ended) cb(ended)
208
    else if (abort) read(abort, function (err) { cb(ended = err || abort) })
209
    else if (numObjects < 0) readHeader(cb)
210
    else if (numObjects > 0) getObject(cb)
211
    else if (expectChecksum) readTrailer(cb)
212
  }
213
214
  return readObject
215
}
216
217
function once(read) {
218
  var done
219
  return function (abort, cb) {
220
    if (done) cb(done)
221
    else done = true, read(abort, cb)
222
  }
223
}
224
225
function encodeVarInt(typeStr, length, cb) {
226
  var type = objectTypeNums[typeStr]
227
  // console.error('TYPE', type, typeStr, 'len', length, typeof cb)
228
  if (!type)
229
    return cb(new Error("Bad object type " + typeStr))
230
231
  var vals = []
232
  var b = (type << 4) | (length & 15)
233
  for (length >>= 4; length; length >>= 7) {
234
    vals.push(b | 0x80)
235
    b = length & 0x7f
236
  }
237
  vals.push(b)
238
  /*
239
  console.error('sending var int', vals, vals.map(function (n) {
240
    return ('00000000' + Number(n).toString(2)).substr(-8)
241
  }))
242
  */
243
  cb(null, new Buffer(vals))
244
}
245
246
function encodePack(numObjects, readObject) {
247
  if (readObject === undefined)
248
    return encodePack.bind(this, numObjects)
249
250
  var header = new Buffer(12)
251
  header.write('PACK')
252
  header.writeUInt32BE(PACK_VERSION, 4)
253
  header.writeUInt32BE(numObjects, 8)
254
  var checksum = createHash('sha1')
255
  var readData
256
257
  return cat([
258
    checksum(cat([
259
      pull.once(header),
260
      encodeObject
261
    ])),
262
    checksum.readDigest
263
  ])
264
265
  function encodeObject(abort, cb) {
266
    if (readData)
267
      readData(abort, function (end, data) {
268
        if (end === true)
269
          readObject(abort, nextObject)
270
        else
271
          cb(end, data)
272
      })
273
    else
274
      readObject(abort, nextObject)
275
276
    function nextObject(end, object) {
277
      if (end) return cb(end)
278
      readData = deflate(object.read)
279
      encodeVarInt(object.type, object.length, cb)
280
    }
281
  }
282
}
283