git ssb

1
var pull = require('pull-stream')
2
var paramap = require('pull-paramap')
3
var lru = require('hashlru')
4
var memo = require('asyncmemo')
5
var u = require('./util')
6
var packidx = require('pull-git-packidx-parser')
7
var Reader = require('pull-reader')
8
var toPull = require('stream-to-pull-stream')
9
var zlib = require('zlib')
10
11
var ObjectNotFoundError = u.customError('ObjectNotFoundError')
12
13
var types = {
14
  blob: true,
15
  commit: true,
16
  tree: true,
17
}
18
19
module.exports = Git
20
21
function Git(app) {
22
  this.app = app
23
24
  this.findObject = memo({
25
    cache: lru(5),
26
    asString: function (opts) {
27
      return opts.obj + opts.headMsgId
28
    }
29
  }, this._findObject.bind(this))
30
31
  this.findObjectInMsg = memo({
32
    cache: lru(5),
33
    asString: function (opts) {
34
      return opts.obj + opts.msg
35
    }
36
  }, this._findObjectInMsg.bind(this))
37
38
  this.getPackIndex = memo({
39
    cache: lru(4),
40
    asString: JSON.stringify
41
  }, this._getPackIndex.bind(this))
42
}
43
44
// open, read, buffer and callback an object
45
Git.prototype.getObject = function (opts, cb) {
46
  var self = this
47
  self.openObject(opts, function (err, obj) {
48
    if (err) return cb(err)
49
    pull(
50
      self.readObject(obj),
51
      u.pullConcat(cb)
52
    )
53
  })
54
}
55
56
// get a message that pushed an object
57
Git.prototype.getObjectMsg = function (opts, cb) {
58
  this.findObject(opts, function (err, loc) {
59
    if (err) return cb(err)
60
    cb(null, loc.msg)
61
  })
62
}
63
64
Git.prototype.openObject = function (opts, cb) {
65
  var self = this
66
  self.findObjectInMsg(opts, function (err, loc) {
67
    if (err) return cb(err)
68
    self.app.ensureHasBlobs([loc.packLink], function (err) {
69
      if (err) return cb(err)
70
      cb(null, {
71
        type: opts.type,
72
        length: opts.length,
73
        offset: loc.offset,
74
        next: loc.next,
75
        packLink: loc.packLink,
76
        idx: loc.idx,
77
        msg: loc.msg,
78
      })
79
    })
80
  })
81
}
82
83
Git.prototype.readObject = function (obj) {
84
  return pull(
85
    this.app.readBlobSlice(obj.packLink, {start: obj.offset, end: obj.next}),
86
    this.decodeObject({
87
      type: obj.type,
88
      length: obj.length,
89
      packLink: obj.packLink,
90
      idx: obj.idx,
91
    })
92
  )
93
}
94
95
// find which packfile contains a git object, and where in the packfile it is
96
// located
97
Git.prototype._findObject = function (opts, cb) {
98
  if (!opts.headMsgId) return cb(new TypeError('missing head message id'))
99
  if (!opts.obj) return cb(new TypeError('missing object id'))
100
  var self = this
101
  var objId = opts.obj
102
  self.findObjectMsgs(opts, function (err, msgs) {
103
    if (err) return cb(err)
104
    if (msgs.length === 0)
105
      return cb(new ObjectNotFoundError('unable to find git object ' + objId))
106
    self.findObjectInMsgs(objId, msgs, cb)
107
  })
108
}
109
110
Git.prototype._findObjectInMsg = function (opts, cb) {
111
  if (!opts.msg) return cb(new TypeError('missing message id'))
112
  if (!opts.obj) return cb(new TypeError('missing object id'))
113
  var self = this
114
  self.app.getMsgDecrypted(opts.msg, function (err, msg) {
115
    if (err) return cb(err)
116
    self.findObjectInMsgs(opts.obj, [msg], cb)
117
  })
118
}
119
120
Git.prototype.findObjectInMsgs = function (objId, msgs, cb) {
121
  var self = this
122
  var objIdBuf = new Buffer(objId, 'hex')
123
  // if blobs may need to be fetched, try to ask the user about as many of them
124
  // at one time as possible
125
  var packidxs = [].concat.apply([], msgs.map(function (msg) {
126
    var c = msg.value.content
127
    var idxs = u.toArray(c.indexes).map(u.toLink)
128
    return u.toArray(c.packs).map(u.toLink).map(function (pack, i) {
129
      var idx = idxs[i]
130
      if (pack && idx) return {
131
        msg: msg,
132
        packLink: pack,
133
        idxLink: idx,
134
      }
135
    })
136
  })).filter(Boolean)
137
  var blobLinks = packidxs.length === 1
138
    ? [packidxs[0].idxLink, packidxs[0].packLink]
139
    : packidxs.map(function (packidx) {
140
      return packidx.idxLink
141
    })
142
  self.app.ensureHasBlobs(blobLinks, function (err) {
143
    if (err) return cb(err)
144
    pull(
145
      pull.values(packidxs),
146
      paramap(function (pack, cb) {
147
        self.getPackIndex(pack.idxLink, function (err, idx) {
148
          if (err) return cb(err)
149
          var offset = idx.find(objIdBuf)
150
          if (!offset) return cb()
151
          cb(null, {
152
            offset: offset.offset,
153
            next: offset.next,
154
            packLink: pack.packLink,
155
            idx: idx,
156
            msg: pack.msg,
157
          })
158
        })
159
      }, 4),
160
      pull.filter(),
161
      pull.take(1),
162
      pull.collect(function (err, offsets) {
163
        if (err) return cb(err)
164
        if (offsets.length === 0)
165
          return cb(new ObjectNotFoundError('unable to find git object '
166
            + objId + ' in ' + msgs.length + ' messages'))
167
        cb(null, offsets[0])
168
      })
169
    )
170
  })
171
}
172
173
// given an object id and ssb msg id, get a set of messages of which at least one pushed the object.
174
Git.prototype.findObjectMsgs = function (opts, cb) {
175
  var self = this
176
  var id = opts.obj
177
  var headMsgId = opts.headMsgId
178
  var ended = false
179
  var waiting = 0
180
  var maybeMsgs = []
181
182
  function cbOnce(err, msgs) {
183
    if (ended) return
184
    ended = true
185
    cb(err, msgs)
186
  }
187
188
  function objectMatches(commit) {
189
    return commit && (commit === id || commit.sha1 === id)
190
  }
191
192
  if (!headMsgId) return cb(new TypeError('missing head message id'))
193
  if (!u.isRef(headMsgId))
194
    return cb(new TypeError('bad head message id \'' + headMsgId + '\''))
195
196
  ;(function getMsg(id) {
197
    waiting++
198
    self.app.getMsgDecrypted(id, function (err, msg) {
199
      waiting--
200
      if (ended) return
201
      if (err && err.name == 'NotFoundError')
202
        return cbOnce(new Error('missing message ' + headMsgId))
203
      if (err) return cbOnce(err)
204
      var c = msg.value.content
205
      if (typeof c === 'string')
206
        return cbOnce(new Error('unable to decrypt message ' + msg.key))
207
      if ((u.toArray(c.object_ids).some(objectMatches))
208
      || (u.toArray(c.tags).some(objectMatches))
209
      || (u.toArray(c.commits).some(objectMatches))) {
210
        // found the object
211
        return cbOnce(null, [msg])
212
      } else if (!c.object_ids) {
213
        // the object might be here
214
        maybeMsgs.push(msg)
215
      }
216
      // traverse the DAG to keep looking for the object
217
      u.toArray(c.repoBranch).filter(u.isRef).forEach(getMsg)
218
      if (waiting === 0) {
219
        cbOnce(null, maybeMsgs)
220
      }
221
    })
222
  })(headMsgId)
223
}
224
225
Git.prototype._getPackIndex = function (idxBlobLink, cb) {
226
  pull(this.app.readBlob(idxBlobLink), packidx(cb))
227
}
228
229
var objectTypes = [
230
  'none', 'commit', 'tree', 'blob',
231
  'tag', 'unused', 'ofs-delta', 'ref-delta'
232
]
233
234
function readTypedVarInt(reader, cb) {
235
  var type, value, shift
236
  reader.read(1, function (end, buf) {
237
    if (ended = end) return cb(end)
238
    var firstByte = buf[0]
239
    type = objectTypes[(firstByte >> 4) & 7]
240
    value = firstByte & 15
241
    shift = 4
242
    checkByte(firstByte)
243
  })
244
245
  function checkByte(byte) {
246
    if (byte & 0x80)
247
      reader.read(1, gotByte)
248
    else
249
      cb(null, type, value)
250
  }
251
252
  function gotByte(end, buf) {
253
    if (ended = end) return cb(end)
254
    var byte = buf[0]
255
    value += (byte & 0x7f) << shift
256
    shift += 7
257
    checkByte(byte)
258
  }
259
}
260
261
function readVarInt(reader, cb) {
262
  var value = 0, shift = 0
263
  reader.read(1, function gotByte(end, buf) {
264
    if (ended = end) return cb(end)
265
    var byte = buf[0]
266
    value += (byte & 0x7f) << shift
267
    shift += 7
268
    if (byte & 0x80)
269
      reader.read(1, gotByte)
270
    else
271
      cb(null, value)
272
  })
273
}
274
275
function inflate(read) {
276
  return toPull(zlib.createInflate())(read)
277
}
278
279
Git.prototype.decodeObject = function (opts) {
280
  var self = this
281
  var packLink = opts.packLink
282
  return function (read) {
283
    var reader = Reader()
284
    reader(read)
285
    return u.readNext(function (cb) {
286
      readTypedVarInt(reader, function (end, type, length) {
287
        if (end === true) cb(new Error('Missing object type'))
288
        else if (end) cb(end)
289
        else if (type === 'ref-delta') getObjectFromRefDelta(length, cb)
290
        else if (opts.type && type !== opts.type)
291
          cb(new Error('expected type \'' + opts.type + '\' ' +
292
            'but found \'' + type + '\''))
293
        else if (opts.length && length !== opts.length)
294
          cb(new Error('expected length ' + opts.length + ' ' +
295
            'but found ' + length))
296
          else cb(null, inflate(reader.read()))
297
      })
298
    })
299
300
    function getObjectFromRefDelta(length, cb) {
301
      reader.read(20, function (end, sourceHash) {
302
        if (end) return cb(end)
303
        var inflatedReader = Reader()
304
        pull(reader.read(), inflate, inflatedReader)
305
        readVarInt(inflatedReader, function (err, expectedSourceLength) {
306
          if (err) return cb(err)
307
          readVarInt(inflatedReader, function (err, expectedTargetLength) {
308
            if (err) return cb(err)
309
            var offset = opts.idx.find(sourceHash)
310
            if (!offset) return cb(null, 'missing source object ' +
311
              sourcehash.toString('hex'))
312
            var readSource = pull(
313
              self.app.readBlobSlice(opts.packLink, {
314
                start: offset.offset,
315
                end: offset.next
316
              }),
317
              self.decodeObject({
318
                type: opts.type,
319
                length: expectedSourceLength,
320
                packLink: opts.packLink,
321
                idx: opts.idx
322
              })
323
            )
324
            cb(null, patchObject(inflatedReader, length, readSource, expectedTargetLength))
325
          })
326
        })
327
      })
328
    }
329
  }
330
}
331
332
function readOffsetSize(cmd, reader, readCb) {
333
  var offset = 0, size = 0
334
335
  function addByte(bit, outPos, cb) {
336
    if (cmd & (1 << bit))
337
      reader.read(1, function (err, buf) {
338
        if (err) readCb(err)
339
        else cb(buf[0] << (outPos << 3))
340
      })
341
    else
342
      cb(0)
343
  }
344
345
  addByte(0, 0, function (val) {
346
    offset = val
347
    addByte(1, 1, function (val) {
348
      offset |= val
349
      addByte(2, 2, function (val) {
350
        offset |= val
351
        addByte(3, 3, function (val) {
352
          offset |= val
353
          addSize()
354
        })
355
      })
356
    })
357
  })
358
  function addSize() {
359
    addByte(4, 0, function (val) {
360
      size = val
361
      addByte(5, 1, function (val) {
362
        size |= val
363
        addByte(6, 2, function (val) {
364
          size |= val
365
          readCb(null, offset, size || 0x10000)
366
        })
367
      })
368
    })
369
  }
370
}
371
372
function patchObject(deltaReader, deltaLength, readSource, targetLength) {
373
  var srcBuf
374
  var ended
375
376
  return u.readNext(function (cb) {
377
    pull(readSource, u.pullConcat(function (err, buf) {
378
      if (err) return cb(err)
379
      srcBuf = buf
380
      cb(null, read)
381
    }))
382
  })
383
384
  function read(abort, cb) {
385
    if (ended) return cb(ended)
386
    deltaReader.read(1, function (end, dBuf) {
387
      if (ended = end) return cb(end)
388
      var cmd = dBuf[0]
389
      if (cmd & 0x80)
390
        // skip a variable amount and then pass through a variable amount
391
        readOffsetSize(cmd, deltaReader, function (err, offset, size) {
392
          if (err) return earlyEnd(err)
393
          var buf = srcBuf.slice(offset, offset + size)
394
          cb(end, buf)
395
        })
396
      else if (cmd)
397
        // insert `cmd` bytes from delta
398
        deltaReader.read(cmd, cb)
399
      else
400
        cb(new Error("unexpected delta opcode 0"))
401
    })
402
403
    function earlyEnd(err) {
404
      cb(err === true ? new Error('stream ended early') : err)
405
    }
406
  }
407
}
408
409
var gitNameRegex = /^(.*) <(([^>@]*)(@[^>]*)?)> (.*) (.*)$/
410
function parseName(line) {
411
  var m = gitNameRegex.exec(line)
412
  if (!m) return null
413
  return {
414
    name: m[1],
415
    email: m[2],
416
    localpart: m[3],
417
    feed: u.isRef(m[4]) && m[4] || undefined,
418
    date: new Date(m[5] * 1000),
419
    tz: m[6],
420
  }
421
}
422
423
Git.prototype.getCommit = function (obj, cb) {
424
  pull(this.readObject(obj), u.pullConcat(function (err, buf) {
425
    if (err) return cb(err)
426
    var commit = {
427
      msg: obj.msg,
428
      parents: [],
429
    }
430
    var authorLine, committerLine
431
    var lines = buf.toString('utf8').split('\n')
432
    for (var line; (line = lines.shift()); ) {
433
      var parts = line.split(' ')
434
      var prop = parts.shift()
435
      var value = parts.join(' ')
436
      switch (prop) {
437
        case 'tree':
438
          commit.tree = value
439
          break
440
        case 'parent':
441
          commit.parents.push(value)
442
          break
443
        case 'author':
444
          authorLine = value
445
          break
446
        case 'committer':
447
          committerLine = value
448
          break
449
        case 'gpgsig':
450
          var sigLines = [value]
451
          while (lines[0] && lines[0][0] == ' ')
452
            sigLines.push(lines.shift().slice(1))
453
          commit.gpgsig = sigLines.join('\n')
454
          break
455
        default:
456
          return cb(new TypeError('unknown git object property ' + prop))
457
      }
458
    }
459
    commit.committer = parseName(committerLine)
460
    if (authorLine !== committerLine) commit.author = parseName(authorLine)
461
    commit.body = lines.join('\n')
462
    cb(null, commit)
463
  }))
464
}
465
466
Git.prototype.getTag = function (obj, cb) {
467
  pull(this.readObject(obj), u.pullConcat(function (err, buf) {
468
    if (err) return cb(err)
469
    var tag = {
470
      msg: obj.msg,
471
    }
472
    var authorLine, tagterLine
473
    var lines = buf.toString('utf8').split('\n')
474
    for (var line; (line = lines.shift()); ) {
475
      var parts = line.split(' ')
476
      var prop = parts.shift()
477
      var value = parts.join(' ')
478
      switch (prop) {
479
        case 'object':
480
          tag.object = value
481
          break
482
        case 'type':
483
          if (!types[value])
484
            return cb(new TypeError('unknown git object type ' + type))
485
          tag.type = value
486
          break
487
        case 'tag':
488
          tag.tag = value
489
          break
490
        case 'tagger':
491
          tag.tagger = parseName(value)
492
          break
493
        default:
494
          return cb(new TypeError('unknown git object property ' + prop))
495
      }
496
    }
497
    tag.body = lines.join('\n')
498
    cb(null, tag)
499
  }))
500
}
501
502
function readCString(reader, cb) {
503
  var chars = []
504
  reader.read(1, function next(err, ch) {
505
    if (err) return cb(err)
506
    if (ch[0] === 0) return cb(null, Buffer.concat(chars).toString('utf8'))
507
    chars.push(ch)
508
    reader.read(1, next)
509
  })
510
}
511
512
Git.prototype.readTree = function (obj) {
513
  var reader = Reader()
514
  reader(this.readObject(obj))
515
  return function (abort, cb) {
516
    if (abort) return reader.abort(abort, cb)
517
    readCString(reader, function (err, str) {
518
      if (err) return cb(err)
519
      var parts = str.split(' ')
520
      var mode = parseInt(parts[0], 8)
521
      var name = parts.slice(1).join(' ')
522
      reader.read(20, function (err, hash) {
523
        if (err) return cb(err)
524
        cb(null, {
525
          name: name,
526
          mode: mode,
527
          hash: hash.toString('hex')
528
        })
529
      })
530
    })
531
  }
532
}
533