git ssb

0+

cel / pull-git-remote-helper



Commit c2f96fca4e0046fd6b5ee048898f8db2c2c1d247

Browse Files

Almost got upload-pack working

Charles Lehner committed on 2/8/2016, 4:37:24 AM
Parent: 20c102b7036f7fee19f151e269d33cd7da837a3b

Files changed

index.jschanged
pack.jschanged
test/git-remote-empty.jschanged
test/git-remote-full.jschanged
test/run.jschanged
test/repo.jsadded
index.jsView
@@ -64,115 +64,93 @@
6464 }
6565 }
6666
6767 // upload-pack: fetch to client
68-function uploadPack(read, objectSource, refSource, wantSink, options) {
68+function uploadPack(read, getObjects, refSource, wantSink, options) {
6969 /* multi_ack thin-pack side-band side-band-64k ofs-delta shallow no-progress
7070 * include-tag multi_ack_detailed symref=HEAD:refs/heads/master
7171 * agent=git/2.7.0 */
7272 var sendRefs = receivePackHeader([
7373 ], refSource, false)
7474
7575 var lines = packLineDecode(read, options)
7676 // var havesSink = pull.drain(console.error.bind(console, 'have:'))
77- var readHave = lines.haves
77+ var readHave = lines.haves()
7878 var acked
7979 var commonHash
8080 var sendPack
8181 var earlyDisconnect
8282
8383 function haveObject(hash, cb) {
8484 cb(/* TODO */)
85+ // cb(true)
8586 }
8687
87- function getObjects(commonObjId, cb) {
88- console.error('get obj', commonObjId)
89- cb(null, 0, function readObject(end, cb) {
90- console.error('read obj', end)
91- // cb(new Error('Not implemented'))
92- cb(true)
93- })
94- }
95-
9688 // Packfile negotiation
97- return packLineEncode(
98- cat([
89+ return cat([
90+ packLineEncode(cat([
9991 sendRefs,
10092 pull.once(''),
10193 function (abort, cb) {
10294 if (abort) return
10395 if (acked) return cb(true)
10496 // read upload request (wants list) from client
105- pull(
106- lines.wants,
107- onThroughEnd(wantsDone),
108- wantSink
109- )
97+ var readWant = lines.wants(wantsDone)
98+ readWant(null, function (end, want) {
99+ if (end === true) {
100+ // client disconnected before sending wants
101+ earlyDisconnect = true
102+ cb(true)
103+ } else if (end) {
104+ cb(end)
105+ } else {
106+ wantSink(readWant)
107+ }
108+ })
110109
111- var gotAnyHaves = false
112-
113110 function wantsDone(err) {
114- console.error('wants done', err)
111+ // console.error('wants done', err, earlyDisconnect)
115112 if (err) return cb(err)
113+ if (earlyDisconnect) return cb(true)
116114 // Read upload haves (haves list).
117115 // On first obj-id that we have, ACK
118116 // If we have none, NAK.
119117 // TODO: implement multi_ack_detailed
120118 readHave(null, function next(end, have) {
121119 if (end === true) {
122- if (gotAnyHaves) {
123- // found no common object
124- acked = true
125- cb(null, 'NAK')
126- } else {
127- earlyDisconnect = true
128- // client disconnected before sending haves.
129- cb(true)
130- }
120+ // found no common object
121+ acked = true
122+ cb(null, 'NAK')
131123 } else if (end)
132124 cb(end)
133125 else if (have.type != 'have')
134126 cb(new Error('Unknown have' + JSON.stringify(have)))
135- else {
136- gotAnyHaves = true
127+ else
128+ console.error('got have', have),
137129 haveObject(have.hash, function (haveIt) {
138130 if (!haveIt)
139131 return readHave(null, next)
140132 commonHash = haveIt
141133 acked = true
142134 cb(null, 'ACK ' + have.hash)
143135 })
144- }
145136 })
146137 }
147- /*
148- function havesDone(err) {
149- console.error('haves done', err)
150- if (err) return cb(err)
151- cb(true)
152- pull(
153- lines.passthrough,
154- pull.drain(function (buf) {
155- console.error('got buf after wants', buf.length, buf.toString('ascii'))
156- })
157- )
158- }
159- */
160138 },
161- function havesDone(abort, cb) {
162- console.error("haves done", abort && typeof abort, sendPack && typeof sendPack)
163- if (abort || earlyDisconnect) return cb(abort || true)
164- // send pack file to client
165- if (!sendPack)
166- getObjects(commonHash, function (err, numObjects, readObject) {
167- sendPack = pack.encode(numObjects, readObject)
168- havesDone(abort, cb)
169- })
170- else
171- sendPack(abort, cb)
172- }
173- ])
174- )
139+ ])),
140+ function havesDone(abort, cb) {
141+ // console.error("haves done", abort && typeof abort, sendPack && typeof sendPack, abort, earlyDisconnect)
142+ if (abort || earlyDisconnect) return cb(abort || true)
143+ // send pack file to client
144+ if (!sendPack)
145+ getObjects(commonHash, function (err, numObjects, readObject) {
146+ sendPack = pack.encode(numObjects, readObject)
147+ havesDone(abort, cb)
148+ })
149+ else
150+ sendPack(abort, cb)
151+ }
152+ ])
175153 }
176154
177155 function packLineEncode(read) {
178156 var ended
@@ -189,9 +167,9 @@
189167 data = ''
190168 var len = data ? data.length + 4 : 0
191169 var hexLen = ('000' + len.toString(16)).substr(-4)
192170 var pkt = hexLen + data
193- // console.error('>', JSON.stringify(pkt))
171+ console.error('>', JSON.stringify(pkt))
194172 cb(end, pkt)
195173 }
196174 })
197175 }
@@ -200,8 +178,15 @@
200178 function rev(str) {
201179 return str === '0000000000000000000000000000000000000000' ? null : str
202180 }
203181
182+/* pull-stream/source.js */
183+function abortCb(cb, abort, onAbort) {
184+ cb(abort)
185+ onAbort && onAbort(abort === true ? null: abort)
186+ return
187+}
188+
204189 function packLineDecode(read, options) {
205190 var b = buffered(read)
206191 var readPrefix = b.chunks(4)
207192 var ended
@@ -252,24 +237,27 @@
252237 })
253238 })
254239 }
255240
256- function readWant(abort, cb) {
257- readPackLineStr(abort, function (end, line) {
258- if (end) return cb(end)
259- if (options.verbosity >= 2)
260- console.error('line', line)
261- // if (!line.length) return cb(true)
262- if (!line.length || line == 'done') return cb(true)
263- var args = split3(line)
264- var caps = args[2]
265- if (caps && options.verbosity >= 2)
266- console.error('want capabilities:', caps)
267- cb(null, {
268- type: args[0],
269- hash: args[1],
241+ function havesWants(onEnd) {
242+ return function readWant(abort, cb) {
243+ readPackLineStr(abort, function (end, line) {
244+ if (end) return abortCb(cb, end, onEnd)
245+ if (options.verbosity >= 2)
246+ console.error('line', line)
247+ // if (!line.length) return cb(true)
248+ if (!line.length || line == 'done')
249+ return abortCb(cb, true, onEnd)
250+ var args = split3(line)
251+ var caps = args[2]
252+ if (caps && options.verbosity >= 2)
253+ console.error('want capabilities:', caps)
254+ cb(null, {
255+ type: args[0],
256+ hash: args[1],
257+ })
270258 })
271- })
259+ }
272260 }
273261
274262 /*
275263 function readWant(abort, cb) {
@@ -294,9 +282,9 @@
294282 */
295283
296284 b.packLines = readPackLine
297285 b.updates = readUpdate
298- b.wants = b.haves = readWant
286+ b.wants = b.haves = havesWants
299287
300288 return b
301289 }
302290
@@ -396,9 +384,11 @@
396384 module.exports = function (opts) {
397385 var ended
398386 var prefix = opts.prefix
399387 var objectSink = opts.objectSink
400- var objectSource = opts.objectSource || pull.empty()
388+ var getObjects = opts.getObjects || function (id, cb) {
389+ cb(null, 0, pull.empty())
390+ }
401391 var refSource = opts.refSource || pull.empty()
402392 var refSink = opts.refSink || pull.drain()
403393 var wantSink = opts.wantSink || pull.drain()
404394
@@ -410,9 +400,9 @@
410400 function handleConnect(cmd, read) {
411401 var args = split2(cmd)
412402 switch (args[0]) {
413403 case 'git-upload-pack':
414- return prepend('\n', uploadPack(read, objectSource, refSource,
404+ return prepend('\n', uploadPack(read, getObjects, refSource,
415405 wantSink, options))
416406 case 'git-receive-pack':
417407 return prepend('\n', receivePack(read, objectSink, refSource,
418408 refSink, options))
pack.jsView
@@ -1,8 +1,8 @@
11 var buffered = require('pull-buffered')
22 var pull = require('pull-stream')
33 var toPull = require('stream-to-pull-stream')
4-var Inflate = require('pako/lib/inflate').Inflate
4+var pako = require('pako')
55 var createHash = require('./util').createHash
66 var cat = require('pull-cat')
77
88 exports.decode = decodePack
@@ -13,17 +13,23 @@
1313 var objectTypes = [
1414 'none', 'commit', 'tree', 'blob',
1515 'tag', 'unused', 'ofs-delta', 'ref-delta'
1616 ]
17+var objectTypeNums = {
18+ commit: 1,
19+ tree: 2,
20+ blob: 3,
21+ tag: 4
22+}
1723
1824 function error(cb) {
1925 return function (err) {
2026 cb(err || true)
2127 }
2228 }
2329
2430 function inflateBytes(read) {
25- var inflate = new Inflate()
31+ var inflate = new pako.Inflate()
2632 var ended, dataOut
2733
2834 inflate.onData = function (data) {
2935 dataOut = new Buffer(data)
@@ -55,8 +61,45 @@
5561 })
5662 }
5763 }
5864
65+function deflate(read) {
66+ var def = new pako.Deflate()
67+ var queue = []
68+ var ended
69+
70+ def.onData = function (data) {
71+ queue.push([null, new Buffer(data)])
72+ }
73+
74+ def.onEnd = function (status) {
75+ queue.push([(status === 0) ? true : new Error(def.msg)])
76+ }
77+
78+ return function readOut(abort, cb) {
79+ /*
80+ function _cb(end, data) {
81+ console.error('sending deflated', end,
82+ data && JSON.stringify(data.toString()))
83+ cb(end, data)
84+ }
85+ */
86+ if (ended)
87+ cb(ended)
88+ else if (queue.length)
89+ cb.apply(this, queue.shift())
90+ else
91+ read(abort, function next(end, data) {
92+ if (data)
93+ console.error('read into deflat', data.length, JSON.stringify(data))
94+ if (end === true) def.push([], true)
95+ else if (end) return cb(end)
96+ else def.push(data)
97+ readOut(null, cb)
98+ })
99+ }
100+}
101+
59102 function decodePack(onEnd, read) {
60103 if (read === undefined)
61104 return decodePack.bind(this, onEnd)
62105
@@ -111,9 +154,9 @@
111154 if (ended = end) return cb(end)
112155 var firstByte = buf[0]
113156 type = objectTypes[(firstByte >> 4) & 7]
114157 value = firstByte & 15
115- // console.error('byte1', firstByte, firstByte.toString(2), value, value.toString(2))
158+ console.error('byte1', firstByte, firstByte.toString(2), value, value.toString(2))
116159 shift = 4
117160 checkByte(firstByte)
118161 })
119162
@@ -128,9 +171,9 @@
128171 if (ended = end) return cb(end)
129172 var byte = buf[0]
130173 value += (byte & 0x7f) << shift
131174 shift += 7
132- // console.error('byte', byte, byte.toString(2), value, value.toString(2))
175+ console.error('byte', byte, byte.toString(2), value, value.toString(2))
133176 checkByte(byte)
134177 }
135178 }
136179
@@ -175,28 +218,87 @@
175218
176219 return readObject
177220 }
178221
222+function once(read) {
223+ var done
224+ return function (abort, cb) {
225+ if (done) cb(done)
226+ else done = true, read(abort, cb)
227+ }
228+}
229+
230+function encodeVarInt(typeStr, length, cb) {
231+ var type = objectTypeNums[typeStr]
232+ // console.error('TYPE', type, typeStr, 'len', length, typeof cb)
233+ if (!type)
234+ return cb(new Error("Bad object type " + typeStr))
235+
236+ var vals = []
237+ var b = (type << 4) | (length & 15)
238+ for (length >>= 4; length; length >>= 7) {
239+ vals.push(b | 0x80)
240+ b = length & 0x7f
241+ }
242+ vals.push(b)
243+ console.error('sending var int', vals, vals.map(function (n) {
244+ return ('00000000' + Number(n).toString(2)).substr(-8)
245+ }))
246+ cb(null, new Buffer(vals))
247+}
248+
249+/*
250+function flow(read) {
251+ return function (abort, cb) {
252+ read(abort, cb, function nextCb(newRead, onEnd) {
253+ read = newRead
254+ return function (end, data) {
255+ cb(end, data)
256+ if (end) onEnd(end === true ? null : end)
257+ }
258+ })
259+ }
260+}
261+*/
262+
179263 function encodePack(numObjects, readObject) {
180- var ended
264+ // var ended
181265 var header = new Buffer(12)
182266 header.write('PACK')
183267 header.writeUInt32BE(PACK_VERSION, 4)
184268 header.writeUInt32BE(numObjects, 8)
185269 var checksum = createHash('sha1')
270+ var readData
186271
272+ /*
187273 return pull.through(function (data) {
188- console.error('> ' + data.length, data.toString())
274+ console.error('> ' + data.length, JSON.stringify(data.toString('ascii')))
189275 })(cat([
276+ */
277+ return cat([
190278 checksum(cat([
191279 pull.once(header),
192- function (abort, cb) {
193- if (ended) return cb(ended)
194- readObject(abort, function (end, type, length, read) {
195- if (ended = end) return cb(end)
196- console.error('TODO: encode object')
197- })
198- }
280+ encodeObject
199281 ])),
200282 checksum.readDigest
201- ]))
283+ ])
284+ // )
285+
286+ function encodeObject(abort, cb) {
287+ if (readData)
288+ readData(abort, function (end, data) {
289+ if (end === true)
290+ readObject(abort, nextObject)
291+ else
292+ cb(end, data)
293+ })
294+ else
295+ readObject(abort, nextObject)
296+
297+ function nextObject(end, type, length, read) {
298+ // console.error('got obj', end, type, length)
299+ if (end) return cb(end)
300+ readData = deflate(read)
301+ encodeVarInt(type, length, cb) // nextCb(deflate(read), encodeObject))
302+ }
303+ }
202304 }
test/git-remote-empty.jsView
@@ -25,8 +25,9 @@
2525 hasher,
2626 pull.collect(function (err, bufs) {
2727 if (err) throw err
2828 var buf = Buffer.concat(bufs)
29+ console.error('obj', type, length, JSON.stringify(buf.toString('ascii')))
2930 process.send({object: {
3031 type: type,
3132 data: buf.toString('ascii'),
3233 length: length,
test/git-remote-full.jsView
@@ -1,8 +1,9 @@
11 #!/usr/bin/env node
22
33 var toPull = require('stream-to-pull-stream')
44 var pull = require('pull-stream')
5+var repo = require('./repo')
56
67 process.on('uncaughtException', function (err) {
78 if (err.stack)
89 err = {stack: err.stack, message: err.message}
@@ -15,16 +16,40 @@
1516 {name: 'refs/heads/master', value: HEAD},
1617 {name: 'HEAD', value: HEAD}
1718 ]
1819
20+var objects = [
21+ {type: 'commit', object: repo.commit},
22+ {type: 'tree', object: repo.tree},
23+ {type: 'blob', object: repo.file}
24+]
25+
26+function streamObject(read) {
27+ var ended
28+ return function readObject(abort, cb) {
29+ read(abort, function (end, item) {
30+ if (ended = end) return cb(end)
31+ var data = item.object.data
32+ cb(null, item.type, data.length, pull.once(data))
33+ })
34+ }
35+}
36+
1937 pull(
2038 toPull(process.stdin),
2139 require('../')({
2240 prefix: 'foo',
2341 refSource: pull.values(refs),
2442 wantSink: pull.drain(function (want) {
2543 process.send({want: want})
2644 }),
45+ getObjects: function (ancestorHash, cb) {
46+ // console.error('get obj!', ancestorHash)
47+ cb(null, objects.length, pull(
48+ pull.values(objects),
49+ streamObject
50+ ))
51+ }
2752 }),
2853 toPull(process.stdout, function (err) {
2954 if (err)
3055 throw err
test/run.jsView
@@ -3,19 +3,14 @@
33 var path = require('path')
44 var mktemp = require('mktemp')
55 var rimraf = require('rimraf')
66 var fs = require('fs')
7+var repo = require('./repo')
78
8-function noop() {}
9-
109 var env = Object.create(process.env)
1110 env.PATH = __dirname + ':' + env.PATH
12-env.GIT_AUTHOR_DATE = env.GIT_COMMITTER_DATE = '1000000000 -0500'
13-var user = {
14- name: 'test',
15- email: 'test@localhost'
16-}
17-user.str = user.name + ' <' + user.email + '>'
11+env.GIT_AUTHOR_DATE = env.GIT_COMMITTER_DATE = repo.date
12+var user = repo.user
1813 var remote = {
1914 empty: 'empty.js://',
2015 full: 'full.js://'
2116 }
@@ -78,38 +73,16 @@
7873 t.end()
7974 })
8075 })
8176
82-function hexToStr(str) {
83- var buf = new Buffer(str.length / 2)
84- buf.hexWrite(str)
85- return buf.toString('ascii')
86-}
87-
8877 tape('make a commit and push', function (t) {
8978 t.plan(8) // write, add, commit, push, ref, commit, tree, blob
9079
91- var file = {
92- name: 'blah.txt',
93- data: 'i am a file',
94- hash: '68bd10497ea68e91fa85024d0a0b2fe54e212914'
95- }
80+ var file = repo.file
81+ var commitMessage = repo.commitMessage
82+ var commit = repo.commit
83+ var tree = repo.tree
9684
97- var tree = {
98- hash: '75c54aa020772a916853987a03bff7079463a861',
99- data: '100644 ' + file.name + '\0' + hexToStr(file.hash)
100- }
101-
102- var commitMessage = 'Initial commit'
103- var commit = {
104- hash: 'edb5b50e8019797925820007d318870f8c346726',
105- data: ['tree ' + tree.hash,
106- 'author ' + user.str + ' 1000000000 -0500',
107- 'committer ' + user.str + ' 1000000000 -0500',
108- '', commitMessage, ''
109- ].join('\n')
110- }
111-
11285 function obj(type, o) {
11386 return {
11487 type: type,
11588 data: o.data,
@@ -165,9 +138,9 @@
165138
166139 0 &&
167140 tape('clone into new dir', function (t) {
168141 var dir = path.join(tmpDir, 'clonedir')
169- t.plan(3)
142+ t.plan(2)
170143 t.git('clone', '-vv', remote.full, dir, function (msg) {
171144 if (msg.want)
172145 t.deepEquals(msg.want, {
173146 type: 'want',
test/repo.jsView
@@ -1,0 +1,41 @@
1+function hexToStr(str) {
2+ var buf = new Buffer(str.length / 2)
3+ buf.hexWrite(str)
4+ return buf.toString('ascii')
5+}
6+
7+var date = '1000000000 -0500'
8+
9+var user = {
10+ name: 'test',
11+ email: 'test@localhost'
12+}
13+user.str = user.name + ' <' + user.email + '>'
14+
15+var file = {
16+ name: 'blah.txt',
17+ data: 'i am a file',
18+ hash: '68bd10497ea68e91fa85024d0a0b2fe54e212914'
19+}
20+
21+var tree = {
22+ hash: '75c54aa020772a916853987a03bff7079463a861',
23+ data: '100644 ' + file.name + '\0' + hexToStr(file.hash)
24+}
25+
26+var commitMessage = 'Initial commit'
27+var commit = {
28+ hash: 'edb5b50e8019797925820007d318870f8c346726',
29+ data: ['tree ' + tree.hash,
30+ 'author ' + user.str + ' ' + date,
31+ 'committer ' + user.str + ' ' + date,
32+ '', commitMessage, ''
33+ ].join('\n')
34+}
35+
36+exports.date = date
37+exports.user = user
38+exports.file = file
39+exports.tree = tree
40+exports.commitMessage = commitMessage
41+exports.commit = commit

Built with git-ssb-web