git ssb - Start implementing pack negotiation and encoding · cel/pull-git-remote-helper@04c77cc7

Commit 04c77cc71b341054c04792a9bc6c74aa4c18d225

Start implementing pack negotiation and encoding

Charles Lehner committed on 2/8/2016, 4:37:24 AM
Parent: b1ed4631de0942640e4c7591c2be9dbdf8e98240

Files changed

index.js	changed
pack.js	changed
test/git-remote-full.js	changed
test/run.js	changed
util.js	changed

index.jsView
		@@ -70,30 +70,94 @@
70	70	* agent=git/2.7.0 */
71	71	var sendRefs = receivePackHeader([
72	72	], refSource, false)
73	73
	74	+ var lines = packLineDecode(read, options)
	75	+ // var havesSink = pull.drain(console.error.bind(console, 'have:'))
	76	+ var readHave = lines.haves
	77	+ var acked
	78	+ var commonHash
	79	+ var sendPack
	80	+
	81	+ function haveObject(hash, cb) {
	82	+ cb(/* TODO */)
	83	+ }
	84	+
	85	+ function getObjects(commonObjId, cb) {
	86	+ console.error('get obj', commonObjId)
	87	+ cb(null, 0, function readObject(end, cb) {
	88	+ console.error('read obj', end)
	89	+ // cb(new Error('Not implemented'))
	90	+ cb(true)
	91	+ })
	92	+ }
	93	+
	94	+ // Packfile negotiation
74	95	return packLineEncode(
75	96	cat([
76	97	sendRefs,
77	98	pull.once(''),
78	99	function (abort, cb) {
79	100	if (abort) return
80		- // read client wants
81		- var lines = packLineDecode(read, options)
	101	+ if (acked) return cb(true)
	102	+ // read upload request (wants list) from client
82	103	pull(
83	104	lines.wants,
84	105	onThroughEnd(wantsDone),
85	106	wantSink
86	107	)
	108	+
87	109	function wantsDone(err) {
	110	+ console.error('wants done', err)
88	111	if (err) return cb(err)
	112	+ // Read upload haves (haves list).
	113	+ // On first obj-id that we have, ACK
	114	+ // If we have none, NAK.
	115	+ // TODO: implement multi_ack_detailed
	116	+ readHave(null, function next(end, have) {
	117	+ ended = end
	118	+ if (end === true) {
	119	+ // found no common object
	120	+ acked = true
	121	+ cb(null, 'NAK')
	122	+ } else if (end)
	123	+ cb(end)
	124	+ else if (have.type != 'have')
	125	+ cb(new Error('Unknown have' + JSON.stringify(have)))
	126	+ else
	127	+ haveObject(have.hash, function (haveIt) {
	128	+ if (!haveIt)
	129	+ return readHave(null, next)
	130	+ commonHash = haveIt
	131	+ acked = true
	132	+ cb(null, 'ACK ' + have.hash)
	133	+ })
	134	+ })
	135	+ }
	136	+ /*
	137	+ function havesDone(err) {
	138	+ console.error('haves done', err)
	139	+ if (err) return cb(err)
	140	+ cb(true)
89	141	pull(
90	142	lines.passthrough,
91	143	pull.drain(function (buf) {
92		- console.error('got buf after wants', buf.toString('ascii'))
	144	+ console.error('got buf after wants', buf.length, buf.toString('ascii'))
93	145	})
94	146	)
95	147	}
	148	+ */
	149	+ },
	150	+ function next(abort, cb) {
	151	+ if (abort) return cb(abort)
	152	+ // send pack file to client
	153	+ if (!sendPack)
	154	+ getObjects(commonHash, function (err, numObjects, readObject) {
	155	+ sendPack = pack.encode(numObjects, readObject)
	156	+ next(abort, cb)
	157	+ })
	158	+ else
	159	+ sendPack(abort, cb)
96	160	}
97	161	])
98	162	)
99	163	}
		@@ -105,10 +169,18 @@
105	169	read(end, function (end, data) {
106	170	if (ended = end) {
107	171	cb(end)
108	172	} else {
109		- var len = data ? data.length + 5 : 0
110		- cb(end, ('000' + len.toString(16)).substr(-4) + data + '\n')
	173	+ // console.error("data", data)
	174	+ if (data)
	175	+ data += '\n'
	176	+ else
	177	+ data = ''
	178	+ var len = data ? data.length + 4 : 0
	179	+ var hexLen = ('000' + len.toString(16)).substr(-4)
	180	+ var pkt = hexLen + data
	181	+ // console.error('>', JSON.stringify(pkt))
	182	+ cb(end, pkt)
111	183	}
112	184	})
113	185	}
114	186	}
		@@ -122,8 +194,9 @@
122	194	var readPrefix = b.chunks(4)
123	195	var ended
124	196
125	197	function readPackLine(abort, cb) {
	198	+ if (ended) return cb(ended)
126	199	readPrefix(abort, function (end, buf) {
127	200	if (ended = end) return cb(end)
128	201	var len = parseInt(buf, 16)
129	202	if (!len)
		@@ -135,8 +208,21 @@
135	208	})
136	209	})
137	210	}
138	211
	212	+ function readPackLineStr(abort, cb) {
	213	+ if (ended) return cb(ended)
	214	+ readPackLine(abort, function (end, buf) {
	215	+ if (ended = end) return cb(end)
	216	+ // trim newline
	217	+ var len = buf.length
	218	+ if (buf[len - 1] == 0xa)
	219	+ len--
	220	+ var line = buf.toString('ascii', 0, len)
	221	+ cb(null, line)
	222	+ })
	223	+ }
	224	+
139	225	function readUpdate(abort, cb) {
140	226	readPackLine(abort, function (end, line) {
141	227	if (end) return cb(end)
142	228	if (options.verbosity >= 2)
		@@ -155,27 +241,50 @@
155	241	})
156	242	}
157	243
158	244	function readWant(abort, cb) {
	245	+ readPackLineStr(abort, function (end, line) {
	246	+ if (end) return cb(end)
	247	+ if (options.verbosity >= 2)
	248	+ console.error('line', line)
	249	+ // if (!line.length) return cb(true)
	250	+ if (!line.length \|\| line == 'done') return cb(true)
	251	+ var args = split3(line)
	252	+ var caps = args[2]
	253	+ if (caps && options.verbosity >= 2)
	254	+ console.error('want capabilities:', caps)
	255	+ cb(null, {
	256	+ type: args[0],
	257	+ hash: args[1],
	258	+ })
	259	+ })
	260	+ }
	261	+
	262	+ /*
	263	+ function readWant(abort, cb) {
159	264	readPackLine(abort, function (end, line) {
160	265	if (end) return cb(end)
161	266	if (options.verbosity >= 2)
162	267	console.error('line', line.toString('ascii'))
163		- if (!line.length \|\| line == 'done') return cb(true)
	268	+ if (!line.length \|\| line == 'done') {
	269	+ console.error('WANTS done', line, line.length)
	270	+ return cb(true)
	271	+ }
164	272	var args = split3(line.toString('ascii'))
165	273	var caps = args[2]
166	274	if (caps && options.verbosity >= 2)
167	275	console.error('want capabilities:', caps)
168	276	cb(null, {
169	277	type: args[0],
170		- hash: args[1],
	278	+ hash: args[1].replace(/\n$/, ''),
171	279	})
172	280	})
173	281	}
	282	+ */
174	283
175	284	b.packLines = readPackLine
176	285	b.updates = readUpdate
177		- b.wants = readWant
	286	+ b.wants = b.haves = readWant
178	287
179	288	return b
180	289	}
181	290

pack.jsView
		@@ -2,11 +2,15 @@
2	2	var pull = require('pull-stream')
3	3	var toPull = require('stream-to-pull-stream')
4	4	var Inflate = require('pako/lib/inflate').Inflate
5	5	var createHash = require('./util').createHash
	6	+var cat = require('pull-cat')
6	7
7	8	exports.decode = decodePack
	9	+exports.encode = encodePack
8	10
	11	+var PACK_VERSION = 2
	12	+
9	13	var objectTypes = [
10	14	'none', 'commit', 'tree', 'blob',
11	15	'tag', 'unused', 'ofs-delta', 'ref-delta'
12	16	]
		@@ -170,4 +174,29 @@
170	174	}
171	175
172	176	return readObject
173	177	}
	178	+
	179	+function encodePack(numObjects, readObject) {
	180	+ var ended
	181	+ var header = new Buffer(12)
	182	+ header.write('PACK')
	183	+ header.writeUInt32BE(PACK_VERSION, 4)
	184	+ header.writeUInt32BE(numObjects, 8)
	185	+ var checksum = createHash('sha1')
	186	+
	187	+ return pull.through(function (data) {
	188	+ console.error('> ' + data.length, data.toString())
	189	+ })(cat([
	190	+ checksum(cat([
	191	+ pull.once(header),
	192	+ function (abort, cb) {
	193	+ if (ended) return cb(ended)
	194	+ readObject(abort, function (end, type, length, read) {
	195	+ if (ended = end) return cb(end)
	196	+ console.error('TODO: encode object')
	197	+ })
	198	+ }
	199	+ ])),
	200	+ checksum.readDigest
	201	+ ]))
	202	+}

test/git-remote-full.jsView
		@@ -21,9 +21,8 @@
21	21	require('../')({
22	22	prefix: 'foo',
23	23	refSource: pull.values(refs),
24	24	wantSink: pull.drain(function (want) {
25		- console.error('got want', want)
26	25	process.send({want: want})
27	26	}),
28	27	}),
29	28	toPull(process.stdout, function (err) {

test/run.jsView
		@@ -20,8 +20,12 @@
20	20	full: 'full.js://'
21	21	}
22	22
23	23	var tmpDir = mktemp.createDirSync(path.join(require('os').tmpdir(), 'XXXXXXX'))
	24	+tape.onFinish(function () {
	25	+ if (tmpDir)
	26	+ rimraf.sync(tmpDir)
	27	+})
24	28
25	29	function handleIpcMessage(t, cb) {
26	30	return function (msg) {
27	31	if (msg.error) {
		@@ -149,17 +153,30 @@
149	153	})
150	154	})
151	155	})
152	156
153		-tape('fetch', function (t) {
	157	+0 &&
	158	+tape('fetch when already up-to-date', function (t) {
154	159	t.git('fetch', '-vv', remote.full, function (msg) {
155		- t.notOk('should not get a message here', msg)
	160	+ t.notOk(msg, 'should not get a message here')
156	161	}, function (code) {
157	162	t.equals(code, 0, 'fetched')
158	163	t.end()
159	164	})
160	165	})
161	166
162		-tape.onFinish(function () {
163		- if (tmpDir)
164		- rimraf.sync(tmpDir)
	167	+0 &&
	168	+tape('clone into new dir', function (t) {
	169	+ var dir = path.join(tmpDir, 'clonedir')
	170	+ t.plan(3)
	171	+ t.git('clone', '-vv', remote.full, dir, function (msg) {
	172	+ if (msg.want)
	173	+ t.deepEquals(msg.want, {
	174	+ type: 'want',
	175	+ hash: 'edb5b50e8019797925820007d318870f8c346726'
	176	+ }, 'got want')
	177	+ else
	178	+ t.notOk(msg, 'unexpected message')
	179	+ }, function (code) {
	180	+ t.equals(code, 0, 'cloned')
	181	+ })
165	182	})

util.jsView
		@@ -3,10 +3,15 @@
3	3
4	4	exports.createHash = function (type) {
5	5	var hash = crypto.createHash(type)
6	6	var hasher = pull.through(hash.update.bind(hash))
	7	+ var digest
7	8	hasher.hash = hash
8	9	hasher.digest = hash.digest.bind(hash)
	10	+ hasher.readDigest = function (abort, cb) {
	11	+ if (digest) cb(true)
	12	+ else cb(null, digest = hash.digest())
	13	+ }
9	14	return hasher
10	15	}
11	16
12	17	exports.createGitObjectHash = function (objectType, objectLength) {

Built with git-ssb-web

cel / pull-git-remote-helper

Commit 04c77cc71b341054c04792a9bc6c74aa4c18d225

Start implementing pack negotiation and encoding

Files changed