git ssb - Improve fetch · cel/pull-git-remote-helper@92d069a0

Commit 92d069a064dddece82e3a8b59ed0e08336dcb001

Improve fetch

- Use repo.hasObjectQuick
- For getting objects, use set of haves rather than a single common hash
- Don't send objects that are reachable from objects the client already has

Charles Lehner committed on 10/11/2016, 2:09:14 AM
Parent: 9f6b7a97b920fe19a6c5b5cde35a13230b4e1b9a

Files changed

index.js

changed

index.jsView
		@@ -62,11 +62,8 @@
62	62 …	// references:
63	63 …	// git/documentation/technical/pack-protocol.txt
64	64 …	// git/documentation/technical/protocol-capabilities.txt
65	65 …	function uploadPack(read, repo, options) {
66		- /* multi_ack thin-pack side-band side-band-64k ofs-delta shallow no-progress
67		- * include-tag multi_ack_detailed
68		- * agent=git/2.7.0 */
69	66 …	var sendRefs = receivePackHeader([
70	67 …	agentCap,
71	68 …	], repo.refs(), repo.symrefs())
72	69 …
		@@ -75,9 +72,9 @@
75	72 …	verbosity: options.verbosity
76	73 …	})
77	74 …	var readWantHave = lines.haves()
78	75 …	var acked
79		- var commonHash
	76 …	+ var haves = {}
80	77 …	var sendPack
81	78 …	var wants = {}
82	79 …	var shallows = {}
83	80 …	var aborted
		@@ -114,14 +111,12 @@
114	111 …	function readHave(abort, cb) {
115	112 …	// Read upload haves (haves list).
116	113 …	// On first obj-id that we have, ACK
117	114 …	// If we have none, NAK.
118		- // TODO: implement multi_ack_detailed
119		- // FIXME!
120	115 …	if (abort) return
121	116 …	if (gotHaves) return cb(true)
122	117 …	readWantHave(null, function next(end, have) {
123		- if (end === true) {
	118 …	+ if (end === true) { // done
124	119 …	gotHaves = true
125	120 …	if (!acked) {
126	121 …	cb(null, 'NAK')
127	122 …	} else {
		@@ -137,19 +132,24 @@
137	132 …	} else if (end)
138	133 …	cb(end)
139	134 …	else if (have.type != 'have')
140	135 …	cb(new Error('Unknown have' + JSON.stringify(have)))
141		- else if (acked)
142		- readWantHave(null, next)
143		- else
144		- repo.hasObjectFromAny(have.hash, function (err, haveIt) {
145		- if (err) return cb(err)
146		- if (!haveIt)
147		- return readWantHave(null, next)
148		- commonHash = haveIt
149		- acked = true
150		- cb(null, 'ACK ' + have.hash)
151		- })
	136 …	+ else {
	137 …	+ haves[have.hash] = true
	138 …	+ if (acked) {
	139 …	+ readWantHave(null, next)
	140 …	+ } else if (repo.hasObjectQuick) {
	141 …	+ gotHasObject(null, repo.hasObjectQuick(have.hash))
	142 …	+ } else {
	143 …	+ repo.hasObjectFromAny(have.hash, gotHasObject)
	144 …	+ }
	145 …	+ }
	146 …	+ function gotHasObject(err, haveIt) {
	147 …	+ if (err) return cb(err)
	148 …	+ if (!haveIt) return readWantHave(null, next)
	149 …	+ acked = true
	150 …	+ cb(null, 'ACK ' + have.hash)
	151 …	+ }
152	152 …	})
153	153 …	}
154	154 …
155	155 …	function readPack(abort, cb) {
		@@ -157,12 +157,12 @@
157	157 …	if (sendPack) return sendPack(abort, cb)
158	158 …	// send pack file to client
159	159 …	if (!hasWants) return cb(true)
160	160 …	if (options.verbosity >= 2) {
161		- console.error('common', commonHash, 'wants', wants)
	161 …	+ console.error('wants', wants)
162	162 …	}
163	163 …	// TODO: show progress during getObjects
164		- getObjects(repo, commonHash, wants, shallows,
	164 …	+ getObjects(repo, wants, shallows, haves,
165	165 …	function (err, numObjects, readObjects) {
166	166 …	if (err) return cb(err)
167	167 …	// var progress = progressObjects(options)
168	168 …	// progress.setNumObjects(numObjects)
		@@ -224,46 +224,68 @@
224	224 …	}
225	225 …	return progress
226	226 …	}
227	227 …
228		-function getObjects(repo, commonHash, heads, shallows, cb) {
229		- // get objects from commonHash to each head, inclusive.
230		- // if commonHash is falsy, use root
	228 …	+function getObjects(repo, heads, shallows, haves, cb) {
	229 …	+ // get objects from heads back to haves
231	230 …	var objects = []
232	231 …	var objectsAdded = {}
	232 …	+ var exclude = {}
233	233 …	var done = multicb({pluck: 1})
234	234 …	var ended
235	235 …
236		- // walk back from heads until get to commonHash
	236 …	+ // walk back from heads until get to a have
237	237 …	for (var hash in heads)
238	238 …	addObject(hash, done())
239	239 …
240		- // TODO: only add new objects
241		-
242	240 …	function addObject(hash, cb) {
243	241 …	if (ended) return cb(ended)
244		- if (hash in objectsAdded \|\| hash == commonHash) return cb()
245		- objectsAdded[hash] = true
	242 …	+ if (hash in objectsAdded \|\| hash in exclude) return cb()
	243 …	+ if (hash in haves) return removeObject(hash, cb)
	244 …	+ var i = objectsAdded[hash] = objects.length++
246	245 …	repo.getObjectFromAny(hash, function (err, object) {
247	246 …	if (err) return cb(err)
	247 …	+ if (hash in exclude) return cb()
248	248 …	if (object.type == 'blob') {
249		- objects.push(object)
	249 …	+ objects[i] = object
250	250 …	cb()
251	251 …	} else {
252	252 …	// object must be read twice, so buffer it
253	253 …	bufferObject(object, function (err, object) {
254	254 …	if (err) return cb(err)
255		- objects.push(object)
256		- var hashes = getObjectLinks(object)
	255 …	+ objects[i] = object
	256 …	+ var hashes = getObjectLinks(object, true)
257	257 …	for (var sha1 in hashes)
258	258 …	addObject(sha1, done())
259	259 …	cb()
260	260 …	})
261	261 …	}
262	262 …	})
263	263 …	}
264	264 …
	265 …	+ function removeObject(hash, cb) {
	266 …	+ // when reach a have, get what trees/blobs it points to
	267 …	+ // and exclude them
	268 …	+ if (ended) return cb(ended)
	269 …	+ if (hash in exclude) return cb()
	270 …	+ exclude[hash] = true
	271 …	+ var i = objectsAdded[hash]
	272 …	+ if (i >= 0) objects[i] = null
	273 …	+ repo.getObjectFromAny(hash, function (err, object) {
	274 …	+ if (err) return cb(err)
	275 …	+ if (object.type === 'blob') cb()
	276 …	+ else bufferObject(object, function (err, object) {
	277 …	+ if (err) return cb(err)
	278 …	+ var hashes = getObjectLinks(object, false)
	279 …	+ for (var sha1 in hashes)
	280 …	+ removeObject(sha1, done())
	281 …	+ cb()
	282 …	+ })
	283 …	+ })
	284 …	+ }
	285 …	+
265	286 …	done(function (err) {
	287 …	+ objects = objects.filter(Boolean)
266	288 …	if (err) return cb(err)
267	289 …	// console.error(objects.reduce(function (n, obj) { return obj.length + n}, 0) + ' bytes')
268	290 …	cb(null, objects.length, pull.values(objects))
269	291 …	})
		@@ -285,17 +307,17 @@
285	307 …	)
286	308 …	}
287	309 …
288	310 …	// get hashes of git objects linked to from other git objects
289		-function getObjectLinks(object, cb) {
	311 …	+function getObjectLinks(object, includeParentCommits) {
290	312 …	switch (object.type) {
291	313 …	case 'blob':
292	314 …	return {}
293	315 …	case 'tree':
294	316 …	return getTreeLinks(object.data)
	317 …	+ case 'commit':
295	318 …	case 'tag':
296		- case 'commit':
297		- return getCommitOrTagLinks(object.data)
	319 …	+ return getCommitOrTagLinks(object.data, includeParentCommits)
298	320 …	}
299	321 …	}
300	322 …
301	323 …	function getTreeLinks(buf) {
		@@ -312,17 +334,18 @@
312	334 …	}
313	335 …	return links
314	336 …	}
315	337 …
316		-function getCommitOrTagLinks(buf) {
	338 …	+function getCommitOrTagLinks(buf, includeParents) {
317	339 …	var lines = buf.toString('utf8').split('\n')
318	340 …	var links = {}
319	341 …	// iterate until reach blank line (indicating start of commit/tag body)
320	342 …	for (var i = 0; lines[i]; i++) {
321	343 …	var args = lines[i].split(' ')
322	344 …	switch (args[0]) {
	345 …	+ case 'parent':
	346 …	+ if (!includeParents) continue
323	347 …	case 'tree':
324		- case 'parent':
325	348 …	case 'object':
326	349 …	var hash = args[1]
327	350 …	if (!(hash in links))
328	351 …	links[hash] = true
		@@ -330,13 +353,8 @@
330	353 …	}
331	354 …	return links
332	355 …	}
333	356 …
334		-/*
335		-TODO: investigate capabilities
336		-report-status delete-refs side-band-64k quiet atomic ofs-delta
337		-*/
338		-
339	357 …	// Get a line for each ref that we have. The first line also has capabilities.
340	358 …	// Wrap with pktLine.encode.
341	359 …	function receivePackHeader(capabilities, refSource, symrefs) {
342	360 …	var first = true

Built with git-ssb-web

cel / pull-git-remote-helper

Commit 92d069a064dddece82e3a8b59ed0e08336dcb001

Improve fetch

Files changed