Commit 04c77cc71b341054c04792a9bc6c74aa4c18d225
Start implementing pack negotiation and encoding
Charles Lehner committed on 2/8/2016, 4:37:24 AMParent: b1ed4631de0942640e4c7591c2be9dbdf8e98240
Files changed
index.js | changed |
pack.js | changed |
test/git-remote-full.js | changed |
test/run.js | changed |
util.js | changed |
index.js | ||
---|---|---|
@@ -70,30 +70,94 @@ | ||
70 | 70 | * agent=git/2.7.0 */ |
71 | 71 | var sendRefs = receivePackHeader([ |
72 | 72 | ], refSource, false) |
73 | 73 | |
74 | + var lines = packLineDecode(read, options) | |
75 | + // var havesSink = pull.drain(console.error.bind(console, 'have:')) | |
76 | + var readHave = lines.haves | |
77 | + var acked | |
78 | + var commonHash | |
79 | + var sendPack | |
80 | + | |
81 | + function haveObject(hash, cb) { | |
82 | + cb(/* TODO */) | |
83 | + } | |
84 | + | |
85 | + function getObjects(commonObjId, cb) { | |
86 | + console.error('get obj', commonObjId) | |
87 | + cb(null, 0, function readObject(end, cb) { | |
88 | + console.error('read obj', end) | |
89 | + // cb(new Error('Not implemented')) | |
90 | + cb(true) | |
91 | + }) | |
92 | + } | |
93 | + | |
94 | + // Packfile negotiation | |
74 | 95 | return packLineEncode( |
75 | 96 | cat([ |
76 | 97 | sendRefs, |
77 | 98 | pull.once(''), |
78 | 99 | function (abort, cb) { |
79 | 100 | if (abort) return |
80 | - // read client wants | |
81 | - var lines = packLineDecode(read, options) | |
101 | + if (acked) return cb(true) | |
102 | + // read upload request (wants list) from client | |
82 | 103 | pull( |
83 | 104 | lines.wants, |
84 | 105 | onThroughEnd(wantsDone), |
85 | 106 | wantSink |
86 | 107 | ) |
108 | + | |
87 | 109 | function wantsDone(err) { |
110 | + console.error('wants done', err) | |
88 | 111 | if (err) return cb(err) |
112 | + // Read upload haves (haves list). | |
113 | + // On first obj-id that we have, ACK | |
114 | + // If we have none, NAK. | |
115 | + // TODO: implement multi_ack_detailed | |
116 | + readHave(null, function next(end, have) { | |
117 | + ended = end | |
118 | + if (end === true) { | |
119 | + // found no common object | |
120 | + acked = true | |
121 | + cb(null, 'NAK') | |
122 | + } else if (end) | |
123 | + cb(end) | |
124 | + else if (have.type != 'have') | |
125 | + cb(new Error('Unknown have' + JSON.stringify(have))) | |
126 | + else | |
127 | + haveObject(have.hash, function (haveIt) { | |
128 | + if (!haveIt) | |
129 | + return readHave(null, next) | |
130 | + commonHash = haveIt | |
131 | + acked = true | |
132 | + cb(null, 'ACK ' + have.hash) | |
133 | + }) | |
134 | + }) | |
135 | + } | |
136 | + /* | |
137 | + function havesDone(err) { | |
138 | + console.error('haves done', err) | |
139 | + if (err) return cb(err) | |
140 | + cb(true) | |
89 | 141 | pull( |
90 | 142 | lines.passthrough, |
91 | 143 | pull.drain(function (buf) { |
92 | - console.error('got buf after wants', buf.toString('ascii')) | |
144 | + console.error('got buf after wants', buf.length, buf.toString('ascii')) | |
93 | 145 | }) |
94 | 146 | ) |
95 | 147 | } |
148 | + */ | |
149 | + }, | |
150 | + function next(abort, cb) { | |
151 | + if (abort) return cb(abort) | |
152 | + // send pack file to client | |
153 | + if (!sendPack) | |
154 | + getObjects(commonHash, function (err, numObjects, readObject) { | |
155 | + sendPack = pack.encode(numObjects, readObject) | |
156 | + next(abort, cb) | |
157 | + }) | |
158 | + else | |
159 | + sendPack(abort, cb) | |
96 | 160 | } |
97 | 161 | ]) |
98 | 162 | ) |
99 | 163 | } |
@@ -105,10 +169,18 @@ | ||
105 | 169 | read(end, function (end, data) { |
106 | 170 | if (ended = end) { |
107 | 171 | cb(end) |
108 | 172 | } else { |
109 | - var len = data ? data.length + 5 : 0 | |
110 | - cb(end, ('000' + len.toString(16)).substr(-4) + data + '\n') | |
173 | + // console.error("data", data) | |
174 | + if (data) | |
175 | + data += '\n' | |
176 | + else | |
177 | + data = '' | |
178 | + var len = data ? data.length + 4 : 0 | |
179 | + var hexLen = ('000' + len.toString(16)).substr(-4) | |
180 | + var pkt = hexLen + data | |
181 | + // console.error('>', JSON.stringify(pkt)) | |
182 | + cb(end, pkt) | |
111 | 183 | } |
112 | 184 | }) |
113 | 185 | } |
114 | 186 | } |
@@ -122,8 +194,9 @@ | ||
122 | 194 | var readPrefix = b.chunks(4) |
123 | 195 | var ended |
124 | 196 | |
125 | 197 | function readPackLine(abort, cb) { |
198 | + if (ended) return cb(ended) | |
126 | 199 | readPrefix(abort, function (end, buf) { |
127 | 200 | if (ended = end) return cb(end) |
128 | 201 | var len = parseInt(buf, 16) |
129 | 202 | if (!len) |
@@ -135,8 +208,21 @@ | ||
135 | 208 | }) |
136 | 209 | }) |
137 | 210 | } |
138 | 211 | |
212 | + function readPackLineStr(abort, cb) { | |
213 | + if (ended) return cb(ended) | |
214 | + readPackLine(abort, function (end, buf) { | |
215 | + if (ended = end) return cb(end) | |
216 | + // trim newline | |
217 | + var len = buf.length | |
218 | + if (buf[len - 1] == 0xa) | |
219 | + len-- | |
220 | + var line = buf.toString('ascii', 0, len) | |
221 | + cb(null, line) | |
222 | + }) | |
223 | + } | |
224 | + | |
139 | 225 | function readUpdate(abort, cb) { |
140 | 226 | readPackLine(abort, function (end, line) { |
141 | 227 | if (end) return cb(end) |
142 | 228 | if (options.verbosity >= 2) |
@@ -155,27 +241,50 @@ | ||
155 | 241 | }) |
156 | 242 | } |
157 | 243 | |
158 | 244 | function readWant(abort, cb) { |
245 | + readPackLineStr(abort, function (end, line) { | |
246 | + if (end) return cb(end) | |
247 | + if (options.verbosity >= 2) | |
248 | + console.error('line', line) | |
249 | + // if (!line.length) return cb(true) | |
250 | + if (!line.length || line == 'done') return cb(true) | |
251 | + var args = split3(line) | |
252 | + var caps = args[2] | |
253 | + if (caps && options.verbosity >= 2) | |
254 | + console.error('want capabilities:', caps) | |
255 | + cb(null, { | |
256 | + type: args[0], | |
257 | + hash: args[1], | |
258 | + }) | |
259 | + }) | |
260 | + } | |
261 | + | |
262 | + /* | |
263 | + function readWant(abort, cb) { | |
159 | 264 | readPackLine(abort, function (end, line) { |
160 | 265 | if (end) return cb(end) |
161 | 266 | if (options.verbosity >= 2) |
162 | 267 | console.error('line', line.toString('ascii')) |
163 | - if (!line.length || line == 'done') return cb(true) | |
268 | + if (!line.length || line == 'done') { | |
269 | + console.error('WANTS done', line, line.length) | |
270 | + return cb(true) | |
271 | + } | |
164 | 272 | var args = split3(line.toString('ascii')) |
165 | 273 | var caps = args[2] |
166 | 274 | if (caps && options.verbosity >= 2) |
167 | 275 | console.error('want capabilities:', caps) |
168 | 276 | cb(null, { |
169 | 277 | type: args[0], |
170 | - hash: args[1], | |
278 | + hash: args[1].replace(/\n$/, ''), | |
171 | 279 | }) |
172 | 280 | }) |
173 | 281 | } |
282 | + */ | |
174 | 283 | |
175 | 284 | b.packLines = readPackLine |
176 | 285 | b.updates = readUpdate |
177 | - b.wants = readWant | |
286 | + b.wants = b.haves = readWant | |
178 | 287 | |
179 | 288 | return b |
180 | 289 | } |
181 | 290 |
pack.js | ||
---|---|---|
@@ -2,11 +2,15 @@ | ||
2 | 2 | var pull = require('pull-stream') |
3 | 3 | var toPull = require('stream-to-pull-stream') |
4 | 4 | var Inflate = require('pako/lib/inflate').Inflate |
5 | 5 | var createHash = require('./util').createHash |
6 | +var cat = require('pull-cat') | |
6 | 7 | |
7 | 8 | exports.decode = decodePack |
9 | +exports.encode = encodePack | |
8 | 10 | |
11 | +var PACK_VERSION = 2 | |
12 | + | |
9 | 13 | var objectTypes = [ |
10 | 14 | 'none', 'commit', 'tree', 'blob', |
11 | 15 | 'tag', 'unused', 'ofs-delta', 'ref-delta' |
12 | 16 | ] |
@@ -170,4 +174,29 @@ | ||
170 | 174 | } |
171 | 175 | |
172 | 176 | return readObject |
173 | 177 | } |
178 | + | |
179 | +function encodePack(numObjects, readObject) { | |
180 | + var ended | |
181 | + var header = new Buffer(12) | |
182 | + header.write('PACK') | |
183 | + header.writeUInt32BE(PACK_VERSION, 4) | |
184 | + header.writeUInt32BE(numObjects, 8) | |
185 | + var checksum = createHash('sha1') | |
186 | + | |
187 | + return pull.through(function (data) { | |
188 | + console.error('> ' + data.length, data.toString()) | |
189 | + })(cat([ | |
190 | + checksum(cat([ | |
191 | + pull.once(header), | |
192 | + function (abort, cb) { | |
193 | + if (ended) return cb(ended) | |
194 | + readObject(abort, function (end, type, length, read) { | |
195 | + if (ended = end) return cb(end) | |
196 | + console.error('TODO: encode object') | |
197 | + }) | |
198 | + } | |
199 | + ])), | |
200 | + checksum.readDigest | |
201 | + ])) | |
202 | +} |
test/git-remote-full.js | ||
---|---|---|
@@ -21,9 +21,8 @@ | ||
21 | 21 | require('../')({ |
22 | 22 | prefix: 'foo', |
23 | 23 | refSource: pull.values(refs), |
24 | 24 | wantSink: pull.drain(function (want) { |
25 | - console.error('got want', want) | |
26 | 25 | process.send({want: want}) |
27 | 26 | }), |
28 | 27 | }), |
29 | 28 | toPull(process.stdout, function (err) { |
test/run.js | ||
---|---|---|
@@ -20,8 +20,12 @@ | ||
20 | 20 | full: 'full.js://' |
21 | 21 | } |
22 | 22 | |
23 | 23 | var tmpDir = mktemp.createDirSync(path.join(require('os').tmpdir(), 'XXXXXXX')) |
24 | +tape.onFinish(function () { | |
25 | + if (tmpDir) | |
26 | + rimraf.sync(tmpDir) | |
27 | +}) | |
24 | 28 | |
25 | 29 | function handleIpcMessage(t, cb) { |
26 | 30 | return function (msg) { |
27 | 31 | if (msg.error) { |
@@ -149,17 +153,30 @@ | ||
149 | 153 | }) |
150 | 154 | }) |
151 | 155 | }) |
152 | 156 | |
153 | -tape('fetch', function (t) { | |
157 | +0 && | |
158 | +tape('fetch when already up-to-date', function (t) { | |
154 | 159 | t.git('fetch', '-vv', remote.full, function (msg) { |
155 | - t.notOk('should not get a message here', msg) | |
160 | + t.notOk(msg, 'should not get a message here') | |
156 | 161 | }, function (code) { |
157 | 162 | t.equals(code, 0, 'fetched') |
158 | 163 | t.end() |
159 | 164 | }) |
160 | 165 | }) |
161 | 166 | |
162 | -tape.onFinish(function () { | |
163 | - if (tmpDir) | |
164 | - rimraf.sync(tmpDir) | |
167 | +0 && | |
168 | +tape('clone into new dir', function (t) { | |
169 | + var dir = path.join(tmpDir, 'clonedir') | |
170 | + t.plan(3) | |
171 | + t.git('clone', '-vv', remote.full, dir, function (msg) { | |
172 | + if (msg.want) | |
173 | + t.deepEquals(msg.want, { | |
174 | + type: 'want', | |
175 | + hash: 'edb5b50e8019797925820007d318870f8c346726' | |
176 | + }, 'got want') | |
177 | + else | |
178 | + t.notOk(msg, 'unexpected message') | |
179 | + }, function (code) { | |
180 | + t.equals(code, 0, 'cloned') | |
181 | + }) | |
165 | 182 | }) |
util.js | ||
---|---|---|
@@ -3,10 +3,15 @@ | ||
3 | 3 | |
4 | 4 | exports.createHash = function (type) { |
5 | 5 | var hash = crypto.createHash(type) |
6 | 6 | var hasher = pull.through(hash.update.bind(hash)) |
7 | + var digest | |
7 | 8 | hasher.hash = hash |
8 | 9 | hasher.digest = hash.digest.bind(hash) |
10 | + hasher.readDigest = function (abort, cb) { | |
11 | + if (digest) cb(true) | |
12 | + else cb(null, digest = hash.digest()) | |
13 | + } | |
9 | 14 | return hasher |
10 | 15 | } |
11 | 16 | |
12 | 17 | exports.createGitObjectHash = function (objectType, objectLength) { |
Built with git-ssb-web