Commit 8063e81512342ad27a3e38e5e3fbbda334b3669b
updated to use new dag
Signed-off-by: wanderer <mjbecze@gmail.com>wanderer committed on 10/1/2017, 6:09:38 AM
Parent: 1deb40ab1712a608872e7ad75b4b5393128c4181
Files changed
benchmark/ethereum.js | changed |
benchmark/package.json | changed |
benchmark/radixTree.js | changed |
index.js | changed |
package.json | changed |
tests/index.js | changed |
dag.js | added |
treeNode.js | added |
benchmark/ethereum.js | |||
---|---|---|---|
@@ -1,36 +1,69 @@ | |||
1 | 1 … | var Trie = require('merkle-patricia-tree') | |
2 | 2 … | const crypto = require('crypto') | |
3 | -const rlp = require('rlp') | ||
3 … | +// const rlp = require('rlp') | ||
4 … | +const level = require('level') | ||
5 … | +const db = level('./eth-testdb') | ||
4 | 6 … | ||
5 | -const trie = new Trie() | ||
7 … | +db._get = db.get | ||
8 … | +let lookups = 0 | ||
9 … | +db.get = function (key, opts, cb) { | ||
10 … | + lookups++ | ||
11 … | + return db._get(key, opts, cb) | ||
12 … | +} | ||
6 | 13 … | ||
14 … | +let trie = new Trie(db) | ||
15 … | + | ||
7 | 16 … | const entries = 100000 | |
8 | 17 … | console.log('entries', entries) | |
9 | 18 … | ||
10 | 19 … | async function run () { | |
20 … | + | ||
21 … | + let start = new Date() | ||
22 … | + let hrstart = process.hrtime() | ||
23 … | + | ||
11 | 24 … | for (let i = 0; i < entries; i++) { | |
12 | 25 … | const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
13 | 26 … | await new Promise((resolve, reject) => { | |
14 | 27 … | trie.put(key, i, resolve) | |
15 | 28 … | }) | |
16 | 29 … | } | |
17 | 30 … | ||
31 … | + let end = new Date() - start | ||
32 … | + let hrend = process.hrtime(hrstart) | ||
33 … | + | ||
34 … | + console.info('state root creation time: %dms', end) | ||
35 … | + console.info('state root creation time (hr): %ds %dms', hrend[0], hrend[1] / 1000000) | ||
36 … | + | ||
37 … | + start = new Date() | ||
38 … | + hrstart = process.hrtime() | ||
39 … | + | ||
40 … | + console.log('root', trie.root.toString('hex')) | ||
41 … | + | ||
18 | 42 … | let proofSize = 0 | |
19 | 43 … | for (let i = 0; i < entries; i++) { | |
20 | 44 … | const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
45 … | + trie = new Trie(db, trie.root) | ||
21 | 46 … | const promise = new Promise((resolve, reject) => { | |
22 | - trie.findPath(key, (err, node, remainder, stack) => { | ||
23 | - let proof = stack.map(el => { | ||
24 | - return el.raw | ||
25 | - }) | ||
26 | - let encoded = rlp.encode(proof) | ||
27 | - proofSize += encoded.length | ||
47 … | + trie.get(key, (err, value) => { | ||
48 … | + // console.log(value) | ||
49 … | + // let proof = stack.map(el => { | ||
50 … | + // return el.raw | ||
51 … | + // }) | ||
52 … | + // let encoded = rlp.encode(proof) | ||
53 … | + // proofSize += encoded.length | ||
28 | 54 … | resolve() | |
29 | 55 … | }) | |
30 | 56 … | }) | |
31 | 57 … | await promise | |
32 | 58 … | } | |
33 | - console.log('rlp size', proofSize / entries) | ||
59 … | + | ||
60 … | + end = new Date() - start | ||
61 … | + hrend = process.hrtime(hrstart) | ||
62 … | + | ||
63 … | + console.info('read time: %dms', end) | ||
64 … | + console.info('read time (hr): %ds %dms', hrend[0], hrend[1] / 1000000) | ||
65 … | + console.info('db lookups', lookups) | ||
66 … | + // console.log('rlp size', proofSize / entries) | ||
34 | 67 … | } | |
35 | 68 … | ||
36 | 69 … | run() |
benchmark/package.json | ||
---|---|---|
@@ -8,8 +8,9 @@ | ||
8 | 8 … | }, |
9 | 9 … | "author": "", |
10 | 10 … | "license": "ISC", |
11 | 11 … | "dependencies": { |
12 … | + "bn.js": "^4.11.8", | |
12 | 13 … | "borc": "^2.0.2", |
13 | 14 … | "lzma": "^2.3.2", |
14 | 15 … | "merkle-patricia-tree": "^2.2.0" |
15 | 16 … | } |
benchmark/radixTree.js | ||
---|---|---|
@@ -1,45 +1,49 @@ | ||
1 | -const IPFS = require('ipfs') | |
2 | 1 … | const crypto = require('crypto') |
3 | 2 … | const RadixTree = require('../') |
4 | 3 … | const cbor = require('borc') |
5 | 4 … | const zlib = require('zlib') |
6 | 5 … | |
7 | -// start ipfs | |
8 | -const node = new IPFS({ | |
9 | - start: false, | |
10 | - repo: './ipfs-repo' | |
11 | -}) | |
6 … | +const level = require('level') | |
7 … | +const db = level('./testdb') | |
8 … | +const DAG = require('../dag.js') | |
9 … | +const proof = require('../proofStore.js') | |
12 | 10 … | |
13 | -node.on('ready', async () => { | |
11 … | +const dag = new DAG(db) | |
12 … | + | |
13 … | +async function main () { | |
14 | 14 … | const tree = new RadixTree({ |
15 | - dag: node.dag, | |
15 … | + dag: dag, | |
16 | 16 … | // root: { '/': 'zdpuArkpWFfw49S1tNLY26YNkHCoKt2CG7rJ6iCaqkcwsGqH7' } |
17 | 17 … | }) |
18 | 18 … | |
19 | - const entries = 10000 // 5117051 | |
19 … | + const entries = 100000 // 5117051 | |
20 | 20 … | console.log('entries', entries) |
21 | 21 … | for (let i = 0; i < entries; i++) { |
22 | 22 … | const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) |
23 | 23 … | await tree.set(key, i) |
24 | 24 … | } |
25 | 25 … | console.log('flushing') |
26 | 26 … | const sr = await tree.flush() |
27 | - console.log('done', sr) | |
27 … | + console.log('done', sr['/'].toString('hex')) | |
28 | 28 … | |
29 | 29 … | let proofSize = 0 |
30 … | + let binaryProofSize = 0 | |
30 | 31 … | let compressedSize = 0 |
31 | 32 … | |
32 | 33 … | for (let i = 0; i < entries; i++) { |
33 | 34 … | const tree = new RadixTree({ |
34 | - dag: node.dag, | |
35 … | + dag: dag, | |
35 | 36 … | root: {'/': sr['/']} |
36 | 37 … | }) |
37 | 38 … | const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) |
38 | 39 … | await tree.get(key) |
39 | - // console.log(JSON.stringify(tree.root, null, 2)) | |
40 | 40 … | const encoded = cbor.encode(tree.root) |
41 | 41 … | proofSize += encoded.length |
42 … | + const binary = proof.encodeProof(tree.root['/']) | |
43 … | + // console.log(JSON.stringify(tree.root, null, 2)) | |
44 … | + // console.log(binary) | |
45 … | + binaryProofSize += binary.length | |
42 | 46 … | |
43 | 47 … | const gzip = zlib.createGzip() |
44 | 48 … | gzip.on('data', (data) => { |
45 | 49 … | compressedSize += data.length |
@@ -52,10 +56,12 @@ | ||
52 | 56 … | gzip.end(encoded) |
53 | 57 … | await promise |
54 | 58 … | } |
55 | 59 … | console.log('cbor size', proofSize / entries) |
56 | - console.log('compressed size', compressedSize / entries) | |
57 | -}) | |
60 … | + console.log('cbor compressed size', compressedSize / entries) | |
61 … | + console.log('binary', binaryProofSize / entries) | |
62 … | +} | |
63 … | +main() | |
58 | 64 … | |
59 | 65 … | // if (i % 10000 === 0) { |
60 | 66 … | // console.log(i) |
61 | 67 … | // console.log(JSON.stringify(tree.root, null, 2)) |
index.js | ||
---|---|---|
@@ -1,28 +1,33 @@ | ||
1 | 1 … | const Graph = require('ipld-graph-builder') |
2 | -const Buffer = require('safe-buffer').Buffer | |
3 | 2 … | const Uint1Array = require('uint1array') |
4 | 3 … | const TextEncoder = require('text-encoding').TextEncoder |
4 … | +const DAG = require('./dag.js') | |
5 … | +const treeNode = require('./treeNode.js') | |
5 | 6 … | |
6 | 7 … | const encoder = new TextEncoder('utf-8') |
7 | 8 … | |
8 | -const RadixTree = module.exports = class RadixTree { | |
9 … | +module.exports = class RadixTree { | |
9 | 10 … | /** |
10 | 11 … | * @param opts |
11 | 12 … | * @param opts.root {object} a merkle root to a radix tree. If none, RadixTree will create an new root. |
12 | 13 … | * @param opts.graph {object} an instance of [ipld-graph-builder](https://github.com/ipld/js-ipld-graph-builder) alternitvly `opts.dag` can be used |
13 | 14 … | * @param opts.dag {object} an instance if [ipfs.dag](https://github.com/ipfs/js-ipfs#dag). If there is no `opts.graph` this will be used to create a new graph instance. |
14 | 15 … | */ |
15 | 16 … | constructor (opts) { |
16 | - this.root = opts.root || {'/': RadixTree.emptyTreeState} | |
17 | - this.graph = opts.graph || new Graph(opts.dag) | |
17 … | + this.root = opts.root || { | |
18 … | + '/': RadixTree.emptyTreeState | |
19 … | + } | |
20 … | + | |
21 … | + this.dag = opts.dag || new DAG(opts.db) | |
22 … | + this.graph = opts.graph || new Graph(this.dag) | |
18 | 23 … | } |
19 | 24 … | |
20 | 25 … | /** |
21 | 26 … | * returns the state of an empty tree |
22 | 27 … | */ |
23 | 28 … | static get emptyTreeState () { |
24 | - return [null, null] | |
29 … | + return [null, null, null] | |
25 | 30 … | } |
26 | 31 … | |
27 | 32 … | /** |
28 | 33 … | * returns an Uint1Array constructir which is used to repersent keys |
@@ -31,25 +36,16 @@ | ||
31 | 36 … | static get ArrayConstructor () { |
32 | 37 … | return Uint1Array |
33 | 38 … | } |
34 | 39 … | |
35 | - /** | |
36 | - * converts a TypedArray or Buffer to an Uint1Array | |
37 | - * @param {TypedArray} array - the array to convert | |
38 | - * @returns {TypedArray} | |
39 | - */ | |
40 | - static toTypedArray (array) { | |
41 | - return new RadixTree.ArrayConstructor(new Uint8Array(array).buffer) | |
42 | - } | |
43 | - | |
44 | 40 … | async _get (key) { |
45 | 41 … | let index = 0 |
46 | 42 … | let root = this.root |
47 | 43 … | let parent |
48 | 44 … | |
49 | 45 … | while (1) { |
50 | 46 … | // load the root |
51 | - const exNode = await this.graph.get(root, EXTENSION, true) | |
47 … | + const exNode = await this.graph.get(root, treeNode.EXTENSION, true) | |
52 | 48 … | if (exNode) { |
53 | 49 … | let subKey = key.subarray(index) |
54 | 50 … | |
55 | 51 … | // let extensionIndex = 0 |
@@ -73,9 +69,9 @@ | ||
73 | 69 … | } |
74 | 70 … | |
75 | 71 … | let keySegment = key[index] |
76 | 72 … | if (keySegment !== undefined) { |
77 | - const branch = getBranch(root) | |
73 … | + const branch = treeNode.getBranch(root) | |
78 | 74 … | await this.graph.get(branch, keySegment, true) |
79 | 75 … | // preseves the '/' |
80 | 76 … | const nextRoot = branch[keySegment] |
81 | 77 … | if (!nextRoot) { |
@@ -93,12 +89,12 @@ | ||
93 | 89 … | |
94 | 90 … | index++ |
95 | 91 … | } |
96 | 92 … | |
97 | - let value = getValue(root) | |
93 … | + let value = treeNode.getValue(root) | |
98 | 94 … | |
99 | 95 … | if (value && value['/']) { |
100 | - value = await this.graph.get(root, VALUE, true) | |
96 … | + value = await this.graph.get(root, treeNode.VALUE, true) | |
101 | 97 … | } |
102 | 98 … | |
103 | 99 … | return { |
104 | 100 … | value: value, |
@@ -126,27 +122,23 @@ | ||
126 | 122 … | */ |
127 | 123 … | async set (key, value) { |
128 | 124 … | key = RadixTree.formatKey(key) |
129 | 125 … | |
130 | - if (value.length > 32) { | |
131 | - value = {'/': value} | |
132 | - } | |
133 | - | |
134 | - if (isEmpty(this.root)) { | |
126 … | + if (treeNode.isEmpty(this.root)) { | |
135 | 127 … | this.root['/'] = createNode(key, [null, null], value)['/'] |
136 | 128 … | } else { |
137 | 129 … | let {root, extensionIndex, extension, index, value: rValue} = await this._get(key) |
138 | 130 … | |
139 | 131 … | if (rValue) { |
140 | - setValue(root, value) | |
132 … | + treeNode.setValue(root, value) | |
141 | 133 … | } else { |
142 | 134 … | if (extensionIndex !== undefined) { |
143 | 135 … | // split the extension node in two |
144 | 136 … | const extensionKey = extension[extensionIndex] |
145 | 137 … | const remExtension = extension.subarray(extensionIndex + 1) |
146 | 138 … | extension = extension.subarray(0, extensionIndex) |
147 | 139 … | |
148 | - setExtension(root, remExtension) | |
140 … | + treeNode.setExtension(root, remExtension) | |
149 | 141 … | const branch = [null, null] |
150 | 142 … | branch[extensionKey] = {'/': root['/']} |
151 | 143 … | root['/'] = createNode(extension, branch)['/'] |
152 | 144 … | } |
@@ -155,13 +147,13 @@ | ||
155 | 147 … | if (index < key.length) { |
156 | 148 … | const keySegment = key[index] |
157 | 149 … | const extension = key.subarray(index + 1, key.length) |
158 | 150 … | const newNode = createNode(extension, [null, null], value) |
159 | - const rootBranch = getBranch(root) | |
151 … | + const rootBranch = treeNode.getBranch(root) | |
160 | 152 … | rootBranch[keySegment] = newNode |
161 | - setBranch(root, rootBranch) | |
153 … | + treeNode.setBranch(root, rootBranch) | |
162 | 154 … | } else { |
163 | - setValue(root, value) | |
155 … | + treeNode.setValue(root, value) | |
164 | 156 … | } |
165 | 157 … | } |
166 | 158 … | } |
167 | 159 … | } |
@@ -177,31 +169,31 @@ | ||
177 | 169 … | if (results.value !== undefined) { |
178 | 170 … | const root = results.root |
179 | 171 … | const parent = results.parent |
180 | 172 … | |
181 | - deleteValue(root) | |
173 … | + treeNode.deleteValue(root) | |
182 | 174 … | |
183 | - const branch = getBranch(root) | |
175 … | + const branch = treeNode.getBranch(root) | |
184 | 176 … | if (branch.some(el => el !== null)) { |
185 | 177 … | joinNodes(root) |
186 | 178 … | } else { |
187 | 179 … | if (!parent) { |
188 | 180 … | root['/'] = RadixTree.emptyTreeState |
189 | 181 … | } else { |
190 | - let branch = getBranch(parent) | |
182 … | + let branch = treeNode.getBranch(parent) | |
191 | 183 … | branch = branch.map(node => node === root ? null : node) |
192 | - setBranch(parent, branch) | |
184 … | + treeNode.setBranch(parent, branch) | |
193 | 185 … | await this.graph.tree(parent, 2) |
194 | 186 … | |
195 | 187 … | joinNodes(parent) |
196 | 188 … | } |
197 | 189 … | } |
198 | 190 … | } |
199 | 191 … | |
200 | 192 … | function joinNodes (root) { |
201 | - if (getValue(root) === undefined) { | |
193 … | + if (treeNode.getValue(root) === undefined) { | |
202 | 194 … | let index |
203 | - const branch = getBranch(root) | |
195 … | + const branch = treeNode.getBranch(root) | |
204 | 196 … | const nodes = branch.filter((node, i) => { |
205 | 197 … | if (node) { |
206 | 198 … | index = i |
207 | 199 … | return true |
@@ -209,17 +201,17 @@ | ||
209 | 201 … | }) |
210 | 202 … | |
211 | 203 … | if (nodes.length === 1) { |
212 | 204 … | const child = nodes[0] |
213 | - const pExtension = getExtension(root) | |
214 | - const childExtension = getExtension(child) | |
205 … | + const pExtension = treeNode.getExtension(root) | |
206 … | + const childExtension = treeNode.getExtension(child) | |
215 | 207 … | const newExtension = new RadixTree.ArrayConstructor(pExtension.length + childExtension.length + 1) |
216 | 208 … | |
217 | 209 … | newExtension.set(pExtension) |
218 | 210 … | newExtension[pExtension.length] = index |
219 | 211 … | newExtension.set(childExtension, pExtension.length + 1) |
220 | 212 … | |
221 | - setExtension(child, newExtension) | |
213 … | + treeNode.setExtension(child, newExtension) | |
222 | 214 … | root['/'] = child['/'] |
223 | 215 … | } |
224 | 216 … | } |
225 | 217 … | } |
@@ -250,75 +242,18 @@ | ||
250 | 242 … | const node = { |
251 | 243 … | '/': [] |
252 | 244 … | } |
253 | 245 … | |
254 | - setValue(node, value) | |
255 | - setExtension(node, ex) | |
256 | - setBranch(node, branch) | |
246 … | + treeNode.setValue(node, value) | |
247 … | + treeNode.setExtension(node, ex) | |
248 … | + treeNode.setBranch(node, branch) | |
257 | 249 … | |
258 | 250 … | return node |
259 | 251 … | } |
260 | 252 … | |
261 | -// helper functions for nodes | |
262 | -const LBRANCH = 0 | |
263 | -const RBRANCH = 1 | |
264 | -const EXTENSION = 2 | |
265 | -const VALUE = 3 | |
266 | - | |
267 | -function setBranch (node, branch) { | |
268 | - node['/'][LBRANCH] = branch[0] | |
269 | - node['/'][RBRANCH] = branch[1] | |
270 | -} | |
271 | - | |
272 | -function getBranch (node) { | |
273 | - return node['/'].slice(LBRANCH, 2) | |
274 | -} | |
275 | - | |
276 | -function getValue (node) { | |
277 | - return node['/'][VALUE] | |
278 | -} | |
279 | - | |
280 | -function deleteValue (node) { | |
281 | - node['/'].pop() | |
282 | -} | |
283 | - | |
284 | -function getExtension (node) { | |
285 | - if (node['/'][EXTENSION]) { | |
286 | - const len = node['/'][EXTENSION][0] | |
287 | - const extension = RadixTree.toTypedArray(node['/'][EXTENSION].subarray(1)) | |
288 | - return extension.subarray(0, extension.length - len) | |
289 | - } else { | |
290 | - return [] | |
291 | - } | |
292 | -} | |
293 | - | |
294 | -function setExtension (node, ex) { | |
295 | - if (ex && ex.length) { | |
296 | - const paddingLen = ((8 - (ex.length % 8)) % 8) | |
297 | - node['/'][EXTENSION] = Buffer.concat([Buffer.from([paddingLen]), Buffer.from(ex.buffer)]) | |
298 | - } else { | |
299 | - if (getValue(node) === undefined && !Array.isArray(node['/'][EXTENSION])) { | |
300 | - node['/'].pop() | |
301 | - } else if (node['/'][EXTENSION] !== undefined) { | |
302 | - node['/'][EXTENSION] = null | |
303 | - } | |
304 | - } | |
305 | -} | |
306 | - | |
307 | -function setValue (node, val) { | |
308 | - if (val !== undefined) { | |
309 | - node['/'][VALUE] = val | |
310 | - } | |
311 | -} | |
312 | - | |
313 | -function isEmpty (node) { | |
314 | - const branch = getBranch(node) | |
315 | - return node['/'].length === 2 && branch[0] === null && branch[1] === null | |
316 | -} | |
317 | - | |
318 | 253 … | function findMatchBits (key, node) { |
319 | 254 … | let extensionIndex = 0 |
320 | - const extension = getExtension(node) | |
255 … | + const extension = treeNode.getExtension(node) | |
321 | 256 … | const extensionLen = extension.length |
322 | 257 … | |
323 | 258 … | // checks the extension against the key |
324 | 259 … | while (extensionIndex < extensionLen && extension[extensionIndex] === key[extensionIndex]) { |
package.json | ||
---|---|---|
@@ -25,9 +25,12 @@ | ||
25 | 25 … | "standard": "^10.0.0", |
26 | 26 … | "tape": "^4.6.3" |
27 | 27 … | }, |
28 | 28 … | "dependencies": { |
29 … | + "borc": "^2.0.2", | |
29 | 30 … | "ipld-graph-builder": "^1.2.4", |
31 … | + "leb128": "0.0.2", | |
32 … | + "level": "^1.7.0", | |
30 | 33 … | "safe-buffer": "^5.1.1", |
31 | 34 … | "text-encoding": "^0.6.4", |
32 | 35 … | "uint1array": "^1.0.5" |
33 | 36 … | }, |
tests/index.js | ||
---|---|---|
@@ -1,155 +1,150 @@ | ||
1 | 1 … | const tape = require('tape') |
2 | 2 … | const crypto = require('crypto') |
3 | -const IPFS = require('ipfs') | |
3 … | +const level = require('level') | |
4 | 4 … | const RadixTree = require('../') |
5 … | +const db = level('./testdb') | |
5 | 6 … | |
6 | -// start ipfs | |
7 | -const node = new IPFS({ | |
8 | - start: false | |
9 | -}) | |
7 … | +tape('set and get', async t => { | |
8 … | + let tree = new RadixTree({ | |
9 … | + db: db | |
10 … | + }) | |
10 | 11 … | |
11 | -node.on('ready', () => { | |
12 | - tape('set and get', async t => { | |
13 | - let tree = new RadixTree({ | |
14 | - dag: node.dag | |
15 | - }) | |
12 … | + await tree.set('test', Buffer.from('cat')) | |
13 … | + let val = await tree.get('test') | |
14 … | + t.equals(val.toString(), 'cat') | |
15 … | + await tree.set('te', Buffer.from('blop')) | |
16 … | + val = await tree.get('test') | |
17 … | + t.equals(val.toString(), 'cat') | |
16 | 18 … | |
17 | - await tree.set('test', 'cat') | |
18 | - let val = await tree.get('test') | |
19 | - t.equals(val, 'cat') | |
20 | - await tree.set('te', 'blop') | |
21 | - val = await tree.get('test') | |
22 | - t.equals(val, 'cat') | |
19 … | + val = await tree.get('te') | |
20 … | + t.equals(val.toString(), 'blop') | |
23 | 21 … | |
24 | - val = await tree.get('te') | |
25 | - t.equals(val, 'blop') | |
22 … | + await tree.set('rad', Buffer.from('cat2')) | |
26 | 23 … | |
27 | - await tree.set('rad', 'cat2') | |
24 … | + val = await tree.get('rad') | |
25 … | + t.equals(val.toString(), 'cat2') | |
28 | 26 … | |
29 | - val = await tree.get('rad') | |
30 | - t.equals(val, 'cat2') | |
27 … | + await tree.set('test', Buffer.from('cat111')) | |
28 … | + val = await tree.get('test') | |
29 … | + t.equals(val.toString(), 'cat111') | |
31 | 30 … | |
32 | - await tree.set('test', 'cat111') | |
33 | - val = await tree.get('test') | |
34 | - t.equals(val, 'cat111') | |
31 … | + const stateRoot = await tree.flush() | |
35 | 32 … | |
36 | - const stateRoot = await tree.flush() | |
33 … | + // try reteriving node from ipfs | |
34 … | + tree = new RadixTree({ | |
35 … | + db: db, | |
36 … | + root: stateRoot | |
37 … | + }) | |
37 | 38 … | |
38 | - // try reteriving node from ipfs | |
39 | - tree = new RadixTree({ | |
40 | - dag: node.dag, | |
41 | - root: stateRoot | |
42 | - }) | |
39 … | + val = await tree.get('te') | |
40 … | + t.equals(val.toString(), 'blop') | |
43 | 41 … | |
44 | - val = await tree.get('te') | |
45 | - t.equals(val, 'blop') | |
42 … | + val = await tree.get('rad') | |
43 … | + t.equals(val.toString(), 'cat2') | |
46 | 44 … | |
47 | - val = await tree.get('rad') | |
48 | - t.equals(val, 'cat2') | |
45 … | + val = await tree.get('test') | |
46 … | + t.equals(val.toString(), 'cat111') | |
47 … | + // console.log(JSON.stringify(tree.root, null, 2)) | |
49 | 48 … | |
50 | - val = await tree.get('test') | |
51 | - t.equals(val, 'cat111') | |
52 | - // console.log(JSON.stringify(tree.root, null, 2)) | |
49 … | + t.end() | |
50 … | +}) | |
53 | 51 … | |
54 | - t.end() | |
52 … | +tape('branch nodes', async t => { | |
53 … | + const tree = new RadixTree({ | |
54 … | + db: db | |
55 | 55 … | }) |
56 | 56 … | |
57 | - tape('branch nodes', async t => { | |
58 | - const tree = new RadixTree({ | |
59 | - dag: node.dag | |
60 | - }) | |
57 … | + let key0 = new RadixTree.ArrayConstructor([1, 1]) | |
58 … | + await tree.set(key0, Buffer.from('cat')) | |
59 … | + let key1 = new RadixTree.ArrayConstructor([0, 1]) | |
60 … | + await tree.set(key1, Buffer.from('cat2')) | |
61 | 61 … | |
62 | - let key0 = new RadixTree.ArrayConstructor([1, 1]) | |
63 | - await tree.set(key0, 'cat') | |
64 | - let key1 = new RadixTree.ArrayConstructor([0, 1]) | |
65 | - await tree.set(key1, 'cat2') | |
62 … | + let key2 = new RadixTree.ArrayConstructor([1, 0]) | |
63 … | + await tree.set(key2, Buffer.from('cat')) | |
64 … | + let key3 = new RadixTree.ArrayConstructor([0, 0]) | |
65 … | + await tree.set(key3, Buffer.from('cat3')) | |
66 | 66 … | |
67 | - let key2 = new RadixTree.ArrayConstructor([1, 0]) | |
68 | - await tree.set(key2, 'cat') | |
69 | - let key3 = new RadixTree.ArrayConstructor([0, 0]) | |
70 | - await tree.set(key3, 'cat3') | |
67 … | + let val = await tree.get(key0) | |
68 … | + t.equals(val.toString(), 'cat') | |
69 … | + val = await tree.get(key1) | |
70 … | + t.equals(val.toString(), 'cat2') | |
71 … | + val = await tree.get(key2) | |
72 … | + t.equals(val.toString(), 'cat') | |
73 … | + val = await tree.get(key3) | |
74 … | + t.equals(val.toString(), 'cat3') | |
71 | 75 … | |
72 | - let val = await tree.get(key0) | |
73 | - t.equals(val, 'cat') | |
74 | - val = await tree.get(key1) | |
75 | - t.equals(val, 'cat2') | |
76 | - val = await tree.get(key2) | |
77 | - t.equals(val, 'cat') | |
78 | - val = await tree.get(key3) | |
79 | - t.equals(val, 'cat3') | |
76 … | + t.end() | |
77 … | +}) | |
80 | 78 … | |
81 | - t.end() | |
79 … | +tape('delete', async t => { | |
80 … | + const tree = new RadixTree({ | |
81 … | + db: db | |
82 | 82 … | }) |
83 … | + try { | |
84 … | + await tree.set('test', Buffer.from('cat')) | |
85 … | + await tree.set('ter', Buffer.from('cat3')) | |
86 … | + await tree.delete('te') | |
87 … | + await tree.delete('test') | |
88 … | + await tree.delete('ter') | |
89 … | + t.deepEquals(tree.root['/'], RadixTree.emptyTreeState) | |
83 | 90 … | |
84 | - tape('delete', async t => { | |
85 | - const tree = new RadixTree({ | |
86 | - dag: node.dag | |
87 | - }) | |
88 | - try { | |
89 | - await tree.set('test', 'cat') | |
90 | - await tree.set('ter', 'cat3') | |
91 | - await tree.delete('te') | |
92 | - await tree.delete('test') | |
93 | - await tree.delete('ter') | |
94 | - t.deepEquals(tree.root['/'], [null, null]) | |
91 … | + // tests delete midle branchs | |
92 … | + await tree.set('test', Buffer.from('cat')) | |
93 … | + await tree.set('te', Buffer.from('cat2')) | |
94 … | + await tree.set('ter', Buffer.from('cat3')) | |
95 … | + await tree.delete('te') | |
96 … | + let val = await tree.get('test') | |
97 … | + t.equals(val.toString(), 'cat') | |
95 | 98 … | |
96 | - // tests delete midle branchs | |
97 | - await tree.set('test', 'cat') | |
98 | - await tree.set('te', 'cat2') | |
99 | - await tree.set('ter', 'cat3') | |
100 | - await tree.delete('te') | |
101 | - let val = await tree.get('test') | |
102 | - t.equals(val, 'cat') | |
99 … | + // tests delete end branchs | |
100 … | + await tree.set('te', 'cat2') | |
101 … | + await tree.delete('ter') | |
102 … | + await tree.delete('te') | |
103 … | + await tree.delete('test') | |
104 … | + t.deepEquals(tree.root['/'], RadixTree.emptyTreeState) | |
105 … | + } catch (e) { | |
106 … | + console.log(e) | |
107 … | + } | |
108 … | + t.end() | |
109 … | +}) | |
103 | 110 … | |
104 | - // tests delete end branchs | |
105 | - await tree.set('te', 'cat2') | |
106 | - await tree.delete('ter') | |
107 | - await tree.delete('te') | |
108 | - await tree.delete('test') | |
109 | - t.deepEquals(tree.root['/'], [null, null]) | |
110 | - } catch (e) { | |
111 | - console.log(e) | |
112 | - } | |
113 | - t.end() | |
111 … | +tape('large values', async t => { | |
112 … | + const tree = new RadixTree({ | |
113 … | + db: db | |
114 | 114 … | }) |
115 … | + const saved = Buffer.alloc(33).fill(1) | |
116 … | + await tree.set('test', saved) | |
117 … | + const value = await tree.get('test') | |
118 … | + t.equals(value.toString(), saved.toString()) | |
119 … | + t.end() | |
120 … | +}) | |
115 | 121 … | |
116 | - tape('large values', async t => { | |
117 | - const tree = new RadixTree({ | |
118 | - dag: node.dag | |
119 | - }) | |
120 | - const saved = Buffer.alloc(33).fill(1) | |
121 | - await tree.set('test', saved) | |
122 | - const value = await tree.get('test') | |
123 | - t.equals(value.toString(), saved.toString()) | |
124 | - t.end() | |
122 … | +tape('random', async t => { | |
123 … | + const tree = new RadixTree({ | |
124 … | + db: db | |
125 | 125 … | }) |
126 … | + const entries = 100 | |
127 … | + for (let i = 0; i < entries; i++) { | |
128 … | + const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
129 … | + await tree.set(key, Buffer.from([i])) | |
130 … | + } | |
131 … | + // console.log(JSON.stringify(tree.root, null, 2)) | |
126 | 132 … | |
127 | - tape('random', async t => { | |
128 | - const tree = new RadixTree({ | |
129 | - dag: node.dag | |
130 | - }) | |
131 | - const entries = 100 | |
132 | - for (let i = 0; i < entries; i++) { | |
133 | - const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
134 | - await tree.set(key, i) | |
135 | - } | |
133 … | + await tree.flush() | |
136 | 134 … | |
137 | - await tree.flush() | |
135 … | + for (let i = 0; i < entries; i++) { | |
136 … | + const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
137 … | + const value = await tree.get(key) | |
138 … | + t.equals(value[0], i) | |
139 … | + } | |
138 | 140 … | |
139 | - for (let i = 0; i < entries; i++) { | |
140 | - const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
141 | - const value = await tree.get(key) | |
142 | - t.equals(value, i) | |
143 | - } | |
141 … | + await tree.flush() | |
142 … | + for (let i = 0; i < entries; i++) { | |
143 … | + const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
144 … | + await tree.delete(key) | |
145 … | + } | |
144 | 146 … | |
145 | - await tree.flush() | |
146 | - for (let i = 0; i < entries; i++) { | |
147 | - const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) | |
148 | - await tree.delete(key) | |
149 | - } | |
147 … | + t.deepEquals(tree.root['/'], RadixTree.emptyTreeState) | |
150 | 148 … | |
151 | - t.deepEquals(tree.root['/'], [null, null]) | |
152 | - | |
153 | - t.end() | |
154 | - }) | |
149 … | + t.end() | |
155 | 150 … | }) |
dag.js | ||
---|---|---|
@@ -1,0 +1,34 @@ | ||
1 … | +const crypto = require('crypto') | |
2 … | +const DAG = require('ipld-graph-builder/dag') | |
3 … | +const treeNode = require('./treeNode.js') | |
4 … | +const HASH_LEN = 20 | |
5 … | + | |
6 … | +module.exports = class TreeDAG extends DAG { | |
7 … | + put (val, options) { | |
8 … | + const encoded = treeNode.encode(val).toString('hex') | |
9 … | + const key = crypto.createHash('sha256').update(encoded).digest().slice(0, HASH_LEN) | |
10 … | + return new Promise((resolve, reject) => { | |
11 … | + this._dag.put(key, encoded, () => { | |
12 … | + resolve(key) | |
13 … | + }) | |
14 … | + }) | |
15 … | + } | |
16 … | + | |
17 … | + get (link) { | |
18 … | + return new Promise((resolve, reject) => { | |
19 … | + this._dag.get(link, (err, val) => { | |
20 … | + if (err) { | |
21 … | + reject(err) | |
22 … | + } else { | |
23 … | + val = Buffer.from(val, 'hex') | |
24 … | + const decoded = treeNode.decode(val) | |
25 … | + resolve(decoded) | |
26 … | + } | |
27 … | + }) | |
28 … | + }) | |
29 … | + } | |
30 … | + | |
31 … | + isValidLink (link) { | |
32 … | + return Buffer.isBuffer(link) && link.length === HASH_LEN | |
33 … | + } | |
34 … | +} |
treeNode.js | ||
---|---|---|
@@ -1,0 +1,133 @@ | ||
1 … | +const leb128 = require('leb128').unsigned | |
2 … | +const LebStream = require('leb128/stream') | |
3 … | +const Uint1Array = require('uint1array') | |
4 … | +const HASH_LEN = 20 | |
5 … | + | |
6 … | +function toTypedArray (array) { | |
7 … | + return new Uint1Array(new Uint8Array(array).buffer) | |
8 … | +} | |
9 … | + | |
10 … | +// helper functions for nodes | |
11 … | +const EXTENSION = exports.EXTENSION = 0 | |
12 … | +const LBRANCH = exports.LBRANCH = 1 | |
13 … | +const RBRANCH = exports.RBRANCH = 2 | |
14 … | +const VALUE = exports.VALUE = 3 | |
15 … | + | |
16 … | +exports.setBranch = function (node, branch) { | |
17 … | + node['/'][LBRANCH] = branch[0] | |
18 … | + node['/'][RBRANCH] = branch[1] | |
19 … | +} | |
20 … | + | |
21 … | +exports.getBranch = function (node) { | |
22 … | + return node['/'].slice(LBRANCH, LBRANCH + 2) | |
23 … | +} | |
24 … | + | |
25 … | +exports.getValue = function (node) { | |
26 … | + return node['/'][VALUE] | |
27 … | +} | |
28 … | + | |
29 … | +exports.deleteValue = function (node) { | |
30 … | + delete node['/'][VALUE] | |
31 … | +} | |
32 … | + | |
33 … | +exports.getExtension = function (node) { | |
34 … | + if (node['/'][EXTENSION]) { | |
35 … | + const len = node['/'][EXTENSION][0] | |
36 … | + const extension = toTypedArray(node['/'][EXTENSION][1]) | |
37 … | + return extension.subarray(0, len) | |
38 … | + } else { | |
39 … | + return [] | |
40 … | + } | |
41 … | +} | |
42 … | + | |
43 … | +exports.setExtension = function (node, ex) { | |
44 … | + if (ex && ex.length) { | |
45 … | + node['/'][EXTENSION] = [ex.length, Buffer.from(ex.buffer)] | |
46 … | + } else { | |
47 … | + delete node['/'][EXTENSION] | |
48 … | + } | |
49 … | +} | |
50 … | + | |
51 … | +exports.setValue = function (node, val) { | |
52 … | + node['/'][VALUE] = val | |
53 … | +} | |
54 … | + | |
55 … | +exports.isEmpty = function (node) { | |
56 … | + const branch = exports.getBranch(node) | |
57 … | + return !node['/'][EXTENSION] && !branch[0] && !branch[1] && node['/'][VALUE] === undefined | |
58 … | +} | |
59 … | + | |
60 … | +// PREFIX := | LBP | RBP | EXT | LB | RB | VALUE | | |
61 … | +// NODE := | PREFIX | LEN | PAYLOAD | |
62 … | +const MASK = { | |
63 … | + EXTENSION: 8, | |
64 … | + LBRANCH: 4, | |
65 … | + RBRANCH: 2, | |
66 … | + VALUE: 1 | |
67 … | +} | |
68 … | + | |
69 … | +exports.encode = function (node, prefix = 0, encodeLen = false) { | |
70 … | + let encoded = [] | |
71 … | + const ext = node[EXTENSION] | |
72 … | + if (ext) { | |
73 … | + const len = leb128.encode(ext[0]) | |
74 … | + encoded.push(len) | |
75 … | + encoded.push(ext[1]) | |
76 … | + prefix += MASK.EXTENSION | |
77 … | + } | |
78 … | + | |
79 … | + const lb = node[LBRANCH] | |
80 … | + if (lb) { | |
81 … | + encoded.push(lb['/']) | |
82 … | + prefix += MASK.LBRANCH | |
83 … | + } | |
84 … | + | |
85 … | + const rb = node[RBRANCH] | |
86 … | + if (rb) { | |
87 … | + encoded.push(rb['/']) | |
88 … | + prefix += MASK.RBRANCH | |
89 … | + } | |
90 … | + | |
91 … | + const val = node[VALUE] | |
92 … | + if (val !== undefined) { | |
93 … | + encoded.push(val) | |
94 … | + prefix += MASK.VALUE | |
95 … | + } | |
96 … | + | |
97 … | + encoded.unshift(Buffer.from([prefix])) | |
98 … | + encoded = Buffer.concat(encoded) | |
99 … | + if (encodeLen) { | |
100 … | + const len = leb128.encode(encoded.length) | |
101 … | + encoded = Buffer.concat([len, encoded]) | |
102 … | + } | |
103 … | + return encoded | |
104 … | +} | |
105 … | + | |
106 … | +exports.decode = function (val) { | |
107 … | + const node = [null, null, null] | |
108 … | + const prefix = val[0] | |
109 … | + const lebStream = new LebStream(val.slice(1)) | |
110 … | + | |
111 … | + if (prefix & MASK.EXTENSION) { | |
112 … | + const len = Number(leb128.read(lebStream)) | |
113 … | + const ext = lebStream.read(Math.ceil(len / 8)) | |
114 … | + node[EXTENSION] = [len, ext] | |
115 … | + } | |
116 … | + | |
117 … | + if (prefix & MASK.LBRANCH) { | |
118 … | + node[LBRANCH] = { | |
119 … | + '/': lebStream.read(HASH_LEN) | |
120 … | + } | |
121 … | + } | |
122 … | + | |
123 … | + if (prefix & MASK.RBRANCH) { | |
124 … | + node[RBRANCH] = { | |
125 … | + '/': lebStream.read(HASH_LEN) | |
126 … | + } | |
127 … | + } | |
128 … | + | |
129 … | + if (prefix & MASK.VALUE) { | |
130 … | + node[VALUE] = lebStream.buffer | |
131 … | + } | |
132 … | + return node | |
133 … | +} |
Built with git-ssb-web