git ssb

0+

Dominic / flumeview-hashtable



Commit 65f914096658ba9cb91db127af9d60c8b7c2cd51

initial

Dominic Tarr committed on 8/3/2017, 11:36:14 PM

Files changed

LICENSEadded
README.mdadded
hashtable.jsadded
index.jsadded
package-lock.jsonadded
package.jsonadded
test/bulk.jsadded
test/flumeview-index.jsadded
test/index.jsadded
LICENSEView
@@ -1,0 +1,22 @@
1 +Copyright (c) 2017 'Dominic Tarr'
2 +
3 +Permission is hereby granted, free of charge,
4 +to any person obtaining a copy of this software and
5 +associated documentation files (the "Software"), to
6 +deal in the Software without restriction, including
7 +without limitation the rights to use, copy, modify,
8 +merge, publish, distribute, sublicense, and/or sell
9 +copies of the Software, and to permit persons to whom
10 +the Software is furnished to do so,
11 +subject to the following conditions:
12 +
13 +The above copyright notice and this permission notice
14 +shall be included in all copies or substantial portions of the Software.
15 +
16 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
18 +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19 +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
20 +ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21 +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22 +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
README.mdView
@@ -1,0 +1,58 @@
1 +# flumeview-hashtable
2 +
3 +A in-memory hashtable based flumeview.
4 +
5 +Creates an unordered key:value mapping over a flumedb.
6 +the key for each indexed record must be unique, and
7 +the value will be the record stored in the log.
8 +Since flumedb uses integer keys for the log implementation,
9 +the hashtable is just a sparse array of integers, implemented
10 +on a buffer. This means that a relatively small buffer (say 1mb)
11 +can hold hundreds of thousands of references, and since buffers
12 +are not traversed by the garbage collector, this is not a burden
13 +on your app. Also, there is no disk-io while building the index
14 +so if the only disk access is very slow (i.e. browser) this is great.
15 +
16 +## example
17 +
18 +``` js
19 +//you need to provide a hash function.
20 +//it doesn't actually need to be cryptographic
21 +
22 +function hash (v) {
23 + return crypto.createHash('sha256')
24 + .update(v.toString())
25 + .digest().readUInt32BE(0)
26 +}
27 +
28 +//default getKey function
29 +function getKey (data) {
30 + return data.key
31 +}
32 +
33 +var db = Flume(
34 + Log(
35 + '/tmp/test-flumeview-hashtable'+Date.now(),
36 + {blockSize: 1024, codec: require('flumecodec/json')}
37 + ))
38 +.use('index', require('../')(1, hash, getKey))
39 +
40 +db.append({key: 'foo', value: ...}, function (err) {
41 + if(err) throw err
42 + db.index.get('foo', function (err, value) {
43 +
44 + })
45 +})
46 +```
47 +
48 +## TODO
49 +
50 +* resize strategy (okay, so it's not really that useful without this)
51 +* incremental resize
52 +* maybe implement multiple records? for not quite unique values?
53 +* support different key sizes.
54 +
55 +## License
56 +
57 +MIT
58 +
hashtable.jsView
@@ -1,0 +1,53 @@
1 +function NotFound (key) {
2 + var err = new Error('Not Found:'+key)
3 + err.notFound = true
4 + return err
5 +}
6 +
7 +module.exports = function (hash, matches, get) {
8 + return function (buffer, slots) {
9 + var self
10 + var size = 4
11 + var count = 0
12 + if(!buffer) {
13 + buffer = new Buffer(size*slots)
14 + buffer.fill(0)
15 + }
16 + else {
17 + slots = buffer.length/size
18 + count = buffer.readUInt32BE(0)
19 + }
20 + function _get (i) {
21 + return buffer.readUInt32BE((i%slots)*size)
22 + }
23 + function _set (i, v) {
24 + buffer.writeUInt32BE(v, (i%slots)*size)
25 + }
26 +
27 + return self = {
28 + get: function (key, cb) {
29 + ;(function next (i) {
30 + var k = _get(i)
31 + if(k === 0)
32 + cb(NotFound(key))
33 + else
34 + get(k, function (err, data) {
35 + if(err) cb(err)
36 + else if(matches(data, key)) cb(null, data)
37 + else next(i + 1)
38 + })
39 + })(hash(key))
40 + },
41 + _get: _get,
42 + add: function (key, value) {
43 + var i = hash(key)
44 + while(_get(i) !== 0) i++
45 + //buffer.writeUInt32BE(value, (i%slots)*size)
46 + _set(i, value)
47 + self.count = ++count
48 + return i
49 + },
50 + buffer: buffer
51 + }
52 + }
53 +}
index.jsView
@@ -1,0 +1,46 @@
1 +var Obv = require('obv')
2 +var HashTable = require('./hashtable')
3 +var Drain = require('pull-stream/sinks/drain')
4 +module.exports = function (version, hash, getKey) {
5 + getKey = getKey || function (data) { return data.key }
6 +
7 + return function (log, name) {
8 + var since = Obv()
9 + since.set(-1)
10 + var ht = HashTable(hash, function (data, key) {
11 + return key === getKey(data)
12 + }, function (offset, cb) {
13 + log.get(offset-1, cb)
14 + })(null, 200000)
15 +
16 + return {
17 + methods: {get: 'async'},
18 + since: since,
19 + createSink: function () {
20 + return Drain(function (data) {
21 + var k = ht.add(getKey(data.value), data.seq+1)
22 + since.set(data.seq)
23 + })
24 + },
25 + get: function (key, cb) {
26 + var called = false
27 + ht.get(key, function (err, value) {
28 + if(called) throw new Error('called already!')
29 + called = true
30 + cb(err, value)
31 + })
32 + },
33 + close: function (cb) { cb () }
34 + }
35 + }
36 +}
37 +
38 +
39 +
40 +
41 +
42 +
43 +
44 +
45 +
46 +
package-lock.jsonView
@@ -1,0 +1,452 @@
1 +{
2 + "requires": true,
3 + "lockfileVersion": 1,
4 + "dependencies": {
5 + "aligned-block-file": {
6 + "version": "1.1.1",
7 + "resolved": "https://registry.npmjs.org/aligned-block-file/-/aligned-block-file-1.1.1.tgz",
8 + "integrity": "sha1-WzORWxgNMhHAUXfeLrOoVj3OzHQ=",
9 + "requires": {
10 + "hashlru": "2.2.0",
11 + "int53": "0.2.4",
12 + "mkdirp": "0.5.1",
13 + "obv": "0.0.0",
14 + "uint48be": "1.0.2"
15 + },
16 + "dependencies": {
17 + "obv": {
18 + "version": "0.0.0",
19 + "resolved": "https://registry.npmjs.org/obv/-/obv-0.0.0.tgz",
20 + "integrity": "sha1-7eq4Ro+R1BkzYu1/kdC5bdOaecE="
21 + }
22 + }
23 + },
24 + "append-batch": {
25 + "version": "0.0.1",
26 + "resolved": "https://registry.npmjs.org/append-batch/-/append-batch-0.0.1.tgz",
27 + "integrity": "sha1-kiSFjlVpl8zAfxHx7poShTKqDSU="
28 + },
29 + "balanced-match": {
30 + "version": "1.0.0",
31 + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
32 + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
33 + },
34 + "brace-expansion": {
35 + "version": "1.1.8",
36 + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz",
37 + "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=",
38 + "requires": {
39 + "balanced-match": "1.0.0",
40 + "concat-map": "0.0.1"
41 + }
42 + },
43 + "concat-map": {
44 + "version": "0.0.1",
45 + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
46 + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
47 + },
48 + "cont": {
49 + "version": "1.0.3",
50 + "resolved": "https://registry.npmjs.org/cont/-/cont-1.0.3.tgz",
51 + "integrity": "sha1-aHTx6TX8qZ0EjK6qrZoK6wILzOA=",
52 + "requires": {
53 + "continuable": "1.2.0",
54 + "continuable-para": "1.2.0",
55 + "continuable-series": "1.2.0"
56 + }
57 + },
58 + "continuable": {
59 + "version": "1.2.0",
60 + "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.2.0.tgz",
61 + "integrity": "sha1-CCd0aNQRNiAAdMz4cpQwjRafJbY="
62 + },
63 + "continuable-hash": {
64 + "version": "0.1.4",
65 + "resolved": "https://registry.npmjs.org/continuable-hash/-/continuable-hash-0.1.4.tgz",
66 + "integrity": "sha1-gcdNQXcdjJJ4Ph4A5fEbNNbfx4w=",
67 + "requires": {
68 + "continuable": "1.1.8"
69 + },
70 + "dependencies": {
71 + "continuable": {
72 + "version": "1.1.8",
73 + "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz",
74 + "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U="
75 + }
76 + }
77 + },
78 + "continuable-list": {
79 + "version": "0.1.6",
80 + "resolved": "https://registry.npmjs.org/continuable-list/-/continuable-list-0.1.6.tgz",
81 + "integrity": "sha1-h88G7FgHFuEN/5X7C4TF8OisrF8=",
82 + "requires": {
83 + "continuable": "1.1.8"
84 + },
85 + "dependencies": {
86 + "continuable": {
87 + "version": "1.1.8",
88 + "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz",
89 + "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U="
90 + }
91 + }
92 + },
93 + "continuable-para": {
94 + "version": "1.2.0",
95 + "resolved": "https://registry.npmjs.org/continuable-para/-/continuable-para-1.2.0.tgz",
96 + "integrity": "sha1-RFUQ9klFndD8NchyAVFGEicxxYM=",
97 + "requires": {
98 + "continuable-hash": "0.1.4",
99 + "continuable-list": "0.1.6"
100 + }
101 + },
102 + "continuable-series": {
103 + "version": "1.2.0",
104 + "resolved": "https://registry.npmjs.org/continuable-series/-/continuable-series-1.2.0.tgz",
105 + "integrity": "sha1-MkM5euk6cdZVswJoNKUVkLlYueg="
106 + },
107 + "deep-equal": {
108 + "version": "1.0.1",
109 + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
110 + "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU="
111 + },
112 + "define-properties": {
113 + "version": "1.1.2",
114 + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
115 + "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=",
116 + "requires": {
117 + "foreach": "2.0.5",
118 + "object-keys": "1.0.11"
119 + }
120 + },
121 + "defined": {
122 + "version": "1.0.0",
123 + "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz",
124 + "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM="
125 + },
126 + "es-abstract": {
127 + "version": "1.7.0",
128 + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.7.0.tgz",
129 + "integrity": "sha1-363ndOAb/Nl/lhgCmMRJyGI/uUw=",
130 + "requires": {
131 + "es-to-primitive": "1.1.1",
132 + "function-bind": "1.1.0",
133 + "is-callable": "1.1.3",
134 + "is-regex": "1.0.4"
135 + }
136 + },
137 + "es-to-primitive": {
138 + "version": "1.1.1",
139 + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz",
140 + "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=",
141 + "requires": {
142 + "is-callable": "1.1.3",
143 + "is-date-object": "1.0.1",
144 + "is-symbol": "1.0.1"
145 + }
146 + },
147 + "explain-error": {
148 + "version": "1.0.4",
149 + "resolved": "https://registry.npmjs.org/explain-error/-/explain-error-1.0.4.tgz",
150 + "integrity": "sha1-p5PTrAytTGq1cemWj7urbLJTKSk="
151 + },
152 + "flumecodec": {
153 + "version": "0.0.0",
154 + "resolved": "https://registry.npmjs.org/flumecodec/-/flumecodec-0.0.0.tgz",
155 + "integrity": "sha1-Ns4Gq+Lg4BxE3WnyoWUwWiMgZJs=",
156 + "requires": {
157 + "level-codec": "6.2.0"
158 + }
159 + },
160 + "flumedb": {
161 + "version": "0.3.1",
162 + "resolved": "https://registry.npmjs.org/flumedb/-/flumedb-0.3.1.tgz",
163 + "integrity": "sha1-E0GmHanBgm9Rw5fv/WLw84QW9iE=",
164 + "requires": {
165 + "cont": "1.0.3",
166 + "explain-error": "1.0.4",
167 + "obv": "0.0.1",
168 + "pull-cont": "0.0.0",
169 + "pull-stream": "3.6.0"
170 + }
171 + },
172 + "flumelog-offset": {
173 + "version": "3.2.2",
174 + "resolved": "https://registry.npmjs.org/flumelog-offset/-/flumelog-offset-3.2.2.tgz",
175 + "integrity": "sha512-ZU51QP3KCNGiv4VoX4MhbMAB1izAdyUNKkBHp7WKiNk5F8ZOiXAJ2u+8qIYW4GUOAWU52KAm9Ugx6C6TPCUKMQ==",
176 + "dev": true,
177 + "requires": {
178 + "aligned-block-file": "1.1.1",
179 + "append-batch": "0.0.1",
180 + "explain-error": "1.0.4",
181 + "hashlru": "2.2.0",
182 + "int53": "0.2.4",
183 + "looper": "4.0.0",
184 + "ltgt": "2.2.0",
185 + "obv": "0.0.1",
186 + "pull-cursor": "2.1.3",
187 + "uint48be": "1.0.2"
188 + }
189 + },
190 + "for-each": {
191 + "version": "0.3.2",
192 + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.2.tgz",
193 + "integrity": "sha1-LEBFC5NI6X8oEyJZO6lnBLmr1NQ=",
194 + "requires": {
195 + "is-function": "1.0.1"
196 + }
197 + },
198 + "foreach": {
199 + "version": "2.0.5",
200 + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
201 + "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
202 + },
203 + "fs.realpath": {
204 + "version": "1.0.0",
205 + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
206 + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
207 + },
208 + "function-bind": {
209 + "version": "1.1.0",
210 + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.0.tgz",
211 + "integrity": "sha1-FhdnFMgBeY5Ojyz391KUZ7tKV3E="
212 + },
213 + "glob": {
214 + "version": "7.1.2",
215 + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
216 + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
217 + "requires": {
218 + "fs.realpath": "1.0.0",
219 + "inflight": "1.0.6",
220 + "inherits": "2.0.3",
221 + "minimatch": "3.0.4",
222 + "once": "1.4.0",
223 + "path-is-absolute": "1.0.1"
224 + }
225 + },
226 + "has": {
227 + "version": "1.0.1",
228 + "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz",
229 + "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=",
230 + "requires": {
231 + "function-bind": "1.1.0"
232 + }
233 + },
234 + "hashlru": {
235 + "version": "2.2.0",
236 + "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.2.0.tgz",
237 + "integrity": "sha1-eTpYlD+QKupXgXfXsDNfE/JpS3E="
238 + },
239 + "inflight": {
240 + "version": "1.0.6",
241 + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
242 + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
243 + "requires": {
244 + "once": "1.4.0",
245 + "wrappy": "1.0.2"
246 + }
247 + },
248 + "inherits": {
249 + "version": "2.0.3",
250 + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
251 + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
252 + },
253 + "int53": {
254 + "version": "0.2.4",
255 + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz",
256 + "integrity": "sha1-XtjXqtbFxlZ8rmmqf/xKEJ7oD4Y="
257 + },
258 + "is-callable": {
259 + "version": "1.1.3",
260 + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz",
261 + "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI="
262 + },
263 + "is-date-object": {
264 + "version": "1.0.1",
265 + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
266 + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY="
267 + },
268 + "is-function": {
269 + "version": "1.0.1",
270 + "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.1.tgz",
271 + "integrity": "sha1-Es+5i2W1fdPRk6MSH19uL0N2ArU="
272 + },
273 + "is-regex": {
274 + "version": "1.0.4",
275 + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz",
276 + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=",
277 + "requires": {
278 + "has": "1.0.1"
279 + }
280 + },
281 + "is-symbol": {
282 + "version": "1.0.1",
283 + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz",
284 + "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI="
285 + },
286 + "level-codec": {
287 + "version": "6.2.0",
288 + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-6.2.0.tgz",
289 + "integrity": "sha1-pLUkS7akwvcj1oodZOmAxTYn2dQ="
290 + },
291 + "looper": {
292 + "version": "4.0.0",
293 + "resolved": "https://registry.npmjs.org/looper/-/looper-4.0.0.tgz",
294 + "integrity": "sha1-dwat7VmpntygbmtUu4bI7BnJUVU="
295 + },
296 + "ltgt": {
297 + "version": "2.2.0",
298 + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.0.tgz",
299 + "integrity": "sha1-tlul/LNJopkkyOMz98alVi8uSEI="
300 + },
301 + "minimatch": {
302 + "version": "3.0.4",
303 + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
304 + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
305 + "requires": {
306 + "brace-expansion": "1.1.8"
307 + }
308 + },
309 + "minimist": {
310 + "version": "1.2.0",
311 + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
312 + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
313 + },
314 + "mkdirp": {
315 + "version": "0.5.1",
316 + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
317 + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
318 + "requires": {
319 + "minimist": "0.0.8"
320 + },
321 + "dependencies": {
322 + "minimist": {
323 + "version": "0.0.8",
324 + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
325 + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
326 + }
327 + }
328 + },
329 + "object-inspect": {
330 + "version": "1.2.2",
331 + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.2.2.tgz",
332 + "integrity": "sha1-yCEV5PzIiK6hTWTCLk8X9qcNXlo="
333 + },
334 + "object-keys": {
335 + "version": "1.0.11",
336 + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz",
337 + "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0="
338 + },
339 + "obv": {
340 + "version": "0.0.1",
341 + "resolved": "https://registry.npmjs.org/obv/-/obv-0.0.1.tgz",
342 + "integrity": "sha1-yyNhBjQVNvDaxIFeBnCCIcrX+14="
343 + },
344 + "once": {
345 + "version": "1.4.0",
346 + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
347 + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
348 + "requires": {
349 + "wrappy": "1.0.2"
350 + }
351 + },
352 + "path-is-absolute": {
353 + "version": "1.0.1",
354 + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
355 + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
356 + },
357 + "path-parse": {
358 + "version": "1.0.5",
359 + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz",
360 + "integrity": "sha1-PBrfhx6pzWyUMbbqK9dKD/BVxME="
361 + },
362 + "pull-cont": {
363 + "version": "0.0.0",
364 + "resolved": "https://registry.npmjs.org/pull-cont/-/pull-cont-0.0.0.tgz",
365 + "integrity": "sha1-P6xIuBrJe3W6ATMgiLDOevjBvg4="
366 + },
367 + "pull-cursor": {
368 + "version": "2.1.3",
369 + "resolved": "https://registry.npmjs.org/pull-cursor/-/pull-cursor-2.1.3.tgz",
370 + "integrity": "sha1-Ah6X19I9eK9mrda4wcF/1v2xK2w=",
371 + "requires": {
372 + "looper": "4.0.0",
373 + "ltgt": "2.2.0",
374 + "pull-stream": "3.6.0"
375 + }
376 + },
377 + "pull-stream": {
378 + "version": "3.6.0",
379 + "resolved": "https://registry.npmjs.org/pull-stream/-/pull-stream-3.6.0.tgz",
380 + "integrity": "sha1-WdAzpoFdTjCX1Hw9KxiTqeWKI1E="
381 + },
382 + "resolve": {
383 + "version": "1.3.3",
384 + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.3.3.tgz",
385 + "integrity": "sha1-ZVkHw0aahoDcLeOidaj91paR8OU=",
386 + "requires": {
387 + "path-parse": "1.0.5"
388 + }
389 + },
390 + "resumer": {
391 + "version": "0.0.0",
392 + "resolved": "https://registry.npmjs.org/resumer/-/resumer-0.0.0.tgz",
393 + "integrity": "sha1-8ej0YeQGS6Oegq883CqMiT0HZ1k=",
394 + "requires": {
395 + "through": "2.3.8"
396 + }
397 + },
398 + "string.prototype.trim": {
399 + "version": "1.1.2",
400 + "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz",
401 + "integrity": "sha1-0E3iyJ4Tf019IG8Ia17S+ua+jOo=",
402 + "requires": {
403 + "define-properties": "1.1.2",
404 + "es-abstract": "1.7.0",
405 + "function-bind": "1.1.0"
406 + }
407 + },
408 + "tape": {
409 + "version": "4.7.0",
410 + "resolved": "https://registry.npmjs.org/tape/-/tape-4.7.0.tgz",
411 + "integrity": "sha512-ePzu2KfZYVtq0v+KKGxBJ9HJWYZ4MaQWeGabD+KpVdMKRen3NJPf6EiwA5BxfMkhQPGtCwnOFWelcB39bhOUng==",
412 + "requires": {
413 + "deep-equal": "1.0.1",
414 + "defined": "1.0.0",
415 + "for-each": "0.3.2",
416 + "function-bind": "1.1.0",
417 + "glob": "7.1.2",
418 + "has": "1.0.1",
419 + "inherits": "2.0.3",
420 + "minimist": "1.2.0",
421 + "object-inspect": "1.2.2",
422 + "resolve": "1.3.3",
423 + "resumer": "0.0.0",
424 + "string.prototype.trim": "1.1.2",
425 + "through": "2.3.8"
426 + }
427 + },
428 + "test-flumeview-index": {
429 + "version": "0.0.1",
430 + "resolved": "https://registry.npmjs.org/test-flumeview-index/-/test-flumeview-index-0.0.1.tgz",
431 + "integrity": "sha1-Dq1z4CGmFrwLb3li8ekqdQ6PUHE=",
432 + "requires": {
433 + "tape": "4.7.0"
434 + }
435 + },
436 + "through": {
437 + "version": "2.3.8",
438 + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
439 + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
440 + },
441 + "uint48be": {
442 + "version": "1.0.2",
443 + "resolved": "https://registry.npmjs.org/uint48be/-/uint48be-1.0.2.tgz",
444 + "integrity": "sha512-jNn1eEi81BLiZfJkjbiAKPDMj7iFrturKazqpBu0aJYLr6evgkn+9rgkX/gUwPBj5j2Ri5oUelsqC/S1zmpWBA=="
445 + },
446 + "wrappy": {
447 + "version": "1.0.2",
448 + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
449 + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
450 + }
451 + }
452 +}
package.jsonView
@@ -1,0 +1,19 @@
1 +{
2 + "name": "flumeview-hashtable",
3 + "description": "",
4 + "version": "0.0.0",
5 + "homepage": "https://github.com/dominictarr/flumeview-hashtable",
6 + "repository": {
7 + "type": "git",
8 + "url": "git://github.com/dominictarr/flumeview-hashtable.git"
9 + },
10 + "dependencies": {
11 + },
12 + "devDependencies": {
13 + },
14 + "scripts": {
15 + "test": "set -e; for t in test/*.js; do node $t; done"
16 + },
17 + "author": "'Dominic Tarr' <dominic.tarr@gmail.com> (dominictarr.com)",
18 + "license": "MIT"
19 +}
test/bulk.jsView
@@ -1,0 +1,39 @@
1 +var Log = require('flumelog-offset')
2 +var Flume = require('flumedb')
3 +var crypto = require('crypto')
4 +
5 +function hash (v) {
6 + return crypto.createHash('sha256')
7 + .update(v.toString())
8 + .digest().readUInt32BE(0)
9 +}
10 +
11 +
12 +var db = Flume(
13 + Log(
14 + '/tmp/test-flumeview-hashtable'+Date.now(),
15 + {blockSize: 1024, codec: require('flumecodec/json')}
16 + ))
17 + .use('index', require('../')(1, hash))
18 +
19 + var N = 10000
20 + var data = []
21 + for(var i = 0; i < N; i++)
22 + data.push({key: '#'+i, value: {
23 + foo: Math.random(), bar: Date.now()
24 + }})
25 +
26 + db.append(data, function (err, offset) {
27 + //ordered reads
28 + var start = Date.now(), n = 0
29 + for(var i = 0; i < N; i++)
30 + db.index.get('#'+i, next)
31 + function next () {
32 + if(++n == N)
33 + console.log('done', Date.now() - start)
34 + }
35 + })
36 +
37 +
38 +
39 +
test/flumeview-index.jsView
@@ -1,0 +1,20 @@
1 +
2 +var Log = require('flumelog-offset')
3 +var Flume = require('flumedb')
4 +var crypto = require('crypto')
5 +function hash (v) {
6 + return crypto.createHash('sha256')
7 + .update(v.toString())
8 + .digest().readUInt32BE(0)
9 +}
10 +
11 +require('test-flumeview-index')(
12 + Flume(
13 + Log(
14 + '/tmp/test-flumeview-hashtable'+Date.now(),
15 + {blockSize: 1024, codec: require('flumecodec/json')}
16 + ))
17 + .use('index', require('../')(1, hash))
18 +)
19 +
20 +
test/index.jsView
@@ -1,0 +1,40 @@
1 +
2 +var tape = require('tape')
3 +var crypto = require('crypto')
4 +
5 +function hash (v) {
6 + return crypto.createHash('sha256').update(v.toString()).digest().readUInt32BE(0)
7 +}
8 +
9 +function matches (k, data) {
10 + return k === hash(data+'value')
11 +}
12 +
13 +function get(k, cb) {
14 + cb(null, k)
15 +}
16 +
17 +tape('random integers', function (t) {
18 +
19 + var ht = require('../hashtable')(hash, matches, get)(null, 2000)
20 +
21 + var r = hash('hello random'+Date.now())
22 +
23 + var k = ht.add('random2'+Date.now(), r)
24 + t.equal(ht._get(k), r)
25 + console.log(k, r)
26 +
27 + for(var i = 0; i < 1000; i++) {
28 + var k = ht.add(i, hash(i+'value'))
29 + ht.get(i, function (err, j) {
30 + if(err) throw err
31 + t.equal(j, hash(i+'value'))
32 + })
33 + t.notEqual(k, i)
34 + }
35 + console.log(ht.buffer.toString('hex'))
36 + t.end()
37 +
38 +})
39 +
40 +

Built with git-ssb-web