git ssb

0+

Dominic / flumeview-hashtable



Commit 65f914096658ba9cb91db127af9d60c8b7c2cd51

Browse Files

initial

Dominic Tarr committed on 8/3/2017, 11:36:14 PM

Files changed

LICENSEadded
README.mdadded
hashtable.jsadded
index.jsadded
package-lock.jsonadded
package.jsonadded
test/bulk.jsadded
test/flumeview-index.jsadded
test/index.jsadded
LICENSEView
@@ -1,0 +1,22 @@
1 …+Copyright (c) 2017 'Dominic Tarr'
2 …+
3 …+Permission is hereby granted, free of charge,
4 …+to any person obtaining a copy of this software and
5 …+associated documentation files (the "Software"), to
6 …+deal in the Software without restriction, including
7 …+without limitation the rights to use, copy, modify,
8 …+merge, publish, distribute, sublicense, and/or sell
9 …+copies of the Software, and to permit persons to whom
10 …+the Software is furnished to do so,
11 …+subject to the following conditions:
12 …+
13 …+The above copyright notice and this permission notice
14 …+shall be included in all copies or substantial portions of the Software.
15 …+
16 …+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 …+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
18 …+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19 …+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
20 …+ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21 …+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22 …+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
README.mdView
@@ -1,0 +1,58 @@
1 …+# flumeview-hashtable
2 …+
3 …+A in-memory hashtable based flumeview.
4 …+
5 …+Creates an unordered key:value mapping over a flumedb.
6 …+the key for each indexed record must be unique, and
7 …+the value will be the record stored in the log.
8 …+Since flumedb uses integer keys for the log implementation,
9 …+the hashtable is just a sparse array of integers, implemented
10 …+on a buffer. This means that a relatively small buffer (say 1mb)
11 …+can hold hundreds of thousands of references, and since buffers
12 …+are not traversed by the garbage collector, this is not a burden
13 …+on your app. Also, there is no disk-io while building the index
14 …+so if the only disk access is very slow (i.e. browser) this is great.
15 …+
16 …+## example
17 …+
18 …+``` js
19 …+//you need to provide a hash function.
20 …+//it doesn't actually need to be cryptographic
21 …+
22 …+function hash (v) {
23 …+ return crypto.createHash('sha256')
24 …+ .update(v.toString())
25 …+ .digest().readUInt32BE(0)
26 …+}
27 …+
28 …+//default getKey function
29 …+function getKey (data) {
30 …+ return data.key
31 …+}
32 …+
33 …+var db = Flume(
34 …+ Log(
35 …+ '/tmp/test-flumeview-hashtable'+Date.now(),
36 …+ {blockSize: 1024, codec: require('flumecodec/json')}
37 …+ ))
38 …+.use('index', require('../')(1, hash, getKey))
39 …+
40 …+db.append({key: 'foo', value: ...}, function (err) {
41 …+ if(err) throw err
42 …+ db.index.get('foo', function (err, value) {
43 …+
44 …+ })
45 …+})
46 …+```
47 …+
48 …+## TODO
49 …+
50 …+* resize strategy (okay, so it's not really that useful without this)
51 …+* incremental resize
52 …+* maybe implement multiple records? for not quite unique values?
53 …+* support different key sizes.
54 …+
55 …+## License
56 …+
57 …+MIT
58 …+
hashtable.jsView
@@ -1,0 +1,53 @@
1 …+function NotFound (key) {
2 …+ var err = new Error('Not Found:'+key)
3 …+ err.notFound = true
4 …+ return err
5 …+}
6 …+
7 …+module.exports = function (hash, matches, get) {
8 …+ return function (buffer, slots) {
9 …+ var self
10 …+ var size = 4
11 …+ var count = 0
12 …+ if(!buffer) {
13 …+ buffer = new Buffer(size*slots)
14 …+ buffer.fill(0)
15 …+ }
16 …+ else {
17 …+ slots = buffer.length/size
18 …+ count = buffer.readUInt32BE(0)
19 …+ }
20 …+ function _get (i) {
21 …+ return buffer.readUInt32BE((i%slots)*size)
22 …+ }
23 …+ function _set (i, v) {
24 …+ buffer.writeUInt32BE(v, (i%slots)*size)
25 …+ }
26 …+
27 …+ return self = {
28 …+ get: function (key, cb) {
29 …+ ;(function next (i) {
30 …+ var k = _get(i)
31 …+ if(k === 0)
32 …+ cb(NotFound(key))
33 …+ else
34 …+ get(k, function (err, data) {
35 …+ if(err) cb(err)
36 …+ else if(matches(data, key)) cb(null, data)
37 …+ else next(i + 1)
38 …+ })
39 …+ })(hash(key))
40 …+ },
41 …+ _get: _get,
42 …+ add: function (key, value) {
43 …+ var i = hash(key)
44 …+ while(_get(i) !== 0) i++
45 …+ //buffer.writeUInt32BE(value, (i%slots)*size)
46 …+ _set(i, value)
47 …+ self.count = ++count
48 …+ return i
49 …+ },
50 …+ buffer: buffer
51 …+ }
52 …+ }
53 …+}
index.jsView
@@ -1,0 +1,46 @@
1 …+var Obv = require('obv')
2 …+var HashTable = require('./hashtable')
3 …+var Drain = require('pull-stream/sinks/drain')
4 …+module.exports = function (version, hash, getKey) {
5 …+ getKey = getKey || function (data) { return data.key }
6 …+
7 …+ return function (log, name) {
8 …+ var since = Obv()
9 …+ since.set(-1)
10 …+ var ht = HashTable(hash, function (data, key) {
11 …+ return key === getKey(data)
12 …+ }, function (offset, cb) {
13 …+ log.get(offset-1, cb)
14 …+ })(null, 200000)
15 …+
16 …+ return {
17 …+ methods: {get: 'async'},
18 …+ since: since,
19 …+ createSink: function () {
20 …+ return Drain(function (data) {
21 …+ var k = ht.add(getKey(data.value), data.seq+1)
22 …+ since.set(data.seq)
23 …+ })
24 …+ },
25 …+ get: function (key, cb) {
26 …+ var called = false
27 …+ ht.get(key, function (err, value) {
28 …+ if(called) throw new Error('called already!')
29 …+ called = true
30 …+ cb(err, value)
31 …+ })
32 …+ },
33 …+ close: function (cb) { cb () }
34 …+ }
35 …+ }
36 …+}
37 …+
38 …+
39 …+
40 …+
41 …+
42 …+
43 …+
44 …+
45 …+
46 …+
package-lock.jsonView
@@ -1,0 +1,452 @@
1 …+{
2 …+ "requires": true,
3 …+ "lockfileVersion": 1,
4 …+ "dependencies": {
5 …+ "aligned-block-file": {
6 …+ "version": "1.1.1",
7 …+ "resolved": "https://registry.npmjs.org/aligned-block-file/-/aligned-block-file-1.1.1.tgz",
8 …+ "integrity": "sha1-WzORWxgNMhHAUXfeLrOoVj3OzHQ=",
9 …+ "requires": {
10 …+ "hashlru": "2.2.0",
11 …+ "int53": "0.2.4",
12 …+ "mkdirp": "0.5.1",
13 …+ "obv": "0.0.0",
14 …+ "uint48be": "1.0.2"
15 …+ },
16 …+ "dependencies": {
17 …+ "obv": {
18 …+ "version": "0.0.0",
19 …+ "resolved": "https://registry.npmjs.org/obv/-/obv-0.0.0.tgz",
20 …+ "integrity": "sha1-7eq4Ro+R1BkzYu1/kdC5bdOaecE="
21 …+ }
22 …+ }
23 …+ },
24 …+ "append-batch": {
25 …+ "version": "0.0.1",
26 …+ "resolved": "https://registry.npmjs.org/append-batch/-/append-batch-0.0.1.tgz",
27 …+ "integrity": "sha1-kiSFjlVpl8zAfxHx7poShTKqDSU="
28 …+ },
29 …+ "balanced-match": {
30 …+ "version": "1.0.0",
31 …+ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
32 …+ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
33 …+ },
34 …+ "brace-expansion": {
35 …+ "version": "1.1.8",
36 …+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz",
37 …+ "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=",
38 …+ "requires": {
39 …+ "balanced-match": "1.0.0",
40 …+ "concat-map": "0.0.1"
41 …+ }
42 …+ },
43 …+ "concat-map": {
44 …+ "version": "0.0.1",
45 …+ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
46 …+ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
47 …+ },
48 …+ "cont": {
49 …+ "version": "1.0.3",
50 …+ "resolved": "https://registry.npmjs.org/cont/-/cont-1.0.3.tgz",
51 …+ "integrity": "sha1-aHTx6TX8qZ0EjK6qrZoK6wILzOA=",
52 …+ "requires": {
53 …+ "continuable": "1.2.0",
54 …+ "continuable-para": "1.2.0",
55 …+ "continuable-series": "1.2.0"
56 …+ }
57 …+ },
58 …+ "continuable": {
59 …+ "version": "1.2.0",
60 …+ "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.2.0.tgz",
61 …+ "integrity": "sha1-CCd0aNQRNiAAdMz4cpQwjRafJbY="
62 …+ },
63 …+ "continuable-hash": {
64 …+ "version": "0.1.4",
65 …+ "resolved": "https://registry.npmjs.org/continuable-hash/-/continuable-hash-0.1.4.tgz",
66 …+ "integrity": "sha1-gcdNQXcdjJJ4Ph4A5fEbNNbfx4w=",
67 …+ "requires": {
68 …+ "continuable": "1.1.8"
69 …+ },
70 …+ "dependencies": {
71 …+ "continuable": {
72 …+ "version": "1.1.8",
73 …+ "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz",
74 …+ "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U="
75 …+ }
76 …+ }
77 …+ },
78 …+ "continuable-list": {
79 …+ "version": "0.1.6",
80 …+ "resolved": "https://registry.npmjs.org/continuable-list/-/continuable-list-0.1.6.tgz",
81 …+ "integrity": "sha1-h88G7FgHFuEN/5X7C4TF8OisrF8=",
82 …+ "requires": {
83 …+ "continuable": "1.1.8"
84 …+ },
85 …+ "dependencies": {
86 …+ "continuable": {
87 …+ "version": "1.1.8",
88 …+ "resolved": "https://registry.npmjs.org/continuable/-/continuable-1.1.8.tgz",
89 …+ "integrity": "sha1-3Id7R0FghwrjvN6HM2Jo6+UFl9U="
90 …+ }
91 …+ }
92 …+ },
93 …+ "continuable-para": {
94 …+ "version": "1.2.0",
95 …+ "resolved": "https://registry.npmjs.org/continuable-para/-/continuable-para-1.2.0.tgz",
96 …+ "integrity": "sha1-RFUQ9klFndD8NchyAVFGEicxxYM=",
97 …+ "requires": {
98 …+ "continuable-hash": "0.1.4",
99 …+ "continuable-list": "0.1.6"
100 …+ }
101 …+ },
102 …+ "continuable-series": {
103 …+ "version": "1.2.0",
104 …+ "resolved": "https://registry.npmjs.org/continuable-series/-/continuable-series-1.2.0.tgz",
105 …+ "integrity": "sha1-MkM5euk6cdZVswJoNKUVkLlYueg="
106 …+ },
107 …+ "deep-equal": {
108 …+ "version": "1.0.1",
109 …+ "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
110 …+ "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU="
111 …+ },
112 …+ "define-properties": {
113 …+ "version": "1.1.2",
114 …+ "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
115 …+ "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=",
116 …+ "requires": {
117 …+ "foreach": "2.0.5",
118 …+ "object-keys": "1.0.11"
119 …+ }
120 …+ },
121 …+ "defined": {
122 …+ "version": "1.0.0",
123 …+ "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz",
124 …+ "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM="
125 …+ },
126 …+ "es-abstract": {
127 …+ "version": "1.7.0",
128 …+ "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.7.0.tgz",
129 …+ "integrity": "sha1-363ndOAb/Nl/lhgCmMRJyGI/uUw=",
130 …+ "requires": {
131 …+ "es-to-primitive": "1.1.1",
132 …+ "function-bind": "1.1.0",
133 …+ "is-callable": "1.1.3",
134 …+ "is-regex": "1.0.4"
135 …+ }
136 …+ },
137 …+ "es-to-primitive": {
138 …+ "version": "1.1.1",
139 …+ "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz",
140 …+ "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=",
141 …+ "requires": {
142 …+ "is-callable": "1.1.3",
143 …+ "is-date-object": "1.0.1",
144 …+ "is-symbol": "1.0.1"
145 …+ }
146 …+ },
147 …+ "explain-error": {
148 …+ "version": "1.0.4",
149 …+ "resolved": "https://registry.npmjs.org/explain-error/-/explain-error-1.0.4.tgz",
150 …+ "integrity": "sha1-p5PTrAytTGq1cemWj7urbLJTKSk="
151 …+ },
152 …+ "flumecodec": {
153 …+ "version": "0.0.0",
154 …+ "resolved": "https://registry.npmjs.org/flumecodec/-/flumecodec-0.0.0.tgz",
155 …+ "integrity": "sha1-Ns4Gq+Lg4BxE3WnyoWUwWiMgZJs=",
156 …+ "requires": {
157 …+ "level-codec": "6.2.0"
158 …+ }
159 …+ },
160 …+ "flumedb": {
161 …+ "version": "0.3.1",
162 …+ "resolved": "https://registry.npmjs.org/flumedb/-/flumedb-0.3.1.tgz",
163 …+ "integrity": "sha1-E0GmHanBgm9Rw5fv/WLw84QW9iE=",
164 …+ "requires": {
165 …+ "cont": "1.0.3",
166 …+ "explain-error": "1.0.4",
167 …+ "obv": "0.0.1",
168 …+ "pull-cont": "0.0.0",
169 …+ "pull-stream": "3.6.0"
170 …+ }
171 …+ },
172 …+ "flumelog-offset": {
173 …+ "version": "3.2.2",
174 …+ "resolved": "https://registry.npmjs.org/flumelog-offset/-/flumelog-offset-3.2.2.tgz",
175 …+ "integrity": "sha512-ZU51QP3KCNGiv4VoX4MhbMAB1izAdyUNKkBHp7WKiNk5F8ZOiXAJ2u+8qIYW4GUOAWU52KAm9Ugx6C6TPCUKMQ==",
176 …+ "dev": true,
177 …+ "requires": {
178 …+ "aligned-block-file": "1.1.1",
179 …+ "append-batch": "0.0.1",
180 …+ "explain-error": "1.0.4",
181 …+ "hashlru": "2.2.0",
182 …+ "int53": "0.2.4",
183 …+ "looper": "4.0.0",
184 …+ "ltgt": "2.2.0",
185 …+ "obv": "0.0.1",
186 …+ "pull-cursor": "2.1.3",
187 …+ "uint48be": "1.0.2"
188 …+ }
189 …+ },
190 …+ "for-each": {
191 …+ "version": "0.3.2",
192 …+ "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.2.tgz",
193 …+ "integrity": "sha1-LEBFC5NI6X8oEyJZO6lnBLmr1NQ=",
194 …+ "requires": {
195 …+ "is-function": "1.0.1"
196 …+ }
197 …+ },
198 …+ "foreach": {
199 …+ "version": "2.0.5",
200 …+ "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
201 …+ "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
202 …+ },
203 …+ "fs.realpath": {
204 …+ "version": "1.0.0",
205 …+ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
206 …+ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
207 …+ },
208 …+ "function-bind": {
209 …+ "version": "1.1.0",
210 …+ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.0.tgz",
211 …+ "integrity": "sha1-FhdnFMgBeY5Ojyz391KUZ7tKV3E="
212 …+ },
213 …+ "glob": {
214 …+ "version": "7.1.2",
215 …+ "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
216 …+ "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
217 …+ "requires": {
218 …+ "fs.realpath": "1.0.0",
219 …+ "inflight": "1.0.6",
220 …+ "inherits": "2.0.3",
221 …+ "minimatch": "3.0.4",
222 …+ "once": "1.4.0",
223 …+ "path-is-absolute": "1.0.1"
224 …+ }
225 …+ },
226 …+ "has": {
227 …+ "version": "1.0.1",
228 …+ "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz",
229 …+ "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=",
230 …+ "requires": {
231 …+ "function-bind": "1.1.0"
232 …+ }
233 …+ },
234 …+ "hashlru": {
235 …+ "version": "2.2.0",
236 …+ "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.2.0.tgz",
237 …+ "integrity": "sha1-eTpYlD+QKupXgXfXsDNfE/JpS3E="
238 …+ },
239 …+ "inflight": {
240 …+ "version": "1.0.6",
241 …+ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
242 …+ "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
243 …+ "requires": {
244 …+ "once": "1.4.0",
245 …+ "wrappy": "1.0.2"
246 …+ }
247 …+ },
248 …+ "inherits": {
249 …+ "version": "2.0.3",
250 …+ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
251 …+ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
252 …+ },
253 …+ "int53": {
254 …+ "version": "0.2.4",
255 …+ "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz",
256 …+ "integrity": "sha1-XtjXqtbFxlZ8rmmqf/xKEJ7oD4Y="
257 …+ },
258 …+ "is-callable": {
259 …+ "version": "1.1.3",
260 …+ "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz",
261 …+ "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI="
262 …+ },
263 …+ "is-date-object": {
264 …+ "version": "1.0.1",
265 …+ "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
266 …+ "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY="
267 …+ },
268 …+ "is-function": {
269 …+ "version": "1.0.1",
270 …+ "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.1.tgz",
271 …+ "integrity": "sha1-Es+5i2W1fdPRk6MSH19uL0N2ArU="
272 …+ },
273 …+ "is-regex": {
274 …+ "version": "1.0.4",
275 …+ "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz",
276 …+ "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=",
277 …+ "requires": {
278 …+ "has": "1.0.1"
279 …+ }
280 …+ },
281 …+ "is-symbol": {
282 …+ "version": "1.0.1",
283 …+ "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz",
284 …+ "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI="
285 …+ },
286 …+ "level-codec": {
287 …+ "version": "6.2.0",
288 …+ "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-6.2.0.tgz",
289 …+ "integrity": "sha1-pLUkS7akwvcj1oodZOmAxTYn2dQ="
290 …+ },
291 …+ "looper": {
292 …+ "version": "4.0.0",
293 …+ "resolved": "https://registry.npmjs.org/looper/-/looper-4.0.0.tgz",
294 …+ "integrity": "sha1-dwat7VmpntygbmtUu4bI7BnJUVU="
295 …+ },
296 …+ "ltgt": {
297 …+ "version": "2.2.0",
298 …+ "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.0.tgz",
299 …+ "integrity": "sha1-tlul/LNJopkkyOMz98alVi8uSEI="
300 …+ },
301 …+ "minimatch": {
302 …+ "version": "3.0.4",
303 …+ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
304 …+ "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
305 …+ "requires": {
306 …+ "brace-expansion": "1.1.8"
307 …+ }
308 …+ },
309 …+ "minimist": {
310 …+ "version": "1.2.0",
311 …+ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
312 …+ "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
313 …+ },
314 …+ "mkdirp": {
315 …+ "version": "0.5.1",
316 …+ "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
317 …+ "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
318 …+ "requires": {
319 …+ "minimist": "0.0.8"
320 …+ },
321 …+ "dependencies": {
322 …+ "minimist": {
323 …+ "version": "0.0.8",
324 …+ "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
325 …+ "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
326 …+ }
327 …+ }
328 …+ },
329 …+ "object-inspect": {
330 …+ "version": "1.2.2",
331 …+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.2.2.tgz",
332 …+ "integrity": "sha1-yCEV5PzIiK6hTWTCLk8X9qcNXlo="
333 …+ },
334 …+ "object-keys": {
335 …+ "version": "1.0.11",
336 …+ "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz",
337 …+ "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0="
338 …+ },
339 …+ "obv": {
340 …+ "version": "0.0.1",
341 …+ "resolved": "https://registry.npmjs.org/obv/-/obv-0.0.1.tgz",
342 …+ "integrity": "sha1-yyNhBjQVNvDaxIFeBnCCIcrX+14="
343 …+ },
344 …+ "once": {
345 …+ "version": "1.4.0",
346 …+ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
347 …+ "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
348 …+ "requires": {
349 …+ "wrappy": "1.0.2"
350 …+ }
351 …+ },
352 …+ "path-is-absolute": {
353 …+ "version": "1.0.1",
354 …+ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
355 …+ "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
356 …+ },
357 …+ "path-parse": {
358 …+ "version": "1.0.5",
359 …+ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz",
360 …+ "integrity": "sha1-PBrfhx6pzWyUMbbqK9dKD/BVxME="
361 …+ },
362 …+ "pull-cont": {
363 …+ "version": "0.0.0",
364 …+ "resolved": "https://registry.npmjs.org/pull-cont/-/pull-cont-0.0.0.tgz",
365 …+ "integrity": "sha1-P6xIuBrJe3W6ATMgiLDOevjBvg4="
366 …+ },
367 …+ "pull-cursor": {
368 …+ "version": "2.1.3",
369 …+ "resolved": "https://registry.npmjs.org/pull-cursor/-/pull-cursor-2.1.3.tgz",
370 …+ "integrity": "sha1-Ah6X19I9eK9mrda4wcF/1v2xK2w=",
371 …+ "requires": {
372 …+ "looper": "4.0.0",
373 …+ "ltgt": "2.2.0",
374 …+ "pull-stream": "3.6.0"
375 …+ }
376 …+ },
377 …+ "pull-stream": {
378 …+ "version": "3.6.0",
379 …+ "resolved": "https://registry.npmjs.org/pull-stream/-/pull-stream-3.6.0.tgz",
380 …+ "integrity": "sha1-WdAzpoFdTjCX1Hw9KxiTqeWKI1E="
381 …+ },
382 …+ "resolve": {
383 …+ "version": "1.3.3",
384 …+ "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.3.3.tgz",
385 …+ "integrity": "sha1-ZVkHw0aahoDcLeOidaj91paR8OU=",
386 …+ "requires": {
387 …+ "path-parse": "1.0.5"
388 …+ }
389 …+ },
390 …+ "resumer": {
391 …+ "version": "0.0.0",
392 …+ "resolved": "https://registry.npmjs.org/resumer/-/resumer-0.0.0.tgz",
393 …+ "integrity": "sha1-8ej0YeQGS6Oegq883CqMiT0HZ1k=",
394 …+ "requires": {
395 …+ "through": "2.3.8"
396 …+ }
397 …+ },
398 …+ "string.prototype.trim": {
399 …+ "version": "1.1.2",
400 …+ "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz",
401 …+ "integrity": "sha1-0E3iyJ4Tf019IG8Ia17S+ua+jOo=",
402 …+ "requires": {
403 …+ "define-properties": "1.1.2",
404 …+ "es-abstract": "1.7.0",
405 …+ "function-bind": "1.1.0"
406 …+ }
407 …+ },
408 …+ "tape": {
409 …+ "version": "4.7.0",
410 …+ "resolved": "https://registry.npmjs.org/tape/-/tape-4.7.0.tgz",
411 …+ "integrity": "sha512-ePzu2KfZYVtq0v+KKGxBJ9HJWYZ4MaQWeGabD+KpVdMKRen3NJPf6EiwA5BxfMkhQPGtCwnOFWelcB39bhOUng==",
412 …+ "requires": {
413 …+ "deep-equal": "1.0.1",
414 …+ "defined": "1.0.0",
415 …+ "for-each": "0.3.2",
416 …+ "function-bind": "1.1.0",
417 …+ "glob": "7.1.2",
418 …+ "has": "1.0.1",
419 …+ "inherits": "2.0.3",
420 …+ "minimist": "1.2.0",
421 …+ "object-inspect": "1.2.2",
422 …+ "resolve": "1.3.3",
423 …+ "resumer": "0.0.0",
424 …+ "string.prototype.trim": "1.1.2",
425 …+ "through": "2.3.8"
426 …+ }
427 …+ },
428 …+ "test-flumeview-index": {
429 …+ "version": "0.0.1",
430 …+ "resolved": "https://registry.npmjs.org/test-flumeview-index/-/test-flumeview-index-0.0.1.tgz",
431 …+ "integrity": "sha1-Dq1z4CGmFrwLb3li8ekqdQ6PUHE=",
432 …+ "requires": {
433 …+ "tape": "4.7.0"
434 …+ }
435 …+ },
436 …+ "through": {
437 …+ "version": "2.3.8",
438 …+ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
439 …+ "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
440 …+ },
441 …+ "uint48be": {
442 …+ "version": "1.0.2",
443 …+ "resolved": "https://registry.npmjs.org/uint48be/-/uint48be-1.0.2.tgz",
444 …+ "integrity": "sha512-jNn1eEi81BLiZfJkjbiAKPDMj7iFrturKazqpBu0aJYLr6evgkn+9rgkX/gUwPBj5j2Ri5oUelsqC/S1zmpWBA=="
445 …+ },
446 …+ "wrappy": {
447 …+ "version": "1.0.2",
448 …+ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
449 …+ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
450 …+ }
451 …+ }
452 …+}
package.jsonView
@@ -1,0 +1,19 @@
1 …+{
2 …+ "name": "flumeview-hashtable",
3 …+ "description": "",
4 …+ "version": "0.0.0",
5 …+ "homepage": "https://github.com/dominictarr/flumeview-hashtable",
6 …+ "repository": {
7 …+ "type": "git",
8 …+ "url": "git://github.com/dominictarr/flumeview-hashtable.git"
9 …+ },
10 …+ "dependencies": {
11 …+ },
12 …+ "devDependencies": {
13 …+ },
14 …+ "scripts": {
15 …+ "test": "set -e; for t in test/*.js; do node $t; done"
16 …+ },
17 …+ "author": "'Dominic Tarr' <dominic.tarr@gmail.com> (dominictarr.com)",
18 …+ "license": "MIT"
19 …+}
test/bulk.jsView
@@ -1,0 +1,39 @@
1 …+var Log = require('flumelog-offset')
2 …+var Flume = require('flumedb')
3 …+var crypto = require('crypto')
4 …+
5 …+function hash (v) {
6 …+ return crypto.createHash('sha256')
7 …+ .update(v.toString())
8 …+ .digest().readUInt32BE(0)
9 …+}
10 …+
11 …+
12 …+var db = Flume(
13 …+ Log(
14 …+ '/tmp/test-flumeview-hashtable'+Date.now(),
15 …+ {blockSize: 1024, codec: require('flumecodec/json')}
16 …+ ))
17 …+ .use('index', require('../')(1, hash))
18 …+
19 …+ var N = 10000
20 …+ var data = []
21 …+ for(var i = 0; i < N; i++)
22 …+ data.push({key: '#'+i, value: {
23 …+ foo: Math.random(), bar: Date.now()
24 …+ }})
25 …+
26 …+ db.append(data, function (err, offset) {
27 …+ //ordered reads
28 …+ var start = Date.now(), n = 0
29 …+ for(var i = 0; i < N; i++)
30 …+ db.index.get('#'+i, next)
31 …+ function next () {
32 …+ if(++n == N)
33 …+ console.log('done', Date.now() - start)
34 …+ }
35 …+ })
36 …+
37 …+
38 …+
39 …+
test/flumeview-index.jsView
@@ -1,0 +1,20 @@
1 …+
2 …+var Log = require('flumelog-offset')
3 …+var Flume = require('flumedb')
4 …+var crypto = require('crypto')
5 …+function hash (v) {
6 …+ return crypto.createHash('sha256')
7 …+ .update(v.toString())
8 …+ .digest().readUInt32BE(0)
9 …+}
10 …+
11 …+require('test-flumeview-index')(
12 …+ Flume(
13 …+ Log(
14 …+ '/tmp/test-flumeview-hashtable'+Date.now(),
15 …+ {blockSize: 1024, codec: require('flumecodec/json')}
16 …+ ))
17 …+ .use('index', require('../')(1, hash))
18 …+)
19 …+
20 …+
test/index.jsView
@@ -1,0 +1,40 @@
1 …+
2 …+var tape = require('tape')
3 …+var crypto = require('crypto')
4 …+
5 …+function hash (v) {
6 …+ return crypto.createHash('sha256').update(v.toString()).digest().readUInt32BE(0)
7 …+}
8 …+
9 …+function matches (k, data) {
10 …+ return k === hash(data+'value')
11 …+}
12 …+
13 …+function get(k, cb) {
14 …+ cb(null, k)
15 …+}
16 …+
17 …+tape('random integers', function (t) {
18 …+
19 …+ var ht = require('../hashtable')(hash, matches, get)(null, 2000)
20 …+
21 …+ var r = hash('hello random'+Date.now())
22 …+
23 …+ var k = ht.add('random2'+Date.now(), r)
24 …+ t.equal(ht._get(k), r)
25 …+ console.log(k, r)
26 …+
27 …+ for(var i = 0; i < 1000; i++) {
28 …+ var k = ht.add(i, hash(i+'value'))
29 …+ ht.get(i, function (err, j) {
30 …+ if(err) throw err
31 …+ t.equal(j, hash(i+'value'))
32 …+ })
33 …+ t.notEqual(k, i)
34 …+ }
35 …+ console.log(ht.buffer.toString('hex'))
36 …+ t.end()
37 …+
38 …+})
39 …+
40 …+

Built with git-ssb-web