git ssb - Merge pull request #11 from dfinity/remote-datastore · wanderer🌟/js-dfinity-radix-tree@523fd9bd

add remote datastore

wanderer authored on 4/5/2018, 7:41:17 PM
GitHub committed on 4/5/2018, 7:41:17 PM
Parent: ccf8f0f9324db6b24936206bd97e1f8554e7dacb
Parent: eb2b91e814ad8589fb6c3b7c0c1d9714a040b688

package-lock.jsonView
The diff is too large to show. Use a local git client to view these changes. Old file size: 358065 bytes New file size: 359194 bytes

package.jsonView
		@@ -20,19 +20,24 @@
20	20	],
21	21	"devDependencies": {
22	22	"coveralls": "^3.0.0",
23	23	"documentation": "^6.1.0",
	24	+ "fs-extra": "^5.0.0",
24	25	"level-browserify": "^1.1.2",
25	26	"nyc": "^11.6.0",
26	27	"standard": "^11.0.1",
27	28	"tape": "^4.6.3"
28	29	},
29	30	"dependencies": {
30	31	"borc": "git+https://github.com/dignifiedquire/borc.git#fix/nested-array",
31	32	"ipld-graph-builder": "^1.3.8",
	33	+ "node-fetch": "^2.1.2",
32	34	"text-encoding": "^0.6.4",
33	35	"uint1array": "^1.0.5"
34	36	},
	37	+ "browser": {
	38	+ "node-fetch": "./fetch.js"
	39	+ },
35	40	"standard": {
36	41	"ignore": [
37	42	"/benchmark/"
38	43	]

tests/index.jsView
		@@ -1,8 +1,11 @@
	1	+const fs = require('fs-extra')
1	2	const tape = require('tape')
2	3	const crypto = require('crypto')
3	4	const level = require('level-browserify')
4	5	const RadixTree = require('../')
	6	+const RemoteDataStore = require('../remoteDatastore')
	7	+const remote = require('./remote')
5	8	const db = level('./testdb')
6	9
7	10	tape('root existance', async t => {
8	11	let tree = new RadixTree({
		@@ -202,4 +205,35 @@
202	205	t.deepEquals(tree._root['/'], RadixTree.emptyTreeState)
203	206
204	207	t.end()
205	208	})
	209	+
	210	+tape('remote', async t => {
	211	+ // remote
	212	+ const remoteTree = new RadixTree({
	213	+ db: db
	214	+ })
	215	+ const server = remote.listen(db)
	216	+
	217	+ const entries = 100
	218	+ for (let i = 0; i < entries; i++) {
	219	+ const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20)
	220	+ remoteTree.set(key, Buffer.from([i]))
	221	+ }
	222	+ const stateRoot = await remoteTree.flush()
	223	+
	224	+ // local
	225	+ fs.removeSync('./localdb')
	226	+ const localTree = new RadixTree({
	227	+ dag: new RemoteDataStore(level('./localdb'), {uri: 'http://localhost:3000'})
	228	+ })
	229	+ localTree.root = stateRoot
	230	+
	231	+ for (let i = 0; i < entries; i++) {
	232	+ const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20)
	233	+ const value = await localTree.get(key)
	234	+ t.equals(value.value[0], i)
	235	+ }
	236	+
	237	+ server.close()
	238	+ t.end()
	239	+})

tests/remote.jsView
		@@ -1,0 +1,26 @@
	1	+const RadixTree = require('../')
	2	+
	3	+const cbor = require('borc')
	4	+const http = require('http')
	5	+
	6	+let tree
	7	+
	8	+const server = http.createServer(async (req, res) => {
	9	+ const key = Buffer.from(req.url.slice(1), 'hex')
	10	+ const value = await tree.graph._dag.get(key)
	11	+ res.end(cbor.encode(value).toString('base64'))
	12	+})
	13	+
	14	+module.exports = {
	15	+ listen: (db, port = 3000) => {
	16	+ server.listen(port, (err) => {
	17	+ if (err) { return console.error(err) }
	18	+
	19	+ tree = new RadixTree({db})
	20	+
	21	+ console.log(`server is listening on ${port}`)
	22	+ })
	23	+
	24	+ return server
	25	+ }
	26	+}

fetch.jsView
		@@ -1,0 +1,1 @@
	1	+module.exports = window.fetch

remoteDatastore.jsView
		@@ -1,0 +1,52 @@
	1	+const Buffer = require('safe-buffer').Buffer
	2	+const TreeDAG = require('./datastore.js')
	3	+const fetch = require('node-fetch')
	4	+
	5	+module.exports = class RemoteTreeDAG extends TreeDAG {
	6	+ /**
	7	+ * @param dag {object} a level db instance
	8	+ * @param remoteOpts
	9	+ * @param remoteOpts.uri {string} the HTTP uri which has an interface: GET /:key -> value
	10	+ * @param remoteOpts.encoding {string} the encoding of the reponse
	11	+ * @param opts.decoder {object} a cbor decoder
	12	+ */
	13	+ constructor (dag, remoteOpts, decoder) {
	14	+ super(dag, decoder)
	15	+ this.remoteOpts = Object.assign({
	16	+ uri: null,
	17	+ encoding: 'base64'
	18	+ }, remoteOpts)
	19	+ }
	20	+
	21	+ async get (link) {
	22	+ try {
	23	+ return await super.get(link)
	24	+ } catch (e) {
	25	+ if (this.remoteOpts.uri) {
	26	+ await this.fetchRemote(link)
	27	+ return super.get(link)
	28	+ }
	29	+ }
	30	+ }
	31	+
	32	+ fetchRemote (key) {
	33	+ if (!Buffer.isBuffer(key)) {
	34	+ key = Buffer.from(key.buffer)
	35	+ }
	36	+
	37	+ const route = `${this.remoteOpts.uri}/${key.toString('hex')}`
	38	+ return fetch(route)
	39	+ .then(res => res.text())
	40	+ .then(text => {
	41	+ const encoded = Buffer.from(text, this.remoteOpts.encoding)
	42	+ return new Promise((resolve, reject) => {
	43	+ this._dag.put(key, encoded.toString('hex'), () => {
	44	+ resolve(key)
	45	+ })
	46	+ })
	47	+ })
	48	+ .catch(err => {
	49	+ console.warn(`error fetching ${route}:`, err.message)
	50	+ })
	51	+ }
	52	+}

wanderer🌟 / js-dfinity-radix-tree