Commit b87b2dc901b0f767b9b36c8aa44f17c94afb8e6a
remove old blobs stuff
Dominic Tarr committed on 6/9/2016, 12:09:20 AMParent: 7804cdeac140e1a4597b0d68f9fcf8185ef7aab8
Files changed
| plugins/blobs.md | deleted |
| plugins/blobs/index.js | deleted |
| plugins/blobs/queue.js | deleted |
| plugins/blobs/quota.js | deleted |
| plugins/blobs/replication.js | deleted |
| test/blobs-quota.js | deleted |
| test/blobs.js | deleted |
| test/blobs2.js | deleted |
| plugins/blobs.md | ||
|---|---|---|
| @@ -1,146 +1,0 @@ | ||
| 1 | -# scuttlebot blobs plugin | |
| 2 | - | |
| 3 | -Send/receive files by content-hashes. | |
| 4 | - | |
| 5 | - | |
| 6 | -How it works: | |
| 7 | - | |
| 8 | - * Get list of wanted blobs via `links`, or explicit calls to `want`. | |
| 9 | - Call `wL.queue(hash)`. | |
| 10 | - * connected to a peer (managed by gossip plugin): rpc.has wants, | |
| 11 | - and subscribe to their blob changes. call `query` on each new connection. | |
| 12 | - * when a new message is queued. call `query` | |
| 13 | - *. in `download`, 5 workers try to download a blob every 300 ms. | |
| 14 | - * `hash` arguments must be valid ssb blob links `&<base64 hash>.sha256` | |
| 15 | - A queued blob has a callback. | |
| 16 | - | |
| 17 | ---- | |
| 18 | - | |
| 19 | -better design: | |
| 20 | - | |
| 21 | - each task has it's own queue. | |
| 22 | - first is queue for has | |
| 23 | - then is queue for download. | |
| 24 | - | |
| 25 | - once you know where a file is, move it to the download queue. | |
| 26 | - if there arn't any peers to get a file from, put it back in has queue. | |
| 27 | - | |
| 28 | - | |
| 29 | - | |
| 30 | -## get: source | |
| 31 | - | |
| 32 | -Get a blob by its ID. | |
| 33 | - | |
| 34 | -```bash | |
| 35 | -get {blobid} | |
| 36 | -``` | |
| 37 | - | |
| 38 | -```js | |
| 39 | -get(blobid) | |
| 40 | -``` | |
| 41 | - | |
| 42 | - | |
| 43 | -## has: async | |
| 44 | - | |
| 45 | -Check if the blob of the given ID is stored in the DB. | |
| 46 | - | |
| 47 | -```bash | |
| 48 | -has {blobid} | |
| 49 | -``` | |
| 50 | - | |
| 51 | -```js | |
| 52 | -has(blobid, cb) | |
| 53 | -``` | |
| 54 | - | |
| 55 | - | |
| 56 | - | |
| 57 | -## add: sink | |
| 58 | - | |
| 59 | -Add a new blob to the DB. | |
| 60 | - | |
| 61 | -```bash | |
| 62 | -cat ./file | add [hash] | |
| 63 | -``` | |
| 64 | - | |
| 65 | -```js | |
| 66 | -pull(source, add(hash, cb)) | |
| 67 | -``` | |
| 68 | - | |
| 69 | -- hash (base64 string): Optional, expected hash of the file. If the file does not match the hash, it is not stored, and an error is emitted. | |
| 70 | - | |
| 71 | - | |
| 72 | -## rm: async | |
| 73 | - | |
| 74 | -Remove a blob from the store. | |
| 75 | - | |
| 76 | -```bash | |
| 77 | -rm hash | |
| 78 | -``` | |
| 79 | - | |
| 80 | -```js | |
| 81 | -rm(hash, cb) | |
| 82 | -``` | |
| 83 | - | |
| 84 | -- hash (base64 string): hash of the file. | |
| 85 | - | |
| 86 | - | |
| 87 | - | |
| 88 | -## ls: source | |
| 89 | - | |
| 90 | -List the hashes of the blobs in the DB. | |
| 91 | - | |
| 92 | -```bash | |
| 93 | -ls | |
| 94 | -``` | |
| 95 | - | |
| 96 | -```js | |
| 97 | -ls() | |
| 98 | -``` | |
| 99 | - | |
| 100 | - | |
| 101 | - | |
| 102 | -## want: async | |
| 103 | - | |
| 104 | -Begin searching the network for the blob of the given hash. | |
| 105 | - | |
| 106 | -```bash | |
| 107 | -want {hash} [--nowait] | |
| 108 | -``` | |
| 109 | - | |
| 110 | -```js | |
| 111 | -want(hash, { nowait: }, cb) | |
| 112 | -``` | |
| 113 | - | |
| 114 | -By default, `want` will not call the `cb` until the blob has been downloaded. | |
| 115 | -If you want the `cb` to be called immediately, specify `nowait: true`. | |
| 116 | -The `cb` will be called with true/false as the value, telling you if the blob was already present. | |
| 117 | - | |
| 118 | - | |
| 119 | - | |
| 120 | -## wants: sync | |
| 121 | - | |
| 122 | -List the currently-wanted blobs' data-structures. | |
| 123 | - | |
| 124 | -```bash | |
| 125 | -wants | |
| 126 | -``` | |
| 127 | - | |
| 128 | -```js | |
| 129 | -wants() | |
| 130 | -``` | |
| 131 | - | |
| 132 | - | |
| 133 | - | |
| 134 | -## changes: source | |
| 135 | - | |
| 136 | -Listen for any newly-downloaded blobs. | |
| 137 | - | |
| 138 | -```bash | |
| 139 | -changes | |
| 140 | -``` | |
| 141 | - | |
| 142 | -```js | |
| 143 | -changes() | |
| 144 | -``` | |
| 145 | - | |
| 146 | -When a blob is downloaded, this stream will emit the hash of the blob. | |
| plugins/blobs/index.js | ||
|---|---|---|
| @@ -1,146 +1,0 @@ | ||
| 1 | -'use strict' | |
| 2 | - | |
| 3 | -var Blobs = require('multiblob') | |
| 4 | -var path = require('path') | |
| 5 | -var pull = require('pull-stream') | |
| 6 | -var isBlob = require('ssb-ref').isBlobId | |
| 7 | -var Quota = require('./quota') | |
| 8 | -var Notify = require('pull-notify') | |
| 9 | -var mdm = require('mdmanifest') | |
| 10 | -var valid = require('../../lib/validators') | |
| 11 | -var apidoc = require('../../lib/apidocs').blobs | |
| 12 | -var Replicate = require('./replication') | |
| 13 | - | |
| 14 | -var mbu = require('multiblob/util') | |
| 15 | - | |
| 16 | -// blobs plugin | |
| 17 | -// methods to read/write the blobstore | |
| 18 | -// and automated blob-fetching from the network | |
| 19 | - | |
| 20 | -function isFunction (f) { | |
| 21 | - return 'function' === typeof f | |
| 22 | -} | |
| 23 | - | |
| 24 | -function _desigil (hash) { | |
| 25 | - return isBlob(hash) ? hash.substring(1) : hash | |
| 26 | -} | |
| 27 | - | |
| 28 | -function _resigil (hash) { | |
| 29 | - return isBlob(hash) ? hash : '&'+hash | |
| 30 | -} | |
| 31 | - | |
| 32 | -function isString (s) { | |
| 33 | - return 'string' === typeof s | |
| 34 | -} | |
| 35 | - | |
| 36 | -//desigil = _desigil | |
| 37 | -// | |
| 38 | -//resigil = _resigil | |
| 39 | - | |
| 40 | - | |
| 41 | -module.exports = { | |
| 42 | - name: 'blobs', | |
| 43 | - version: '0.0.0', | |
| 44 | - manifest: mdm.manifest(apidoc), | |
| 45 | - permissions: { | |
| 46 | - anonymous: {allow: ['has', 'get', 'changes']}, | |
| 47 | - }, | |
| 48 | - init: function (sbot, opts) { | |
| 49 | - | |
| 50 | - var notify = Notify() | |
| 51 | - var config = opts | |
| 52 | - | |
| 53 | - var blobs = sbot._blobs = Blobs({ | |
| 54 | - dir: path.join(config.path, 'blobs'), | |
| 55 | - hash: 'sha256', | |
| 56 | - encode: function (buf, alg) { | |
| 57 | - return _resigil(mbu.encode(buf, alg)) | |
| 58 | - }, | |
| 59 | - decode: function (str) { | |
| 60 | - return mbu.decode(_desigil(str)) | |
| 61 | - }, | |
| 62 | - isHash: isBlob | |
| 63 | - }) | |
| 64 | - | |
| 65 | - var userQuotas = {} // map of { feedId => quotaUsage }, for rate-limiting | |
| 66 | - var drain = Quota(sbot, blobs, userQuotas) | |
| 67 | - var wantList = Replicate(sbot, config, notify, userQuotas) | |
| 68 | - | |
| 69 | - return { | |
| 70 | - get: valid.source(blobs.get, 'blobId'), | |
| 71 | - | |
| 72 | - has: valid.async(function (hash, cb) { | |
| 73 | - //emit blobs:has event when this api is called remotely. | |
| 74 | - //needed to make tests pass. should probably remove this. | |
| 75 | - if(this.id) sbot.emit('blobs:has', hash) | |
| 76 | - blobs.has(hash, cb) | |
| 77 | - }, 'blobId|array'), | |
| 78 | - | |
| 79 | - size: valid.async(blobs.size, 'blobId|array'), | |
| 80 | - | |
| 81 | - add: valid.sink(function (hash, cb) { | |
| 82 | - // cb once blob is successfully added. | |
| 83 | - // sink cbs are not exposed over rpc | |
| 84 | - // so this is only available when using this api | |
| 85 | - if(isFunction(hash)) cb = hash, hash = null | |
| 86 | - | |
| 87 | - return blobs.add(hash, function (err, hash) { | |
| 88 | - if(!err) { | |
| 89 | - sbot.emit('blobs:got', hash) | |
| 90 | - notify(hash) | |
| 91 | - //wait until quotas have been calculated | |
| 92 | - //befor returning (tests will fail without this) | |
| 93 | - if(cb) drain(function () { | |
| 94 | - cb(null, hash) | |
| 95 | - }) | |
| 96 | - } | |
| 97 | - else { | |
| 98 | - if(cb) cb(err, hash) | |
| 99 | - else console.error(err.stack) | |
| 100 | - } | |
| 101 | - }) | |
| 102 | - }, 'string?'), | |
| 103 | - | |
| 104 | - rm: valid.async(blobs.rm, 'string'), | |
| 105 | - | |
| 106 | - ls: blobs.ls, | |
| 107 | - // request to retrieve a blob, | |
| 108 | - // calls back when that file is available. | |
| 109 | - // - `opts.nowait`: call cb immediately if not found (dont register for callback) | |
| 110 | - want: valid.async(function (hash, opts, cb) { | |
| 111 | - if (isFunction(opts)) { | |
| 112 | - cb = opts | |
| 113 | - opts = null | |
| 114 | - } | |
| 115 | - var id = this.id | |
| 116 | - if(!isBlob(hash)) return cb(new Error('not a hash:' + hash)) | |
| 117 | - | |
| 118 | - sbot.emit('blobs:wants', hash) | |
| 119 | - blobs.has(hash, function (_, has) { | |
| 120 | - if (has) return cb(null, true) | |
| 121 | - // update queue | |
| 122 | - wantList.want(hash, id, cb) | |
| 123 | - }) | |
| 124 | - }, 'blobId', 'object?'), | |
| 125 | - | |
| 126 | - changes: notify.listen, | |
| 127 | - | |
| 128 | - quota: valid.sync(function (id) { | |
| 129 | - return wantList.quota(id) | |
| 130 | - }, 'feedId'), | |
| 131 | - | |
| 132 | - // get current want list | |
| 133 | - wants: function () { | |
| 134 | - return wantList.jobs | |
| 135 | - } | |
| 136 | - } | |
| 137 | - } | |
| 138 | -} | |
| 139 | - | |
| 140 | - | |
| 141 | - | |
| 142 | - | |
| 143 | - | |
| 144 | - | |
| 145 | - | |
| 146 | - | |
| plugins/blobs/queue.js | ||
|---|---|---|
| @@ -1,128 +1,0 @@ | ||
| 1 | -// Queue | |
| 2 | -// returns a function which... | |
| 3 | -// - only acts if not already acting | |
| 4 | -// - automatically requeues if the task is not yet done | |
| 5 | -// - `delay`: ms, amount of time to wait before calling again | |
| 6 | -// - `n`: number, amount of simultaneous calls allowed | |
| 7 | -// - `label`: string, name of the task (for logging) | |
| 8 | -// - `fun`: function(cb(done?)), calls cb(true) when done, cb(false) when needs to requeue | |
| 9 | - | |
| 10 | -function isFunction (f) { | |
| 11 | - return 'function' === typeof f | |
| 12 | -} | |
| 13 | - | |
| 14 | -function Work(avgWait, n, label, fun) { | |
| 15 | - n = 1 | |
| 16 | - var doing = 0, timeout | |
| 17 | - | |
| 18 | - var timers = [] | |
| 19 | - | |
| 20 | - function clear (timer) { | |
| 21 | - var i = timers.indexOf(timer) | |
| 22 | - clearTimeout(timer[i]) | |
| 23 | - times.splice(i, 1) | |
| 24 | - } | |
| 25 | - | |
| 26 | - function delay (job, d) { | |
| 27 | - var i | |
| 28 | - var timer = setTimeout(function () { | |
| 29 | - timers.splice(timers.indexOf(timer), 1); job() | |
| 30 | - }, d) | |
| 31 | - timer.unref() | |
| 32 | - timers.push(timer) | |
| 33 | - | |
| 34 | - return timer | |
| 35 | - } | |
| 36 | - | |
| 37 | - function job () { | |
| 38 | - // abort if already doing too many | |
| 39 | - | |
| 40 | - if(doing >= n) return | |
| 41 | - doing++ | |
| 42 | - | |
| 43 | - // run the behavior | |
| 44 | - fun(function (done) { | |
| 45 | - doing-- | |
| 46 | - if(done) { | |
| 47 | - // we're done, dont requeue | |
| 48 | - return | |
| 49 | - } | |
| 50 | - | |
| 51 | - // requeue after a delay | |
| 52 | - var wait = ~~(avgWait/2 + avgWait*Math.random()) | |
| 53 | - delay(job, wait) | |
| 54 | - }) | |
| 55 | - } | |
| 56 | - | |
| 57 | - job.abort = function () { | |
| 58 | - timers.forEach(clear) | |
| 59 | - } | |
| 60 | - | |
| 61 | - return job | |
| 62 | -} | |
| 63 | - | |
| 64 | -function find (jobs, test) { | |
| 65 | - for(var k in jobs) | |
| 66 | - if(test(jobs[k])) return k | |
| 67 | - return -1 | |
| 68 | -} | |
| 69 | - | |
| 70 | -function max (jobs, test) { | |
| 71 | - var M = -Infinity, i = -1 | |
| 72 | - for(var k in jobs) { | |
| 73 | - var m = test(jobs[k], k, jobs) | |
| 74 | - if(m > M) { | |
| 75 | - M = m | |
| 76 | - i = k | |
| 77 | - } | |
| 78 | - } | |
| 79 | - return k | |
| 80 | -} | |
| 81 | - | |
| 82 | -module.exports = function (work) { | |
| 83 | - | |
| 84 | - var jobs = [] | |
| 85 | - | |
| 86 | - var queue = { | |
| 87 | - push: function (job) { | |
| 88 | - jobs.push(job) | |
| 89 | - }, | |
| 90 | - | |
| 91 | - pull: function (id) { | |
| 92 | - var test = isFunction(id) ? id : function (e) { return e.id === id } | |
| 93 | - if(!this.length()) return | |
| 94 | - if(!id) | |
| 95 | - return jobs.shift() | |
| 96 | - else { | |
| 97 | - var index = find(jobs, test) | |
| 98 | - if(~index) return jobs.splice(index, 1)[0] | |
| 99 | - } | |
| 100 | - }, | |
| 101 | - | |
| 102 | - each: function (iter) { | |
| 103 | - jobs.forEach(iter) | |
| 104 | - }, | |
| 105 | - | |
| 106 | - length: function () { | |
| 107 | - return jobs.length | |
| 108 | - }, | |
| 109 | - | |
| 110 | - toJSON: function () { | |
| 111 | - return jobs.slice() | |
| 112 | - } | |
| 113 | - } | |
| 114 | - | |
| 115 | - Work(100, 2, null, function (done) { | |
| 116 | - if(!queue.length()) return done() | |
| 117 | - work(queue, done) | |
| 118 | - }) () | |
| 119 | - | |
| 120 | - return queue | |
| 121 | -} | |
| 122 | - | |
| 123 | - | |
| 124 | - | |
| 125 | - | |
| 126 | - | |
| 127 | - | |
| 128 | - | |
| plugins/blobs/quota.js | ||
|---|---|---|
| @@ -1,62 +1,0 @@ | ||
| 1 | -var pull = require('pull-stream') | |
| 2 | -var paramap = require('pull-paramap') | |
| 3 | - | |
| 4 | -module.exports = function (sbot, blobs, userQuotas, cb) { | |
| 5 | - var listeners = [] | |
| 6 | - | |
| 7 | - //recalculate the quota, with live updates. | |
| 8 | - | |
| 9 | - // share a file size between the feeds that link to it. | |
| 10 | - // At the time we download it. | |
| 11 | - | |
| 12 | - // More feeds might link to it later, and these | |
| 13 | - // won't be included in the calculation. | |
| 14 | - // but it's simplest to do it this way. | |
| 15 | - | |
| 16 | - // this is only in memory, so it will be recalculated | |
| 17 | - // when sbot is restarted. | |
| 18 | - | |
| 19 | - var total = 0 | |
| 20 | - var start = Date.now() | |
| 21 | - var inflight = 0 | |
| 22 | - | |
| 23 | - pull( | |
| 24 | - blobs.ls({long: true, live: true}), | |
| 25 | - paramap(function (data, cb) { | |
| 26 | - if(data.sync) return cb(null, data) | |
| 27 | - | |
| 28 | - var acc = {}, count = 0 | |
| 29 | - total += data.size | |
| 30 | - | |
| 31 | - inflight ++ | |
| 32 | - pull( | |
| 33 | - sbot.links({dest: data.id}), | |
| 34 | - pull.drain(function (link) { | |
| 35 | - if(!acc[link.source]) { | |
| 36 | - acc[link.source] = true | |
| 37 | - count ++ | |
| 38 | - } | |
| 39 | - return acc | |
| 40 | - }, function (err) { | |
| 41 | - inflight -- | |
| 42 | - if(err) return cb(err) | |
| 43 | - var size = data.size | |
| 44 | - for(var k in acc) | |
| 45 | - userQuotas[k] = (userQuotas[k] || 0) + size/count | |
| 46 | - | |
| 47 | - if(inflight === 0) | |
| 48 | - while (listeners.length) listeners.shift()() | |
| 49 | - | |
| 50 | - cb(null, data) | |
| 51 | - }) | |
| 52 | - ) | |
| 53 | - }), | |
| 54 | - pull.drain() | |
| 55 | - ) | |
| 56 | - | |
| 57 | - return function (listener) { | |
| 58 | - if(!inflight) listener() | |
| 59 | - else listeners.push(listener) | |
| 60 | - } | |
| 61 | -} | |
| 62 | - | |
| plugins/blobs/replication.js | ||
|---|---|---|
| @@ -1,232 +1,0 @@ | ||
| 1 | -var pull = require('pull-stream') | |
| 2 | -var Queue = require('./queue') | |
| 3 | - | |
| 4 | -function each (obj, iter) { | |
| 5 | - for(var k in obj) | |
| 6 | - iter(obj[k], k, obj) | |
| 7 | -} | |
| 8 | - | |
| 9 | -function first (obj, test) { | |
| 10 | - var v | |
| 11 | - for (var k in obj) | |
| 12 | - if(v = test(obj[k], k, obj)) | |
| 13 | - return v | |
| 14 | -} | |
| 15 | - | |
| 16 | -function union (a, b) { | |
| 17 | - b = toArray(b) | |
| 18 | - a = toArray(a) | |
| 19 | - if(!a.length) return b | |
| 20 | - if(a.length < b.length) { | |
| 21 | - var t = b; b = a; a = t | |
| 22 | - } | |
| 23 | - b.forEach(function (e) { | |
| 24 | - if(!~a.indexOf(e)) a.push(e) | |
| 25 | - }) | |
| 26 | - return a | |
| 27 | -} | |
| 28 | - | |
| 29 | -function toArray (s) { | |
| 30 | - return s != null ? (Array.isArray(s) ? s : [s]) : [] | |
| 31 | -} | |
| 32 | - | |
| 33 | -function isFunction (f) { | |
| 34 | - return 'function' === typeof f | |
| 35 | -} | |
| 36 | - | |
| 37 | -var MB = 1024*1024 | |
| 38 | -//default replication limits. | |
| 39 | -var defaults = {limit: [-1, 100*MB, 20*MB], minLimit: 5*MB} | |
| 40 | -module.exports = function (sbot, opts, notify, userQuotas) { | |
| 41 | - var jobs = {}, hasQueue, getQueue | |
| 42 | - var conf = opts.blobs || defaults, wl | |
| 43 | - | |
| 44 | - //keep track of who is over quota so that it doesn't get logged again and again. | |
| 45 | - var over = {} | |
| 46 | - | |
| 47 | - // calculate quotas for each feed. | |
| 48 | - // start with size of each blob | |
| 49 | - // divided between the feeds that mention it. | |
| 50 | - // getting a use for each feed. | |
| 51 | - | |
| 52 | - function createJob(id, owner, cb) { | |
| 53 | - console.log('CREAT', id) | |
| 54 | - toArray(owner).forEach(function (e) { | |
| 55 | - if(e[0] !== '@') throw new Error('not a owner:'+e) | |
| 56 | - }) | |
| 57 | - if(jobs[id]) { | |
| 58 | - jobs[id].owner = union(jobs[id].owner, owner || []) | |
| 59 | - jobs[id].cbs.push(cb) | |
| 60 | - return | |
| 61 | - } | |
| 62 | - hasQueue.push(jobs[id] = { | |
| 63 | - id: id, has: {}, owner: toArray(owner), | |
| 64 | - cbs: cb ? [cb] : [], done: false | |
| 65 | - }) | |
| 66 | - } | |
| 67 | - | |
| 68 | - function finishJob(job) { | |
| 69 | - if(!job) return | |
| 70 | - delete jobs[job.id] | |
| 71 | - while(job.cbs && job.cbs.length) { | |
| 72 | - var cb = job.cbs.shift() | |
| 73 | - if(isFunction(cb)) cb(null, true) | |
| 74 | - } | |
| 75 | - } | |
| 76 | - | |
| 77 | - //if want is called and then it's locally added, | |
| 78 | - //handle that by calling back want. | |
| 79 | - pull(notify.listen(), pull.drain(function (hash) { | |
| 80 | - finishJob(hasQueue.pull(hash) || getQueue.pull(hash)) | |
| 81 | - })) | |
| 82 | - | |
| 83 | - function hasPeers () { | |
| 84 | - return Object.keys(sbot.peers).length !== 0 | |
| 85 | - } | |
| 86 | - | |
| 87 | - function hops (id) { | |
| 88 | - var p = sbot.friends.path({ | |
| 89 | - source: sbot.id, dest: id, hops: conf.limit.length | |
| 90 | - }) | |
| 91 | - return p ? p.length - 1 : -1 | |
| 92 | - } | |
| 93 | - | |
| 94 | - function limitFor(id) { | |
| 95 | - if(opts.party) return -1 | |
| 96 | - var h = hops(id) | |
| 97 | - if(hops === -1) return conf.minLimit | |
| 98 | - return conf.limit[h] || conf.minLimit | |
| 99 | - } | |
| 100 | - | |
| 101 | - function filter (job) { | |
| 102 | - //set config.blobs.party = true | |
| 103 | - //to disable all quotas. | |
| 104 | - if(conf.party) return true | |
| 105 | - return job.owner.every(function (id) { | |
| 106 | - var l = limitFor(id) | |
| 107 | - if(l < 0) return true | |
| 108 | - else if ((userQuotas[id] || 0) < l) | |
| 109 | - return true | |
| 110 | - else if(!over[id]) { | |
| 111 | - over[id] = userQuotas[id] | |
| 112 | - console.log('Over Quota:', id, wl.quota(id)) | |
| 113 | - } | |
| 114 | - }) | |
| 115 | - } | |
| 116 | - | |
| 117 | - hasQueue = Queue(function (_, done) { | |
| 118 | - //check if there is a something in the has queue. | |
| 119 | - //filter out cases where work is impossible... | |
| 120 | - //(empty queue, or no peers) | |
| 121 | - if(!hasPeers()) return done() | |
| 122 | - | |
| 123 | - var job = hasQueue.pull(filter) | |
| 124 | - | |
| 125 | - if(!job || job.done) return done() | |
| 126 | - | |
| 127 | - var n = 0, found = false | |
| 128 | - each(sbot.peers, function (peers, id) { | |
| 129 | - if(('undefined' !== typeof job.has[id]) || !peers[0]) return | |
| 130 | - n++ | |
| 131 | - peers[0].blobs.has(job.id, function (err, has) { | |
| 132 | - found = found || (job.has[id] = has) | |
| 133 | - if(--n) return | |
| 134 | - next() | |
| 135 | - }) | |
| 136 | - }) | |
| 137 | - if(!n) return hasQueue.push(job), done() | |
| 138 | - | |
| 139 | - function next () { | |
| 140 | - (found ? getQueue : hasQueue).push(job) | |
| 141 | - done() | |
| 142 | - } | |
| 143 | - }) | |
| 144 | - | |
| 145 | - getQueue = Queue(function (_, done) { | |
| 146 | - if(!hasPeers()) return done() | |
| 147 | - | |
| 148 | - //check if this file is over quota. | |
| 149 | - var job = getQueue.pull(filter) | |
| 150 | - if(!job) return done() | |
| 151 | - //this covers weird edgecase where a blob is added | |
| 152 | - //while something is looking for it. covered in | |
| 153 | - //test/blobs2.js | |
| 154 | - if(job.done) { | |
| 155 | - delete jobs[job.id] | |
| 156 | - return done() | |
| 157 | - } | |
| 158 | - | |
| 159 | - var remote = first(job.has, function (has, id) { | |
| 160 | - if (has) | |
| 161 | - return getPeer(id) | |
| 162 | - }) | |
| 163 | - | |
| 164 | - if(!remote) { | |
| 165 | - hasQueue.push(job); return done() | |
| 166 | - } | |
| 167 | - | |
| 168 | - pull( | |
| 169 | - remote.blobs.get(job.id), | |
| 170 | - //only accept blobs that have the correct size. | |
| 171 | - sbot.blobs.add(job.id, function (err) { | |
| 172 | - if(!err) { | |
| 173 | - finishJob(job) | |
| 174 | - return done() //success | |
| 175 | - } | |
| 176 | - // remove the remote, it may be misbehaving | |
| 177 | - delete job.has[remote.id] | |
| 178 | - // put it back on the get or has queue | |
| 179 | - if(Object.keys(job.has).length) getQueue.push(job) | |
| 180 | - else hasQueue.push(job) | |
| 181 | - done() | |
| 182 | - }) | |
| 183 | - ) | |
| 184 | - }) | |
| 185 | - | |
| 186 | - function getPeer(id) { | |
| 187 | - return sbot.peers[id] && sbot.peers[id][0] | |
| 188 | - } | |
| 189 | - | |
| 190 | - // monitor the feed for new links to blobs | |
| 191 | - pull( | |
| 192 | - sbot.links({dest: '&', live: true}), | |
| 193 | - pull.drain(function (data) { | |
| 194 | - // do we have the referenced blob yet? | |
| 195 | - sbot.blobs.has(data.dest, function (_, has) { | |
| 196 | - if(!has) createJob(data.dest, data.source) | |
| 197 | - }) | |
| 198 | - }) | |
| 199 | - ) | |
| 200 | - | |
| 201 | - //handle weird edge case where something is added locally | |
| 202 | - //but we are already looking for it because we saw a link. | |
| 203 | - sbot.on('blobs:got', function (hash) { | |
| 204 | - if(jobs[hash]) jobs[hash].done = true | |
| 205 | - }) | |
| 206 | - | |
| 207 | - sbot.on('rpc:connect', function (rpc) { | |
| 208 | - for(id in jobs) | |
| 209 | - if(false === jobs[id].has[rpc.id]) | |
| 210 | - delete jobs[id].has[rpc.id] | |
| 211 | - }) | |
| 212 | - | |
| 213 | - return wl = { | |
| 214 | - has: hasQueue, | |
| 215 | - get: getQueue, | |
| 216 | - want: function (id, owner, cb) { | |
| 217 | - createJob(id, owner || sbot.id, cb) | |
| 218 | - }, | |
| 219 | - quota: function (id) { | |
| 220 | - var l = limitFor(id), q = userQuotas[id] || 0 | |
| 221 | - return { | |
| 222 | - limit: l, | |
| 223 | - usage: q, | |
| 224 | - hops: hops(id), | |
| 225 | - percent: ((q/l)*100).toPrecision(4)+'%' | |
| 226 | - } | |
| 227 | - } | |
| 228 | - } | |
| 229 | -} | |
| 230 | - | |
| 231 | - | |
| 232 | - | |
| test/blobs-quota.js | ||
|---|---|---|
| @@ -1,209 +1,0 @@ | ||
| 1 | -var fs = require('fs') | |
| 2 | -var tape = require('tape') | |
| 3 | -var path = require('path') | |
| 4 | -var toPull = require('stream-to-pull-stream') | |
| 5 | -var pull = require('pull-stream') | |
| 6 | -var cont = require('cont') | |
| 7 | -var ssbKeys = require('ssb-keys') | |
| 8 | -var u = require('./util') | |
| 9 | - | |
| 10 | -var crypto = require('crypto') | |
| 11 | - | |
| 12 | -// create 3 servers | |
| 13 | -// give them all pub servers (on localhost) | |
| 14 | -// and get them to follow each other... | |
| 15 | -var gossip = require('../plugins/gossip') | |
| 16 | -var blobs = require('../plugins/blobs') | |
| 17 | -var friends = require('../plugins/friends') | |
| 18 | -var replicate = require('../plugins/replicate') | |
| 19 | - | |
| 20 | -function read (filename) { | |
| 21 | - return toPull.source(fs.createReadStream(filename)) | |
| 22 | -} | |
| 23 | - | |
| 24 | -function createHash () { | |
| 25 | - var hash = crypto.createHash('sha256') | |
| 26 | - var hasher = pull.through(function (data) { | |
| 27 | - hash.update(data) | |
| 28 | - }, function () { | |
| 29 | - hasher.digest = '&'+hash.digest('base64')+'.sha256' | |
| 30 | - }) | |
| 31 | - return hasher | |
| 32 | -} | |
| 33 | - | |
| 34 | -var createSbot = require('../') | |
| 35 | - .use(friends).use(blobs) | |
| 36 | - | |
| 37 | -function create (name, config) { | |
| 38 | - return createSbot({ | |
| 39 | - temp: 'test-blobs-quotas-' + name, timeout: 1000, | |
| 40 | - keys: ssbKeys.generate(), | |
| 41 | - blobs: config | |
| 42 | - }) | |
| 43 | -} | |
| 44 | - | |
| 45 | -var K = 1024 | |
| 46 | - | |
| 47 | -function test (t, a, e) { | |
| 48 | - t.equal(a.limit, e.limit) | |
| 49 | - t.equal(a.usage, e.usage) | |
| 50 | - t.equal(a.hops, e.hops) | |
| 51 | -} | |
| 52 | - | |
| 53 | -function hash(data) { | |
| 54 | - return '&'+crypto.createHash('sha256').update(data).digest('base64')+'.sha256' | |
| 55 | -} | |
| 56 | - | |
| 57 | -tape('test blob quota api', function (t) { | |
| 58 | - | |
| 59 | - //create 4 feeds on one sbot to test quota apis. | |
| 60 | - var alice = create('alice', { limit: [-1, 4*K, 2*K], minLimit: 256}) | |
| 61 | - var bob = alice.createFeed() | |
| 62 | - var carol = alice.createFeed() | |
| 63 | - var dan = alice.createFeed() | |
| 64 | - | |
| 65 | - t.test('limits based on follows', function (t) { | |
| 66 | - cont.para([ | |
| 67 | - alice.publish(u.follow(bob.id)), | |
| 68 | - bob.add(u.follow(carol.id)), | |
| 69 | - carol.add(u.follow(dan.id)), | |
| 70 | - ])(function (err, data) { | |
| 71 | - if(err) throw err | |
| 72 | - console.log(alice.blobs.quota(alice.id)) | |
| 73 | - console.log(alice.blobs.quota(bob.id)) | |
| 74 | - console.log(alice.blobs.quota(carol.id)) | |
| 75 | - console.log(alice.blobs.quota(dan.id)) | |
| 76 | - | |
| 77 | - test(t, alice.blobs.quota(alice.id), {limit: -1, usage: 0, hops: 0}) | |
| 78 | - test(t, alice.blobs.quota(bob.id), {limit: 4*K, usage: 0, hops: 1}) | |
| 79 | - test(t, alice.blobs.quota(carol.id), {limit: 2*K, usage: 0, hops: 2}) | |
| 80 | - test(t, alice.blobs.quota(dan.id), {limit: 256, usage: 0, hops: 3}) | |
| 81 | - | |
| 82 | - t.end() | |
| 83 | - }) | |
| 84 | - }) | |
| 85 | - | |
| 86 | - t.test('file size is added correctly to a peer', function (t) { | |
| 87 | - var rand = crypto.randomBytes(1024) | |
| 88 | - var h = hash(rand) | |
| 89 | - | |
| 90 | - dan.add(u.file(h), function (err, msg) { | |
| 91 | - if(err) throw err | |
| 92 | - | |
| 93 | - pull( | |
| 94 | - pull.once(rand), | |
| 95 | - alice.blobs.add(function (err, _hash) { | |
| 96 | - if(err) throw err | |
| 97 | - test(t, alice.blobs.quota(dan.id), {limit: 256, usage: 1024, hops: 3}) | |
| 98 | - t.end() | |
| 99 | - }) | |
| 100 | - ) | |
| 101 | - }) | |
| 102 | - }) | |
| 103 | - | |
| 104 | - t.test('file size is divided between peers', function (t) { | |
| 105 | - var rand = crypto.randomBytes(3*K) | |
| 106 | - var h = hash(rand) | |
| 107 | - | |
| 108 | - cont.para([ | |
| 109 | - alice.publish(u.file(h)), | |
| 110 | - bob.add(u.file(h)) | |
| 111 | - ]) (function (err, msg) { | |
| 112 | - if(err) throw err | |
| 113 | - pull( | |
| 114 | - pull.once(rand), | |
| 115 | - alice.blobs.add(function (err, _hash) { | |
| 116 | - if(err) throw err | |
| 117 | - t.equal(_hash, h) | |
| 118 | - test(t, alice.blobs.quota(alice.id), {limit: -1, usage: 1.5*K, hops: 0}) | |
| 119 | - test(t, alice.blobs.quota(bob.id), {limit: 4*K, usage: 1.5*K, hops: 1}) | |
| 120 | - t.end() | |
| 121 | - }) | |
| 122 | - ) | |
| 123 | - }) | |
| 124 | - }) | |
| 125 | - | |
| 126 | - t.test('if over quota, do not download more', function (t) { | |
| 127 | - var rand = crypto.randomBytes(3*K) | |
| 128 | - var h = hash(rand) | |
| 129 | - var h2 = hash(crypto.randomBytes(1*K)) | |
| 130 | - | |
| 131 | - cont.para([ | |
| 132 | - dan.add(u.file(h)), | |
| 133 | - dan.add(u.file(h2)), | |
| 134 | - function (cb) { | |
| 135 | - pull(pull.once(rand), alice.blobs.add(cb)) | |
| 136 | - }, | |
| 137 | - ]) (function (err, msg) { | |
| 138 | - console.log(err) | |
| 139 | - test(t, alice.blobs.quota(dan.id), {limit: 256, usage: 4*K, hops: 3}) | |
| 140 | - t.end() | |
| 141 | - }) | |
| 142 | - }) | |
| 143 | - | |
| 144 | - | |
| 145 | - t.test('cleanup', function (t) { | |
| 146 | - alice.close(true); t.end() | |
| 147 | - }) | |
| 148 | - | |
| 149 | -}) | |
| 150 | - | |
| 151 | -// I used this test to get the blobs replicator to actually go into | |
| 152 | -// overquota situation. There was a bug where it would go into a CPU | |
| 153 | -// spin... but there isn't a good way to actually test that... | |
| 154 | -// but i'll leave this code here just incase. | |
| 155 | - | |
| 156 | -tape('test actual overquota situation', function (t) { | |
| 157 | - var hasher = createHash() | |
| 158 | - | |
| 159 | - var rand = crypto.randomBytes(3*K) | |
| 160 | - var h = hash(rand) | |
| 161 | - var h2 = hash(crypto.randomBytes(3*K)) | |
| 162 | - | |
| 163 | - var bob = create('bob', { limit: [-1, 1*K], minLimit: 256}) | |
| 164 | - var carol = create('carol', { limit: [-1, 1*K], minLimit: 256}) | |
| 165 | - | |
| 166 | - cont.para([ | |
| 167 | - carol.publish(u.file(h)), | |
| 168 | - carol.publish(u.file(h2)), | |
| 169 | - ]) (function (err) { | |
| 170 | - if(err) throw err | |
| 171 | - //copy bob's feed to alice. bob should be over quota now. | |
| 172 | - pull( | |
| 173 | - carol.createHistoryStream({id: carol.id, sequence: 0, keys: false}), | |
| 174 | - pull.through(console.log), | |
| 175 | - bob.createWriteStream(function (err, n) { | |
| 176 | - if(err) throw err | |
| 177 | - pull(bob.createLogStream(), pull.log()) | |
| 178 | - pull(pull.once(rand), bob.blobs.add(function (err) { | |
| 179 | - if(err) throw err | |
| 180 | - var bq = bob.blobs.quota(carol.id) | |
| 181 | - t.ok(bq.limit < bq.usage) | |
| 182 | - bob.close(true); carol.close(true) | |
| 183 | - t.end() | |
| 184 | - })) | |
| 185 | - }) | |
| 186 | - ) | |
| 187 | - }) | |
| 188 | -}) | |
| 189 | - | |
| 190 | - | |
| 191 | - | |
| 192 | - | |
| 193 | - | |
| 194 | - | |
| 195 | - | |
| 196 | - | |
| 197 | - | |
| 198 | - | |
| 199 | - | |
| 200 | - | |
| 201 | - | |
| 202 | - | |
| 203 | - | |
| 204 | - | |
| 205 | - | |
| 206 | - | |
| 207 | - | |
| 208 | - | |
| 209 | - | |
| test/blobs.js | ||
|---|---|---|
| @@ -1,247 +1,0 @@ | ||
| 1 | -var fs = require('fs') | |
| 2 | -var tape = require('tape') | |
| 3 | -var path = require('path') | |
| 4 | -var toPull = require('stream-to-pull-stream') | |
| 5 | -var pull = require('pull-stream') | |
| 6 | -var cont = require('cont') | |
| 7 | -var ssbKeys = require('ssb-keys') | |
| 8 | -var u = require('./util') | |
| 9 | - | |
| 10 | -var crypto = require('crypto') | |
| 11 | - | |
| 12 | -// create 3 servers | |
| 13 | -// give them all pub servers (on localhost) | |
| 14 | -// and get them to follow each other... | |
| 15 | -var gossip = require('../plugins/gossip') | |
| 16 | -var blobs = require('../plugins/blobs') | |
| 17 | -var friends = require('../plugins/friends') | |
| 18 | -var replicate = require('../plugins/replicate') | |
| 19 | - | |
| 20 | -function read (filename) { | |
| 21 | - return toPull.source(fs.createReadStream(filename)) | |
| 22 | -} | |
| 23 | - | |
| 24 | -function createHash () { | |
| 25 | - var hash = crypto.createHash('sha256') | |
| 26 | - var hasher = pull.through(function (data) { | |
| 27 | - hash.update(data) | |
| 28 | - }, function () { | |
| 29 | - hasher.digest = '&'+hash.digest('base64')+'.sha256' | |
| 30 | - }) | |
| 31 | - return hasher | |
| 32 | -} | |
| 33 | - | |
| 34 | -var createSbot = require('../') | |
| 35 | - .use(friends).use(blobs) | |
| 36 | - | |
| 37 | -tape('test blob api', function (t) { | |
| 38 | - var sbot = createSbot({ | |
| 39 | - temp: 'test-blobs-alice', timeout: 1000, | |
| 40 | - keys: ssbKeys.generate() | |
| 41 | - }) | |
| 42 | - | |
| 43 | - t.test(function (t) { | |
| 44 | - pull( | |
| 45 | - read(path.join(__filename)), | |
| 46 | - sbot.blobs.add(function (err, hash) { | |
| 47 | - t.notOk(err) | |
| 48 | - | |
| 49 | - pull( | |
| 50 | - read(path.join(__filename)), | |
| 51 | - sbot.blobs.add(hash, function (err, _hash) { | |
| 52 | - t.notOk(err) | |
| 53 | - t.equal(_hash, hash) | |
| 54 | - | |
| 55 | - pull( | |
| 56 | - pull.values([new Buffer([])]), | |
| 57 | - sbot.blobs.add(hash, function (err) { | |
| 58 | - t.ok(err) | |
| 59 | - t.end() | |
| 60 | - sbot.close(true) | |
| 61 | - }) | |
| 62 | - ) | |
| 63 | - }) | |
| 64 | - ) | |
| 65 | - }) | |
| 66 | - ) | |
| 67 | - }) | |
| 68 | -}) | |
| 69 | - | |
| 70 | -tape('a client can request a blob', function (t) { | |
| 71 | - | |
| 72 | - var sbotA = createSbot({ | |
| 73 | - temp: 'test-blobs-alice0', timeout: 1000, | |
| 74 | - keys: ssbKeys.generate() | |
| 75 | - }) | |
| 76 | - | |
| 77 | - var bob = ssbKeys.generate() | |
| 78 | - pull( | |
| 79 | - read(path.join(__filename)), | |
| 80 | - sbotA.blobs.add(function (err, hash) { | |
| 81 | - if(err) throw err | |
| 82 | - | |
| 83 | - console.log('alice.address', sbotA.getAddress()) | |
| 84 | - createSbot.createClient({keys: bob}) | |
| 85 | - (sbotA.getAddress(), function (err, rpc) { | |
| 86 | - if(err) throw err | |
| 87 | - rpc.blobs.has(hash, function (err) { | |
| 88 | - if(err) throw err | |
| 89 | - pull( | |
| 90 | - rpc.blobs.get(hash), | |
| 91 | - pull.collect(function (err, ary) { | |
| 92 | - if(err) throw err | |
| 93 | - var data = Buffer.concat(ary) | |
| 94 | - sbotA.close(true) | |
| 95 | - t.equal('&'+ssbKeys.hash(data), hash) | |
| 96 | - t.end() | |
| 97 | - }) | |
| 98 | - ) | |
| 99 | - }) | |
| 100 | - }) | |
| 101 | - }) | |
| 102 | - ) | |
| 103 | -}) | |
| 104 | - | |
| 105 | -tape('replicate blobs between 2 peers - explicit want request', function (t) { | |
| 106 | - var hasher = createHash() | |
| 107 | - | |
| 108 | - var alice | |
| 109 | - var sbotA = createSbot({ | |
| 110 | - temp: 'test-blobs-alice1', timeout: 1000, | |
| 111 | - keys: alice = ssbKeys.generate() | |
| 112 | - }) | |
| 113 | - | |
| 114 | - var bob | |
| 115 | - var sbotB = createSbot({ | |
| 116 | - temp: 'test-blobs-bob1', timeout: 1000, | |
| 117 | - keys: bob = ssbKeys.generate() | |
| 118 | - }) | |
| 119 | - | |
| 120 | - pull( | |
| 121 | - read(path.join(__filename)), | |
| 122 | - hasher, | |
| 123 | - sbotA.blobs.add(function (err) { | |
| 124 | - if(err) throw err | |
| 125 | - | |
| 126 | - var hash = hasher.digest | |
| 127 | - console.log('added:', hash) | |
| 128 | - sbotB.blobs.want(hash, function (err, has) { | |
| 129 | - console.log('got:', hash) | |
| 130 | - if(err) throw err | |
| 131 | - t.ok(has) | |
| 132 | - sbotB.blobs.has(hash, function (err, has) { | |
| 133 | - if(err) throw err | |
| 134 | - t.ok(has) | |
| 135 | - t.end() | |
| 136 | - sbotA.close(true) | |
| 137 | - sbotB.close(true) | |
| 138 | - console.log('TEST ENDED') | |
| 139 | - }) | |
| 140 | - }) | |
| 141 | - | |
| 142 | - }) | |
| 143 | - ) | |
| 144 | - | |
| 145 | -// sbotA.on('blobs:got', function (hash) { | |
| 146 | - //console.log('BLOBS', hash) | |
| 147 | - //console.log('added', hash) | |
| 148 | - //}) | |
| 149 | - | |
| 150 | - sbotA.connect(sbotB.getAddress(), function (err) { | |
| 151 | - if(err) throw err | |
| 152 | - }) | |
| 153 | - | |
| 154 | -}) | |
| 155 | - | |
| 156 | -tape('replicate published blobs between 2 peers', function (t) { | |
| 157 | - createSbot.use(friends).use(replicate).use(gossip) | |
| 158 | - | |
| 159 | - var alice = createSbot({ | |
| 160 | - temp: 'test-blobs-alice2', timeout: 1000, | |
| 161 | - keys: ssbKeys.generate() | |
| 162 | - }) | |
| 163 | - | |
| 164 | - var bob = createSbot({ | |
| 165 | - temp: 'test-bobs-alice2', timeout: 1000, | |
| 166 | - keys: ssbKeys.generate(), | |
| 167 | - seeds: [alice.getAddress()] | |
| 168 | - }) | |
| 169 | - | |
| 170 | - var hasher = createHash() | |
| 171 | - | |
| 172 | - pull( | |
| 173 | - read(__filename), | |
| 174 | - hasher, | |
| 175 | - alice.blobs.add(null, function (err) { | |
| 176 | - if(err) throw err | |
| 177 | - var hash = hasher.digest | |
| 178 | - cont.para([ | |
| 179 | - alice.publish(u.file(hash)), | |
| 180 | - alice.publish(u.follow(bob.id)), | |
| 181 | - bob.publish(u.follow(alice.id)) | |
| 182 | - ])(function (err, data) { | |
| 183 | - if(err) throw err | |
| 184 | - }) | |
| 185 | - | |
| 186 | - pull( | |
| 187 | - bob.blobs.changes(), | |
| 188 | - pull.through(console.log), | |
| 189 | - pull.drain(function (_hash) { | |
| 190 | - | |
| 191 | - if(_hash === hash) | |
| 192 | - bob.blobs.has(hash, function (err, okay) { | |
| 193 | - t.ok(okay, 'file replicated:' + hash) | |
| 194 | - t.end() | |
| 195 | - alice.close(true); bob.close(true) | |
| 196 | - }) | |
| 197 | - | |
| 198 | - }) | |
| 199 | - ) | |
| 200 | - | |
| 201 | - | |
| 202 | - // bob should request the blob, | |
| 203 | - // and then emit this event. | |
| 204 | - | |
| 205 | - }) | |
| 206 | - ) | |
| 207 | -}) | |
| 208 | - | |
| 209 | -tape('blob add resolves blob want', function (t) { | |
| 210 | - var sbot = createSbot({ | |
| 211 | - temp: 'test-blobs-foo', timeout: 1000, | |
| 212 | - keys: ssbKeys.generate() | |
| 213 | - }) | |
| 214 | - | |
| 215 | - var hasher = createHash() | |
| 216 | - pull( | |
| 217 | - read(path.join(__filename)), | |
| 218 | - hasher, | |
| 219 | - pull.drain(null, function (err) { | |
| 220 | - t.error(err, 'read file hash') | |
| 221 | - var hash = hasher.digest | |
| 222 | - | |
| 223 | - sbot.blobs.want(hash, function (err) { | |
| 224 | - t.error(err, 'blob want') | |
| 225 | - t.end() | |
| 226 | - sbot.close(true) | |
| 227 | - }) | |
| 228 | - | |
| 229 | - pull( | |
| 230 | - read(path.join(__filename)), | |
| 231 | - sbot.blobs.add(hash, function (err, _hash) { | |
| 232 | - t.error(err, 'blob add') | |
| 233 | - t.equal(_hash, hash, 'added the wanted blob') | |
| 234 | - }) | |
| 235 | - ) | |
| 236 | - }) | |
| 237 | - ) | |
| 238 | -}) | |
| 239 | - | |
| 240 | - | |
| 241 | - | |
| 242 | - | |
| 243 | - | |
| 244 | - | |
| 245 | - | |
| 246 | - | |
| 247 | - | |
| test/blobs2.js | ||
|---|---|---|
| @@ -1,203 +1,0 @@ | ||
| 1 | -var fs = require('fs') | |
| 2 | -var tape = require('tape') | |
| 3 | -var path = require('path') | |
| 4 | -var toPull = require('stream-to-pull-stream') | |
| 5 | -var pull = require('pull-stream') | |
| 6 | -var cont = require('cont') | |
| 7 | -var Hasher = require('multiblob/util').createHash | |
| 8 | -var ssbKeys = require('ssb-keys') | |
| 9 | - | |
| 10 | -var u = require('./util') | |
| 11 | - | |
| 12 | -// create 3 servers | |
| 13 | -// give them all pub servers (on localhost) | |
| 14 | -// and get them to follow each other... | |
| 15 | -var gossip = require('../plugins/gossip') | |
| 16 | -var blobs = require('../plugins/blobs') | |
| 17 | -var friends = require('../plugins/friends') | |
| 18 | -var replicate = require('../plugins/replicate') | |
| 19 | - | |
| 20 | -var createSbot = require('../') | |
| 21 | - .use(friends).use(gossip) | |
| 22 | - .use(replicate).use(blobs).use(require('../plugins/logging')) | |
| 23 | - | |
| 24 | -function read (filename) { | |
| 25 | - return toPull.source(fs.createReadStream(filename)) | |
| 26 | -} | |
| 27 | - | |
| 28 | -var alg = 'sha256' | |
| 29 | - | |
| 30 | -tape('avoid flooding a peer with blob requests', function (t) { | |
| 31 | - | |
| 32 | - var alice = createSbot({ | |
| 33 | - temp: 'test-blobs-alice3', timeout: 1000, | |
| 34 | - keys: ssbKeys.generate() | |
| 35 | - }) | |
| 36 | - | |
| 37 | - var bob = createSbot({ | |
| 38 | - temp: 'test-blobs-bob3', timeout: 1000, | |
| 39 | - seeds: [alice.getAddress()], | |
| 40 | - keys: ssbKeys.generate() | |
| 41 | - }) | |
| 42 | - | |
| 43 | - var hasher = Hasher(alg) | |
| 44 | - | |
| 45 | - pull( | |
| 46 | - read(__filename), | |
| 47 | - hasher, | |
| 48 | - pull.drain(null, function (err) { | |
| 49 | - | |
| 50 | - var hash = '&'+hasher.digest | |
| 51 | - | |
| 52 | - cont.para([ | |
| 53 | - alice.publish(u.file(hash)), | |
| 54 | - alice.publish(u.follow(bob.id)), | |
| 55 | - bob.publish(u.follow(alice.id)) | |
| 56 | - ])(function (err, data) { | |
| 57 | - if(err) throw err | |
| 58 | - }) | |
| 59 | - // bob should not request `hash` more than once. | |
| 60 | - | |
| 61 | - t.plan(1) | |
| 62 | - | |
| 63 | - var has = 0 | |
| 64 | - | |
| 65 | - alice.on('blobs:has', function (h) { | |
| 66 | - t.equal(h, hash) | |
| 67 | - | |
| 68 | - if(has++==0) { | |
| 69 | - alice.close(true); bob.close(true) | |
| 70 | - t.end() | |
| 71 | - } | |
| 72 | - }) | |
| 73 | - }) | |
| 74 | - ) | |
| 75 | -}) | |
| 76 | - | |
| 77 | -tape('emit "has" event to let peer know you have blob now', function (t) { | |
| 78 | - | |
| 79 | - var alice = createSbot({ | |
| 80 | - temp: 'test-blobs-alice5', timeout: 1000, | |
| 81 | - keys: ssbKeys.generate() | |
| 82 | - }) | |
| 83 | - | |
| 84 | - var bob = createSbot({ | |
| 85 | - temp: 'test-blobs-bob5', timeout: 1000, | |
| 86 | - seeds: [alice.getAddress()], | |
| 87 | - keys: ssbKeys.generate() | |
| 88 | - }) | |
| 89 | - console.log('BOB', bob.id) | |
| 90 | - | |
| 91 | - var hasher = Hasher(alg) | |
| 92 | - | |
| 93 | - alice.on('blobs:has', function (r) { | |
| 94 | - console.log('REQUEST', r) | |
| 95 | - }) | |
| 96 | - | |
| 97 | - var count = 1 | |
| 98 | - | |
| 99 | - pull( | |
| 100 | - read(__filename), | |
| 101 | - hasher, | |
| 102 | - pull.drain(null, function (err) { | |
| 103 | - if(err) throw err | |
| 104 | - var hash = '&'+hasher.digest | |
| 105 | - console.log('WANT:', hash) | |
| 106 | - | |
| 107 | - cont.para([ | |
| 108 | - alice.publish(u.file(hash)), | |
| 109 | - alice.publish(u.follow(bob.id)), | |
| 110 | - bob.publish(u.follow(alice.id)) | |
| 111 | - ])(function (err, data) { | |
| 112 | - if(err) throw err | |
| 113 | - }) | |
| 114 | - // bob should not request `hash` more than once. | |
| 115 | - | |
| 116 | - t.plan(2) | |
| 117 | - | |
| 118 | - bob.on('blobs:got', function (h) { | |
| 119 | - if(--count) throw new Error('blobs:got should only trigger once') | |
| 120 | - console.log('BLOBS GOT', h) | |
| 121 | - t.equal(h, hash) | |
| 122 | - alice.close(true); bob.close(true) | |
| 123 | - t.end() | |
| 124 | - }) | |
| 125 | - | |
| 126 | - //wait for bob to request the hash | |
| 127 | - //then add that file. | |
| 128 | - alice.on('blobs:has', function (h) { | |
| 129 | - console.log('BLOBS HAS', h) | |
| 130 | - t.equal(h, hash) | |
| 131 | - | |
| 132 | - pull( | |
| 133 | - read(__filename), | |
| 134 | - bob.blobs.add(function (err, hash) { | |
| 135 | - if(err) throw err | |
| 136 | - //have now added the blob to | |
| 137 | - }) | |
| 138 | - ) | |
| 139 | - }) | |
| 140 | - }) | |
| 141 | - ) | |
| 142 | - | |
| 143 | - //this test should only require one connection. | |
| 144 | - var n = 0 | |
| 145 | - bob.on('rpc:connect', function (rpc) { | |
| 146 | - if(++n > 1) throw new Error('connected twice') | |
| 147 | - }) | |
| 148 | -}) | |
| 149 | - | |
| 150 | -tape('request missing blobs again after reconnect', function (t) { | |
| 151 | - | |
| 152 | - var alice = createSbot({ | |
| 153 | - temp: 'test-blobs-alice4', timeout: 2000, | |
| 154 | - keys: ssbKeys.generate() | |
| 155 | - }) | |
| 156 | - | |
| 157 | - var bob = createSbot({ | |
| 158 | - temp: 'test-blobs-bob4', timeout: 2000, | |
| 159 | - seeds: [alice.getAddress()], | |
| 160 | - keys: ssbKeys.generate() | |
| 161 | - }) | |
| 162 | - | |
| 163 | - var hasher = Hasher(alg) | |
| 164 | - | |
| 165 | - pull( | |
| 166 | - read(__filename), | |
| 167 | - hasher, | |
| 168 | - pull.drain(null, function (err) { | |
| 169 | - | |
| 170 | - var hash = '&'+hasher.digest | |
| 171 | - | |
| 172 | - cont.para([ | |
| 173 | - alice.publish(u.file(hash)), | |
| 174 | - alice.publish(u.follow(bob.id)), | |
| 175 | - bob.publish(u.follow(alice.id)) | |
| 176 | - ])(function (err, data) { | |
| 177 | - if(err) throw err | |
| 178 | - }) | |
| 179 | - // bob should not request `hash` more than once. | |
| 180 | - | |
| 181 | - var has = 0, connects = 0 | |
| 182 | - | |
| 183 | - alice.on('blobs:has', function (h) { | |
| 184 | - console.log('----HAS', h, has) | |
| 185 | - t.equal(h, hash) | |
| 186 | - | |
| 187 | - if(has++ == 1) { | |
| 188 | - t.equal(has, connects) | |
| 189 | - alice.close(true); bob.close(true) | |
| 190 | - t.end() | |
| 191 | - } | |
| 192 | - }) | |
| 193 | - | |
| 194 | - bob.on('rpc:connect', function () { | |
| 195 | - connects ++ | |
| 196 | - }) | |
| 197 | - | |
| 198 | - }) | |
| 199 | - ) | |
| 200 | -}) | |
| 201 | - | |
| 202 | - | |
| 203 | - | |
Built with git-ssb-web