git ssb

10+

Matt McKegg / patchwork



Commit a6aab6f1d2a86e5cf04121dbc040f657ecdca01f

rewrite search: now much more responsive and faster

fixes #441
Matt McKegg committed on 6/28/2017, 9:09:40 AM
Parent: 8947f767e44f337b643fc56c3a939cfd7da4a8c1

Files changed

modules/page/html/render/search.jschanged
sbot/index.jschanged
sbot/search.jsadded
styles/search-page.mcssadded
modules/page/html/render/search.jsView
@@ -1,17 +1,16 @@
1-const { h, Struct, Value, when, computed } = require('mutant')
2-const pull = require('pull-stream')
3-const Scroller = require('../../../../lib/scroller')
4-const TextNodeSearcher = require('text-node-searcher')
5-const whitespace = /\s+/
6-const pullAbortable = require('pull-abortable')
1+var { h, Value, when, computed } = require('mutant')
2+var pull = require('pull-stream')
3+var TextNodeSearcher = require('text-node-searcher')
4+var whitespace = /\s+/
5+var pullAbortable = require('pull-abortable')
6+var Scroller = require('../../../../lib/scroller')
7+var nextStepper = require('../../../../lib/next-stepper')
78 var nest = require('depnest')
8-var Next = require('pull-next')
9-var defer = require('pull-defer')
10-var pullCat = require('pull-cat')
9+var Proxy = require('mutant/proxy')
1110
1211 exports.needs = nest({
13- 'sbot.pull.log': 'first',
12+ 'sbot.pull.stream': 'first',
1413 'keys.sync.id': 'first',
1514 'message.html.render': 'first'
1615 })
1716
@@ -22,98 +21,55 @@
2221 if (path[0] !== '?') return
2322
2423 var queryStr = path.substr(1).trim()
2524 var query = queryStr.split(whitespace)
26- var matchesQuery = searchFilter(query)
25+ var done = Value(false)
26+ var loading = Proxy(true)
27+ var count = Value(0)
28+ var updates = Value(0)
29+ var aborter = null
2730
28- const search = Struct({
29- isLinear: Value(),
30- linear: Struct({
31- checked: Value(0),
32- isDone: Value(false)
33- }),
34- fulltext: Struct({
35- isDone: Value(false)
36- }),
37- matches: Value(0)
38- })
39-
40- const hasNoFulltextMatches = computed([
41- search.fulltext.isDone, search.matches
42- ], (done, matches) => {
43- return done && matches === 0
44- })
45-
46- const searchHeader = h('Search', [
47- h('div', {className: 'PageHeading'}, [
48- h('h1', [h('strong', 'Search Results:'), ' ', query.join(' ')]),
49- when(search.isLinear,
50- h('div.meta', ['Searched: ', search.linear.checked]),
51- h('section.details', when(hasNoFulltextMatches, h('div.matches', 'No matches')))
52- )
53- ])
31+ const searchHeader = h('div', {className: 'PageHeading'}, [
32+ h('h1', [h('strong', 'Search Results:'), ' ', query.join(' ')])
5433 ])
5534
56- var content = h('section.content')
35+ var content = Proxy()
5736 var container = h('Scroller', {
5837 style: { overflow: 'auto' }
5938 }, [
6039 h('div.wrapper', [
61- searchHeader,
62- content,
63- when(search.linear.isDone, null, h('Loading -search'))
40+ h('SearchPage', [
41+ searchHeader,
42+ content,
43+ when(loading, h('Loading -search'), h('div', {
44+ style: {
45+ 'padding': '60px 0',
46+ 'font-size': '150%'
47+ }
48+ }, [h('strong', 'Search completed.'), ' ', count, ' ', plural(count, 'result', 'results'), ' found']))
49+ ])
6450 ])
6551 ])
6652
6753 var realtimeAborter = pullAbortable()
6854
6955 pull(
70- api.sbot.pull.log({old: false}),
71- pull.filter(matchesQuery),
56+ api.sbot.pull.stream(sbot => sbot.patchwork.linearSearch({old: false, query})),
7257 realtimeAborter,
73- Scroller(container, content, renderMsg)
58+ pull.drain(msg => {
59+ updates.set(updates() + 1)
60+ })
7461 )
7562
76- // pull(
77- // nextStepper(api.sbot.pull.search, {query: queryStr, reverse: true, limit: 500, live: false}),
78- // fallback((err) => {
79- // if (err === true) {
80- // search.fulltext.isDone.set(true)
81- // } else if (/^no source/.test(err.message)) {
82- // search.isLinear.set(true)
83- // return pull(
84- // nextStepper(api.sbot.pull.log, {reverse: true, limit: 500, live: false}),
85- // pull.through((msg) => search.linear.checked.set(search.linear.checked() + 1)),
86- // pull.filter(matchesQuery)
87- // )
88- // }
89- // }),
90- // pull.through(() => search.matches.set(search.matches() + 1)),
91- // Scroller(container, content, renderMsg, false, false)
92- // )
63+ refresh()
9364
94- // disable full text for now
95- var aborter = pullAbortable()
96- search.isLinear.set(true)
97- pull(
98- nextStepper(api.sbot.pull.log, {reverse: true, limit: 500, live: false, delay: 100}),
99- pull.through((msg) => search.linear.checked.set(search.linear.checked() + 1)),
100- pull.filter(matchesQuery),
101- pull.through(() => search.matches.set(search.matches() + 1)),
102- aborter,
103- Scroller(container, content, renderMsg, false, false, err => {
104- if (err) console.log(err)
105- search.linear.isDone.set(true)
106- })
107- )
108-
10965 return h('SplitView', {
11066 hooks: [
11167 RemoveHook(() => {
11268 // terminate search if removed from dom
11369 // this is triggered whenever a new search is started
11470 realtimeAborter.abort()
115- aborter.abort()
71+ aborter && aborter.abort()
11672 })
11773 ],
11874 uniqueKey: 'search'
11975 }, [
@@ -121,39 +77,57 @@
12177 ])
12278
12379 // scoped
12480
81+ function refresh () {
82+ if (aborter) {
83+ aborter.abort()
84+ }
85+
86+ aborter = pullAbortable()
87+
88+ updates.set(0)
89+ content.set(h('section.content'))
90+
91+ var scroller = Scroller(container, content(), renderMsg, err => {
92+ if (err) console.log(err)
93+ done.set(true)
94+ })
95+
96+ pull(
97+ api.sbot.pull.stream(sbot => nextStepper(getStream, {
98+ reverse: true,
99+ limit: 5,
100+ query
101+ })),
102+ pull.through(() => count.set(count() + 1)),
103+ aborter,
104+ pull.filter(msg => msg.value),
105+ scroller
106+ )
107+
108+ loading.set(computed([done, scroller.queue], (done, queue) => {
109+ return !done && queue < 5
110+ }))
111+ }
112+
113+ function getStream (opts) {
114+ if (opts.lt != null && !opts.lt.marker) {
115+ // if an lt has been specified that is not a marker, assume stream is finished
116+ return pull.empty()
117+ } else {
118+ return api.sbot.pull.stream(sbot => sbot.patchwork.linearSearch(opts))
119+ }
120+ }
121+
125122 function renderMsg (msg) {
126- var el = h('div.result', api.message.html.render(msg))
123+ var el = h('FeedEvent', api.message.html.render(msg))
127124 highlight(el, createOrRegExp(query))
128125 return el
129126 }
130127 })
131128 }
132129
133-function andSearch (terms, inputs) {
134- for (var i = 0; i < terms.length; i++) {
135- var match = false
136- for (var j = 0; j < inputs.length; j++) {
137- if (terms[i].test(inputs[j])) match = true
138- }
139- // if a term was not matched by anything, filter this one
140- if (!match) return false
141- }
142- return true
143-}
144-
145-function searchFilter (terms) {
146- return function (msg) {
147- var c = msg && msg.value && msg.value.content
148- return c && (
149- msg.key === terms[0] || andSearch(terms.map(function (term) {
150- return new RegExp('\\b' + term + '\\b', 'i')
151- }), [c.text, c.name, c.title])
152- )
153- }
154-}
155-
156130 function createOrRegExp (ary) {
157131 return new RegExp(ary.map(function (e) {
158132 return '\\b' + e + '\\b'
159133 }).join('|'), 'i')
@@ -167,106 +141,19 @@
167141 return el
168142 }
169143 }
170144
171-function fallback (reader) {
172- var fallbackRead
173- return function (read) {
174- return function (abort, cb) {
175- read(abort, function next (end, data) {
176- if (end && reader && (fallbackRead = reader(end))) {
177- reader = null
178- read = fallbackRead
179- read(abort, next)
180- } else {
181- cb(end, data)
182- }
183- })
184- }
185- }
186-}
187-
188-function nextStepper (createStream, opts, property, range) {
189- range = range || (opts.reverse ? 'lt' : 'gt')
190- property = property || 'timestamp'
191-
192- var last = null
193- var count = -1
194-
195- return Next(function () {
196- if (last) {
197- if (count === 0) return
198- var value = opts[range] = get(last, property)
199- if (value == null) return
200- last = null
201- }
202- var result = defer.source()
203-
204- window.requestIdleCallback(() => {
205- result.resolve(pull(
206- createStream(clone(opts)),
207- pull.through(function (msg) {
208- count++
209- if (!msg.sync) {
210- last = msg
211- }
212- }, function (err) {
213- // retry on errors...
214- if (err) {
215- count = -1
216- return count
217- }
218- // end stream if there were no results
219- if (last == null) last = {}
220- })
221- ))
222- })
223-
224- return pauseAfter(result, opts.delay)
225- })
226-}
227-
228-function get (obj, path) {
229- if (!obj) return undefined
230- if (typeof path === 'string') return obj[path]
231- if (Array.isArray(path)) {
232- for (var i = 0; obj && i < path.length; i++) {
233- obj = obj[path[i]]
234- }
235- return obj
236- }
237-}
238-
239-function clone (obj) {
240- var _obj = {}
241- for (var k in obj) _obj[k] = obj[k]
242- return _obj
243-}
244-
245145 function RemoveHook (fn) {
246146 return function (element) {
247147 return fn
248148 }
249149 }
250150
251-function pauseAfter (stream, delay) {
252- if (!delay) {
253- return stream
254- } else {
255- return pullCat([
256- stream,
257- wait(delay)
258- ])
259- }
260-}
261-
262-function wait (delay) {
263- return function (abort, cb) {
264- if (abort) {
265- cb(true)
151+function plural (value, single, many) {
152+ return computed(value, (value) => {
153+ if (value === 1) {
154+ return single
266155 } else {
267- setTimeout(() => {
268- cb(true)
269- }, delay)
156+ return many
270157 }
271- }
158+ })
272159 }
sbot/index.jsView
@@ -1,16 +1,18 @@
11 var Channels = require('./channels')
22 var Subscriptions = require('./subscriptions')
33 var Roots = require('./roots')
44 var Progress = require('./progress')
5+var Search = require('./search')
56
67 exports.name = 'patchwork'
78 exports.version = require('../package.json').version
89 exports.manifest = {
910 channels: 'source',
1011 subscriptions: 'source',
1112 roots: 'source',
1213 latest: 'source',
14+ linearSearch: 'source',
1315 progress: 'source',
1416 getSubscriptions: 'async',
1517 getChannels: 'async'
1618 }
@@ -19,16 +21,17 @@
1921 var progress = Progress(ssb, config)
2022 var channels = Channels(ssb, config)
2123 var subscriptions = Subscriptions(ssb, config)
2224 var roots = Roots(ssb, config)
25+ var search = Search(ssb, config)
2326
2427 return {
2528 channels: channels.stream,
2629 subscriptions: subscriptions.stream,
2730 roots: roots.read,
2831 latest: roots.latest,
2932 progress: progress.stream,
30-
33+ linearSearch: search.linear,
3134 getSubscriptions: subscriptions.get,
3235 getChannels: channels.get
3336 }
3437 }
sbot/search.jsView
@@ -1,0 +1,81 @@
1+var pull = require('pull-stream')
2+var pullCat = require('pull-cat')
3+
4+module.exports = function (ssb, config) {
5+ return {
6+ linear: function ({lt, gt, reverse, limit, query, old, live}) {
7+ // handle markers passed in to lt / gt
8+ var opts = {reverse, old, live}
9+ if (lt && typeof lt.timestamp === 'number') lt = lt.timestamp
10+ if (gt && typeof gt.timestamp === 'number') gt = gt.timestamp
11+ if (typeof lt === 'number') opts.lt = lt
12+ if (typeof gt === 'number') opts.gt = gt
13+
14+ var matchesQuery = searchFilter(query)
15+ var marker = {marker: true, timestamp: null}
16+
17+ var stream = pull(
18+ ssb.createLogStream(opts),
19+ pull.map(msg => {
20+ if (msg.value && typeof msg.value.content === 'string') {
21+ var unboxed = ssb.private.unbox(msg)
22+ if (unboxed) {
23+ return unboxed
24+ }
25+ }
26+ return msg
27+ }),
28+ pull.through(msg => {
29+ marker.timestamp = msg.timestamp
30+ }),
31+ pull.filter(matchesQuery)
32+ )
33+
34+ // TRUNCATE
35+ if (typeof limit === 'number') {
36+ var count = 0
37+ return pullCat([
38+ pull(
39+ stream,
40+ pull.take(limit),
41+ pull.through(() => {
42+ count += 1
43+ })
44+ ),
45+
46+ // send truncated marker for resuming search
47+ pull(
48+ pull.values([marker]),
49+ pull.filter(() => count === limit)
50+ )
51+ ])
52+ } else {
53+ return stream
54+ }
55+ }
56+ }
57+}
58+
59+function searchFilter (terms) {
60+ return function (msg) {
61+ if (msg.sync) return true
62+ var c = msg && msg.value && msg.value.content
63+ return c && (
64+ msg.key === terms[0] || andSearch(terms.map(function (term) {
65+ return new RegExp('\\b' + term + '\\b', 'i')
66+ }), [c.text, c.name, c.title])
67+ )
68+ }
69+}
70+
71+function andSearch (terms, inputs) {
72+ for (var i = 0; i < terms.length; i++) {
73+ var match = false
74+ for (var j = 0; j < inputs.length; j++) {
75+ if (terms[i].test(inputs[j])) match = true
76+ }
77+ // if a term was not matched by anything, filter this one
78+ if (!match) return false
79+ }
80+ return true
81+}
styles/search-page.mcssView
@@ -1,0 +1,7 @@
1+SearchPage {
2+ section.content {
3+ div.result {
4+ margin: 10px 0
5+ }
6+ }
7+}

Built with git-ssb-web