git ssb

2+

cel / scuttlebot.io



Commit 86cfe7c6ec39c87ba43e41c3ed13c7902bf07fd8

Browse Files

add pull-stream api docs

Paul Frazee committed on 3/16/2016, 1:04:11 AM
Parent: a738ecf91d844ee886e94857a82177ab446113f1

Files changed

tmpl/apis/pull-stream/core-sinks.html.jsadded
tmpl/apis/pull-stream/core-sinks.mdadded
tmpl/apis/pull-stream/core-sources.html.jsadded
tmpl/apis/pull-stream/core-sources.mdadded
tmpl/apis/pull-stream/core-throughs.html.jsadded
tmpl/apis/pull-stream/core-throughs.mdadded
tmpl/apis/pull-stream/pull-cat.html.jsadded
tmpl/apis/pull-stream/pull-cat.mdadded
tmpl/apis/pull-stream/pull-notify.html.jsadded
tmpl/apis/pull-stream/pull-notify.mdadded
tmpl/apis/pull-stream/pull-otherwise.html.jsadded
tmpl/apis/pull-stream/pull-otherwise.mdadded
tmpl/apis/pull-stream/pull-paramap.html.jsadded
tmpl/apis/pull-stream/pull-paramap.mdadded
tmpl/apis/pull-stream/pull-pause.html.jsadded
tmpl/apis/pull-stream/pull-pause.mdadded
tmpl/apis/pull-stream/pull-pushable.html.jsadded
tmpl/apis/pull-stream/pull-pushable.mdadded
tmpl/apis/pull-stream/pull-stream-to-stream.html.jsadded
tmpl/apis/pull-stream/pull-stream-to-stream.mdadded
tmpl/apis/pull-stream/pull-stream.html.jsadded
tmpl/apis/pull-stream/pull-stream.mdadded
tmpl/apis/pull-stream/pull-timeout.html.jsadded
tmpl/apis/pull-stream/pull-timeout.mdadded
tmpl/apis/pull-stream/pull-window.html.jsadded
tmpl/apis/pull-stream/pull-window.mdadded
tmpl/apis/pull-stream/pull-ws-server.html.jsadded
tmpl/apis/pull-stream/pull-ws-server.mdadded
tmpl/apis/pull-stream/stream-to-pull-stream.html.jsadded
tmpl/apis/pull-stream/stream-to-pull-stream.mdadded
tmpl/css/com.part.jschanged
tmpl/css/index.css.jschanged
tmpl/leftnav.part.jschanged
tmpl/tabs.part.jschanged
tmpl/apis/pull-stream/core-sinks.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/core-sinks.html',
7+ content: md.doc(__dirname+'/core-sinks.md')
8+})
tmpl/apis/pull-stream/core-sinks.mdView
@@ -1,0 +1,38 @@
1+# Sinks
2+
3+A Sink is a stream that is not readable.
4+You *must* have a sink at the end of a pipeline
5+for data to move towards.
6+
7+You can only use _one_ sink per pipeline.
8+
9+``` js
10+pull(source, through, sink)
11+```
12+
13+See also:
14+* [Sources](./core-sources.html)
15+* [Throughs](./core-throughs.html)
16+
17+## drain (op?, done?)
18+
19+Drain the stream, calling `op` on each `data`.
20+call `done` when stream is finished.
21+If op returns `===false`, abort the stream.
22+
23+## reduce (reduce, initial, cb)
24+
25+reduce stream into single value, then callback.
26+
27+## collect(cb)
28+
29+Read the stream into an array, then callback.
30+
31+## onEnd (cb)
32+
33+Drain the stream and then callback when done.
34+
35+## log
36+
37+output the stream to `console.log`
38+
tmpl/apis/pull-stream/core-sources.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/core-sources.html',
7+ content: md.doc(__dirname+'/core-sources.md')
8+})
tmpl/apis/pull-stream/core-sources.mdView
@@ -1,0 +1,48 @@
1+# Sources
2+
3+A source is a stream that is not writable.
4+You *must* have a source at the start of a pipeline
5+for data to move through.
6+
7+in general:
8+
9+``` js
10+pull(source, through, sink)
11+```
12+
13+See also:
14+* [Throughs](./core-throughs.html)
15+* [Sinks](./core-sinks.html)
16+
17+## values (array | object)
18+
19+create a SourceStream that reads the values from an array or object and then stops.
20+
21+## keys (array | object)
22+
23+stream the key names from an object (or array)
24+
25+## count (max)
26+
27+create a stream that outputs `0 ... max`.
28+by default, `max = Infinity`, see
29+[take](https://github.com/dominictarr/pull-stream/blob/master/docs/throughs.md#take_test)
30+
31+## infinite (generator)
32+
33+create an unending stream by repeatedly calling a generator
34+function (by default, `Math.random`)
35+see
36+[take](https://github.com/dominictarr/pull-stream/blob/master/docs/throughs.md#take_test)
37+
38+## empty
39+
40+A stream with no contents (it just ends immediately)
41+
42+``` js
43+pull.empty().pipe(pull.collect(function (err, ary) {
44+ console.log(arg)
45+ // ==> []
46+})
47+```
48+
tmpl/apis/pull-stream/core-throughs.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/core-throughs.html',
7+ content: md.doc(__dirname+'/core-throughs.md')
8+})
tmpl/apis/pull-stream/core-throughs.mdView
@@ -1,0 +1,79 @@
1+# Throughs
2+
3+A Through is a stream that both reads and is read by
4+another stream.
5+
6+Through streams are optional.
7+
8+Put through streams in-between [sources](https://github.com/dominictarr/pull-stream/blob/master/docs/sources.md) and [sinks](https://github.com/dominictarr/pull-stream/blob/master/docs/sinks.md),
9+like this:
10+
11+```js
12+pull(source, through, sink)
13+```
14+
15+Also, if you don't have the source/sink yet,
16+you can pipe multiple through streams together
17+to get one through stream!
18+
19+```js
20+var throughABC = function () {
21+ return throughA()
22+ .pipe(throughB())
23+ .pipe(throughC())
24+}
25+```
26+
27+Which can then be treated like a normal through stream!
28+
29+```js
30+source().pipe(throughABC()).pipe(sink())
31+```
32+
33+See also:
34+* [Sources](./core-sources.html)
35+* [Sinks](./core-sinks.html)
36+
37+## map (fun)
38+
39+Like `[].map(function (data) {return data})`
40+
41+## asyncMap (fun)
42+
43+Like `map` but the signature of `fun` must be
44+`function (data, cb) { cb(null, data) }`
45+
46+## filter (test)
47+
48+Like `[].filter(function (data) {return true || false})`
49+only `data` where `test(data) == true` are let through
50+to the next stream.
51+
52+
53+## filterNot (test)
54+
55+Like filter, but remove items where the filter returns true.
56+
57+## unique (prop)
58+
59+Filter items that have a repeated value for `prop()`,
60+by default, `prop = function (it) {return it }`, if prop is a string,
61+it will filter nodes which have repeated values for that property.
62+
63+## nonUnique (prop)
64+
65+Filter unique items -- get the duplicates.
66+The inverse of `unique`
67+
68+## take (test [, opts])
69+
70+If test is a function, read data from the source stream and forward it downstream until test(data) returns false.
71+
72+If `opts.last` is set to true, the data for which the test failed will be included in what is forwarded.
73+
74+If test is an integer, take n item from the source.
75+
76+## flatten ()
77+
78+Turn a stream of arrays into a stream of their items, (undoes group).
79+
tmpl/apis/pull-stream/pull-cat.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-cat.html',
7+ content: md.doc(__dirname+'/pull-cat.md')
8+})
tmpl/apis/pull-stream/pull-cat.mdView
@@ -1,0 +1,19 @@
1+# pull-cat
2+
3+concatinate pull-streams
4+
5+# example
6+
7+``` js
8+var cat = require('pull-cat')
9+var pull = require('pull-stream')
10+cat([pull.values([1,2,3]), pull.values([4,5,6])])
11+ .pipe(...)
12+```
13+
14+Reads from the each stream until it is finished.
15+If a stream errors, stop all the streams.
16+
17+## License
18+
19+MIT
tmpl/apis/pull-stream/pull-notify.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-notify.html',
7+ content: md.doc(__dirname+'/pull-notify.md')
8+})
tmpl/apis/pull-stream/pull-notify.mdView
@@ -1,0 +1,35 @@
1+# pull-notify
2+
3+Notify many listeners via pull-streams.
4+
5+you could use when you might otherwise use an event emitter.
6+Why not just use an event emitter? EventEmitters have a weird
7+security contract: anyone who can listen can also emit,
8+and they can emit or listen to any events!
9+
10+Instead, events should travel down a single channel,
11+and the ability to emit an event should be separated from
12+the ability to listen.
13+
14+
15+``` js
16+var Notify = require('pull-notify')
17+
18+var notify = Notify()
19+
20+//create a pull stream that listens on events.
21+//it will eventually get all events.
22+pull(notify.listen(), pull.drain(console.log))
23+
24+notify('hello') //emit an event.
25+
26+notify.end() //tell all listeners it's over.
27+```
28+
29+listers can abort (using the normal pull-stream abort),
30+and that will remove them from the list.
31+
32+
33+## License
34+
35+MIT
tmpl/apis/pull-stream/pull-otherwise.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-otherwise.html',
7+ content: md.doc(__dirname+'/pull-otherwise.md')
8+})
tmpl/apis/pull-stream/pull-otherwise.mdView
@@ -1,0 +1,37 @@
1+# pull-otherwise
2+
3+Read from an alternative source, if the main ended.
4+
5+## Usage
6+
7+### otherwise(alt);
8+
9+`alt` - Alternative source stream.
10+
11+## Example
12+
13+```js
14+var pull = require("pull-stream");
15+var otherwise = require("pull-otherwise");
16+
17+var mainSource = pull.values([1,2,3,4]);
18+var alternative = pull.values([5,6,7,8,9]);
19+
20+pull(
21+ mainSource,
22+ otherwise(alternative),
23+ pull.log()
24+)
25+```
26+
27+## install
28+
29+With [npm](https://npmjs.org) do:
30+
31+```
32+npm install pull-otherwise
33+```
34+
35+## license
36+
37+MIT
tmpl/apis/pull-stream/pull-paramap.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-paramap.html',
7+ content: md.doc(__dirname+'/pull-paramap.md')
8+})
tmpl/apis/pull-stream/pull-paramap.mdView
@@ -1,0 +1,31 @@
1+# pull-paramap
2+
3+parallel mapping pull-stream.
4+
5+[![travis](https://travis-ci.org/dominictarr/pull-paramap.png?branch=master)
6+](https://travis-ci.org/dominictarr/pull-paramap)
7+
8+[![testling](http://ci.testling.com/dominictarr/pull-paramap.png)
9+](http://ci.testling.com/dominictarr/pull-paramap)
10+
11+## example
12+
13+``` js
14+var pull = require('pull-stream')
15+var paramap = require('pull-paramap')
16+
17+pull(
18+ pull.values([....]),
19+ //perform an async job in parallel,
20+ //but return results in the same order as they went in.
21+ paramap(function (data, cb) {
22+ asyncJob(data, cb)
23+ }, width), //optional number.
24+ //limits stream to process width items at once
25+ pull.collect(cb)
26+)
27+```
28+
29+## License
30+
31+MIT
tmpl/apis/pull-stream/pull-pause.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-pause.html',
7+ content: md.doc(__dirname+'/pull-pause.md')
8+})
tmpl/apis/pull-stream/pull-pause.mdView
@@ -1,0 +1,28 @@
1+# pull-pause
2+
3+a through pull-stream that can be turned on and off like a tap.
4+
5+## Example
6+
7+``` js
8+
9+var pull = require('pull-stream')
10+var pause = require('pull-pause')()
11+
12+
13+pull(
14+ source,
15+ pause,
16+ sink
17+)
18+
19+pause.pause() //stop reading.
20+
21+pause.resume() //resume reading.
22+
23+
24+```
25+
26+## License
27+
28+MIT
tmpl/apis/pull-stream/pull-pushable.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-pushable.html',
7+ content: md.doc(__dirname+'/pull-pushable.md')
8+})
tmpl/apis/pull-stream/pull-pushable.mdView
@@ -1,0 +1,50 @@
1+# pull-pushable
2+
3+A pull-stream with a pushable interface.
4+
5+Use this when you really can't pull from your source.
6+For example, often I like to have a "live" stream.
7+This would read a series of data, first old data,
8+but then stay open and read new data as it comes in.
9+
10+In that case, the new data needs to be queued up while the old data is read,
11+and also, the rate things are pushed into the queue doesn't affect the rate of reads.
12+
13+If there is no realtime aspect to this stream, it's likely that you don't need pushable.
14+Instead try just using `pull.values(array)`.
15+
16+## Example
17+
18+``` js
19+var Pushable = require('pull-pushable')
20+var pull = require('pull-stream')
21+var p = Pushable()
22+
23+pull(p, pull.drain(console.log))
24+
25+p.push(1)
26+p.end()
27+```
28+
29+Also, can provide a listener for when the stream is closed.
30+
31+``` js
32+var Pushable = require('pull-pushable')
33+var pull = require('pull-stream')
34+var p = Pushable(function (err) {
35+ console.log('stream closed!')
36+})
37+
38+//read 3 times then abort.
39+pull(p, pull.take(3), pull.drain(console.log))
40+
41+p.push(1)
42+p.push(2)
43+p.push(3)
44+p.push(4) //stream will be aborted before this is output
45+
46+```
47+
48+## License
49+
50+MIT
tmpl/apis/pull-stream/pull-stream-to-stream.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-stream-to-stream.html',
7+ content: md.doc(__dirname+'/pull-stream-to-stream.md')
8+})
tmpl/apis/pull-stream/pull-stream-to-stream.mdView
@@ -1,0 +1,26 @@
1+# pull-stream-to-stream
2+
3+turn a pull-stream into a regular node stream.
4+
5+## example
6+
7+``` js
8+var toStream = require('pull-stream-to-stream')
9+
10+//if the pull-stream is duplex (an object with two streams: {source, sink})
11+
12+stream = toStream(pullDuplex)
13+
14+//if the stream is a sink ("writable")
15+stream = toStream.sink(pullSink)
16+
17+//if the stream is a source ("readable")
18+
19+stream = toStream.source(pullSource)
20+
21+```
22+
23+
24+## License
25+
26+MIT
tmpl/apis/pull-stream/pull-stream.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-stream.html',
7+ content: md.doc(__dirname+'/pull-stream.md')
8+})
tmpl/apis/pull-stream/pull-stream.mdView
@@ -1,0 +1,292 @@
1+# pull-stream
2+
3+Minimal Pipeable Pull-stream
4+
5+In [classic-streams](1),
6+streams _push_ data to the next stream in the pipeline.
7+In [new-streams](https://github.com/joyent/node/blob/v0.10/doc/api/stream.markdown),
8+data is pulled out of the source stream, into the destination.
9+In [new-classic-streams](
10+`pull-stream` is a minimal take on streams,
11+pull streams work great for "object" streams as well as streams of raw text or binary data.
12+
13+
14+## Quick Example
15+
16+Stat some files:
17+
18+```js
19+pull(
20+ pull.values(['file1', 'file2', 'file3']),
21+ pull.asyncMap(fs.stat),
22+ pull.collect(function (err, array) {
23+ console.log(array)
24+ })
25+)
26+```
27+note that `pull(a, b, c)` is basically the same as `a.pipe(b).pipe(c)`.
28+
29+The best thing about pull-stream is that it can be completely lazy.
30+This is perfect for async traversals where you might want to stop early.
31+
32+## Compatibily with node streams
33+
34+pull-streams are not _directly_ compatible with node streams,
35+but pull-streams can be converted into node streams with
36+[pull-stream-to-stream](https://github.com/dominictarr/pull-stream-to-stream)
37+and node streams can be converted into pull-stream using [stream-to-pull-stream](https://github.com/dominictarr/stream-to-pull-stream)
38+
39+
40+### Readable & Reader vs. Readable & Writable
41+
42+Instead of a readable stream, and a writable stream, there is a `readable` stream,
43+ (aka "Source") and a `reader` stream (aka "Sink"). Through streams
44+is a Sink that returns a Source.
45+
46+See also:
47+* [Sources](./core-sources.html)
48+* [Throughs](./core-throughs.html)
49+* [Sinks](./core-sinks.html)
50+
51+### Source (aka, Readable)
52+
53+The readable stream is just a `function read(end, cb)`,
54+that may be called many times,
55+and will (asynchronously) `cb(null, data)` once for each call.
56+
57+To signify an end state, the stream eventually returns `cb(err)` or `cb(true)`.
58+When indicating a terminal state, `data` *must* be ignored.
59+
60+The `read` function *must not* be called until the previous call has called back.
61+Unless, it is a call to abort the stream (`read(truthy, cb)`).
62+
63+```js
64+//a stream of 100 random numbers.
65+var i = 100
66+var random = function () {
67+ return function (end, cb) {
68+ if(end) return cb(end)
69+ //only read 100 times
70+ if(i-- < 0) return cb(true)
71+ cb(null, Math.random())
72+ }
73+}
74+
75+```
76+
77+### Sink; (aka, Reader, "writable")
78+
79+A sink is just a `reader` function that calls a Source (read function),
80+until it decideds to stop, or the readable ends. `cb(err || true)`
81+
82+All [Throughs](./core-throughs.html)
83+and [Sinks](./core-sinks.html)
84+are reader streams.
85+
86+```js
87+//read source and log it.
88+var logger = function () {
89+ return function (read) {
90+ read(null, function next(end, data) {
91+ if(end === true) return
92+ if(end) throw end
93+
94+ console.log(data)
95+ read(null, next)
96+ })
97+ }
98+}
99+```
100+
101+Since these are just functions, you can pass them to each other!
102+
103+```js
104+var rand = random()
105+var log = logger()
106+
107+log(rand) //"pipe" the streams.
108+
109+```
110+
111+but, it's easier to read if you use's pull-stream's `pull` method
112+
113+```js
114+var pull = require('pull-stream')
115+
116+pull(random(), logger())
117+```
118+
119+### Through
120+
121+A through stream is a reader on one end and a readable on the other.
122+It's Sink that returns a Source.
123+That is, it's just a function that takes a `read` function,
124+and returns another `read` function.
125+
126+```js
127+var map = function (read, map) {
128+ //return a readable function!
129+ return function (end, cb) {
130+ read(end, function (end, data) {
131+ cb(end, data != null ? map(data) : null)
132+ })
133+ }
134+}
135+```
136+
137+### Pipeability
138+
139+Every pipeline must go from a `source` to a `sink`.
140+Data will not start moving until the whole thing is connected.
141+
142+```js
143+pull(source, through, sink)
144+```
145+
146+some times, it's simplest to describe a stream in terms of other streams.
147+pull can detect what sort of stream it starts with (by counting arguments)
148+and if you pull together through streams, it gives you a new through stream.
149+
150+```js
151+var tripleThrough =
152+ pull(through1(), through2(), through3())
153+//THE THREE THROUGHS BECOME ONE
154+
155+pull(source(), tripleThrough, sink())
156+```
157+
158+pull detects if it's missing a Source by checking function arity,
159+if the function takes only one argument it's either a sink or a through.
160+Otherwise it's a Source.
161+
162+## Duplex Streams
163+
164+Duplex streams, which are used to communicate between two things,
165+(i.e. over a network) are a little different. In a duplex stream,
166+messages go both ways, so instead of a single function that represents the stream,
167+you need a pair of streams. `{source: sourceStream, sink: sinkStream}`
168+
169+pipe duplex streams like this:
170+
171+``` js
172+var a = duplex()
173+var b = duplex()
174+
175+pull(a.source, b.sink)
176+pull(b.source, a.sink)
177+
178+//which is the same as
179+
180+b.sink(a.source); a.sink(b.source)
181+
182+//but the easiest way is to allow pull to handle this
183+
184+pull(a, b, a)
185+
186+//"pull from a to b and then back to a"
187+
188+```
189+
190+## Design Goals & Rationale
191+
192+There is a deeper,
193+[platonic abstraction](http://en.wikipedia.org/wiki/Platonic_idealism),
194+where a streams is just an array in time, instead of in space.
195+And all the various streaming "abstractions" are just crude implementations
196+of this abstract idea.
197+
198+[classic-streams](https://github.com/joyent/node/blob/v0.8.16/doc/api/stream.markdown),
199+[new-streams](https://github.com/joyent/node/blob/v0.10/doc/api/stream.markdown),
200+[reducers](https://github.com/Gozala/reducers)
201+
202+The objective here is to find a simple realization of the best features of the above.
203+
204+### Type Agnostic
205+
206+A stream abstraction should be able to handle both streams of text and streams
207+of objects.
208+
209+### A pipeline is also a stream.
210+
211+Something like this should work: `a.pipe(x.pipe(y).pipe(z)).pipe(b)`
212+this makes it possible to write a custom stream simply by
213+combining a few available streams.
214+
215+### Propagate End/Error conditions.
216+
217+If a stream ends in an unexpected way (error),
218+then other streams in the pipeline should be notified.
219+(this is a problem in node streams - when an error occurs,
220+the stream is disconnected, and the user must handle that specially)
221+
222+Also, the stream should be able to be ended from either end.
223+
224+### Transparent Backpressure & Laziness
225+
226+Very simple transform streams must be able to transfer back pressure
227+instantly.
228+
229+This is a problem in node streams, pause is only transfered on write, so
230+on a long chain (`a.pipe(b).pipe(c)`), if `c` pauses, `b` will have to write to it
231+to pause, and then `a` will have to write to `b` to pause.
232+If `b` only transforms `a`'s output, then `a` will have to write to `b` twice to
233+find out that `c` is paused.
234+
235+[reducers](https://github.com/Gozala/reducers) reducers has an interesting method,
236+where synchronous tranformations propagate back pressure instantly!
237+
238+This means you can have two "smart" streams doing io at the ends, and lots of dumb
239+streams in the middle, and back pressure will work perfectly, as if the dumb streams
240+are not there.
241+
242+This makes laziness work right.
243+
244+### handling end, error, and abort.
245+
246+in pull streams, any part of the stream (source, sink, or through)
247+may terminate the stream. (this is the case with node streams too,
248+but it's not handled well).
249+
250+#### source: end, error
251+
252+A source may end (`cb(true)` after read) or error (`cb(error)` after read)
253+After ending, the source *must* never `cb(null, data)`
254+
255+#### sink: abort
256+
257+Sinks do not normally end the stream, but if they decide they do
258+not need any more data they may "abort" the source by calling `read(true, cb)`.
259+A abort (`read(true, cb)`) may be called before a preceding read call
260+has called back.
261+
262+### handling end/abort/error in through streams
263+
264+Rules for implementing `read` in a through stream:
265+1) Sink wants to stop. sink aborts the through
266+
267+ just forward the exact read() call to your source,
268+ any future read calls should cb(true).
269+
270+2) We want to stop. (abort from the middle of the stream)
271+
272+ abort your source, and then cb(true) to tell the sink we have ended.
273+ If the source errored during abort, end the sink by cb read with `cb(err)`.
274+ (this will be an ordinary end/error for the sink)
275+
276+3) Source wants to stop. (`read(null, cb) -> cb(err||true)`)
277+
278+ forward that exact callback towards the sink chain,
279+ we must respond to any future read calls with `cb(err||true)`.
280+
281+In none of the above cases data is flowing!
282+4) If data is flowing (normal operation: `read(null, cb) -> cb(null, data)`
283+
284+ forward data downstream (towards the Sink)
285+ do none of the above!
286+
287+There either is data flowing (4) OR you have the error/abort cases (1-3), never both.
288+
289+
290+## License
291+
292+MIT
tmpl/apis/pull-stream/pull-timeout.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-timeout.html',
7+ content: md.doc(__dirname+'/pull-timeout.md')
8+})
tmpl/apis/pull-stream/pull-timeout.mdView
@@ -1,0 +1,48 @@
1+# pull-timeout
2+
3+Timeout pull streams.
4+
5+If you have long running streams that depend on extermal resources, you might want abort the stream when timing out.
6+
7+## Usage
8+
9+`timeout(ms)`
10+
11+## Example
12+
13+```js
14+var pull = require("pull-stream");
15+var timeout = require("pull-timeout");
16+
17+pull(
18+ pull.values([1,2,3,4,5,6,7,8,9,10]),
19+ pull.asyncMap( function (data, done) {
20+ setTimeout( function () {
21+ done(null, data);
22+ }, Math.round(Math.random()*4) == 0 ? 1500 : 100)
23+ }),
24+ timeout(1000),
25+ pull.Through( function (read) {
26+ return function next (end, cb) {
27+ read(end, function (end, data) {
28+ console.log(end, data);
29+ if (end && end !== true) return next(null, cb);
30+ cb(end, data);
31+ })
32+ }
33+ })(),
34+ pull.drain(function (){})
35+)
36+```
37+
38+## install
39+
40+With [npm](https://npmjs.org) do:
41+
42+```
43+npm install pull-timeout
44+```
45+
46+## license
47+
48+MIT
tmpl/apis/pull-stream/pull-window.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-window.html',
7+ content: md.doc(__dirname+'/pull-window.md')
8+})
tmpl/apis/pull-stream/pull-window.mdView
@@ -1,0 +1,152 @@
1+# pull-window
2+
3+Aggregate a pull-stream into windows.
4+
5+Several helpers are provided for particular types of windows,
6+sliding, tumbling, etc.
7+
8+And also, a low level
9+
10+## Example: "tumbling" window
11+
12+sum every 10 items.
13+
14+``` js
15+var pull = require('pull-stream')
16+var window = require('pull-window')
17+
18+function everyTen () {
19+ var i = 0
20+ //window calls init with each data item,
21+ //and a callback to close that window.
22+ return window(function (data, cb) {
23+ //if you don't want to start a window here,
24+ //return undefined
25+ if(i != 0) return
26+ var sum = 0
27+
28+ //else return a function.
29+ //this will be called all data
30+ //until you callback.
31+ return function (end, data) {
32+ if(end) return cb(null, sum)
33+ sum += data
34+ if(++i >= 10) {
35+ i = 0
36+ cb(null, sum)
37+ }
38+ }
39+ }
40+}
41+
42+pull(
43+ pull.count(1000),
44+ everyTen(),
45+ pull.log()
46+)
47+```
48+
49+## Example: variable sized window
50+
51+Each window doesn't have to be the same size...
52+
53+``` js
54+var pull = require('pull-stream')
55+var window = require('pull-window')
56+
57+function groupTo100 () {
58+ var sum = null
59+ return window(function (_, cb) {
60+ if(sum != null) return
61+
62+ //sum stuff together until you have 100 or more
63+ return function (end, data) {
64+ if(end) return cb(null, sum)
65+ sum += data
66+ if(sum >= 100) {
67+ //copy sum like this, incase the next item
68+ //comes through sync
69+ var _sum = sum; sum = null
70+ cb(null, _sum)
71+ }
72+ }
73+ })
74+}
75+
76+pull(
77+ pull.count(1000)
78+ groupTo100(),
79+ pull.log()
80+)
81+```
82+
83+## Example: sliding window
84+
85+to make more over lapping windows
86+just return the window function more often.
87+
88+``` js
89+var pull = require('pull-stream')
90+var window = require('pull-window')
91+
92+function sliding () {
93+ return window(function (_, cb) {
94+ var sum = 0, i = 0
95+
96+ //sum stuff together until you have 100 or more
97+ return function (end, data) {
98+ if(end) return cb(null, sum)
99+ sum += data
100+ if(++i >= 10) {
101+ //in this example, each window gets it's own sum,
102+ //so we don't need to copy it.
103+ cb(null, sum)
104+ }
105+ }
106+ })
107+}
108+
109+pull(
110+ pull.count(100)
111+ sliding(),
112+ pull.log()
113+)
114+```
115+
116+
117+## API
118+
119+
120+### window (start, map)
121+``` js
122+
123+window(function startWindow (data, cb) {
124+
125+ //called on each chunk
126+ //including the first one
127+ return function addToWindow (end, data) {
128+ //cb(null, aggregate) when done.
129+ }
130+}, function mapWindow (start, data) {
131+ //(optional)
132+ //map the window to something that tracks start, also
133+})
134+```
135+
136+By default, windows are mapped to `{start: firstData, data: aggregate}`.
137+unless you pass in an different `mapWindow` function.
138+
139+
140+### window.sliding(reduce, size)
141+
142+reduce every `size` items into a single value, in a sliding window
143+
144+### window.recent(size, time)
145+
146+tumbling window that groups items onto an array,
147+either every `size` items, or within `time` ms,
148+which ever occurs earliest.
149+
150+## License
151+
152+MIT
tmpl/apis/pull-stream/pull-ws-server.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/pull-ws-server.html',
7+ content: md.doc(__dirname+'/pull-ws-server.md')
8+})
tmpl/apis/pull-stream/pull-ws-server.mdView
@@ -1,0 +1,53 @@
1+# pull-ws-server
2+
3+create pull stream websockets, servers, and clients.
4+
5+## example
6+
7+one duplex service you may want to use this with is [muxrpc](https://github.com/dominictarr/muxrpc)
8+
9+``` js
10+var ws = require('pull-ws-server')
11+var pull = require('pull-stream')
12+
13+ws.createServer(function (stream) {
14+ //pipe duplex style to your service.
15+ pull(stream, service.createStream(), stream)
16+})
17+.listen(9999)
18+
19+var stream = ws.connect('ws://localhost:9999')
20+
21+pull(stream, client.createStream(), stream)
22+```
23+
24+if the connection fails, the first read from the stream will be an error,
25+otherwise, to get a handle of stream end/error pass a callback to connect.
26+
27+``` js
28+ws.connect('ws://localhost:9999', function (err, stream) {
29+ if(err) return handleError(err)
30+ //stream is now ready
31+})
32+
33+```
34+
35+To run the server over TLS:
36+
37+```js
38+var tlsOpts = {
39+ key: fs.readFileSync('test/fixtures/keys/agent2-key.pem'),
40+ cert: fs.readFileSync('test/fixtures/keys/agent2-cert.pem')
41+};
42+ws.createServer(tlsOpts, function (stream) {
43+ //pipe duplex style to your service.
44+ pull(stream, service.createStream(), stream)
45+})
46+.listen(9999)
47+```
48+
49+## License
50+
51+MIT
52+
53+
tmpl/apis/pull-stream/stream-to-pull-stream.html.jsView
@@ -1,0 +1,8 @@
1+var md = require('../../../markdown')
2+var page = require('../../page.part')
3+module.exports = () => page({
4+ section: 'apis',
5+ tab: 'apis-pull-stream',
6+ path: '/apis/pull-stream/stream-to-pull-stream.html',
7+ content: md.doc(__dirname+'/stream-to-pull-stream.md')
8+})
tmpl/apis/pull-stream/stream-to-pull-stream.mdView
@@ -1,0 +1,27 @@
1+# stream-to-pull-stream
2+
3+Convert a classic-stream, or a new-stream into a
4+[pull-stream](https://github.com/dominictarr/pull-stream)
5+
6+## example
7+
8+``` js
9+var toPull = require('stream-to-pull-stream')
10+var pull = require('pull-stream')
11+
12+pull(
13+ toPull.source(fs.createReadStream(__filename)),
14+ pull.map(function (e) { return e.toString().toUpperCase() }),
15+ toPull.sink(process.stdout, function (err) {
16+ if(err) throw err
17+ console.log('done')
18+ })
19+)
20+```
21+
22+if the node steam is a duplex (i.e. net, ws) then use `toPull.duplex(stream, cb?)`
23+`duplex` takes an optional callback in the same way that `sink` does.
24+
25+## License
26+
27+MIT
tmpl/css/com.part.jsView
@@ -42,8 +42,11 @@
4242 margin-left: 1em;
4343 padding: 0.25em 0.25em 0;
4444 }
4545
46+.code-examples {
47+ max-width: 540px;
48+}
4649 .code-examples .head {
4750 display: flex;
4851 color: #333;
4952 }
tmpl/css/index.css.jsView
@@ -18,8 +18,11 @@
1818 }
1919 pre {
2020 font-size: 14px;
2121 }
22+pre code {
23+ white-space: pre-wrap;
24+}
2225
2326 p {
2427 line-height: 1.5;
2528 }
@@ -45,8 +48,9 @@
4548 box-sizing: border-box;
4649 }
4750
4851 .content {
52+ max-width: 540px;
4953 flex: 1;
5054 padding: 2em 1em 2em 2em;
5155 }
5256 .content > :first-child {
tmpl/leftnav.part.jsView
@@ -72,10 +72,23 @@
7272 </ul>`
7373
7474 module.exports['apis-pull-stream'] = (c) => `<ul class="nav">
7575 ${item(c, '/apis/pull-stream/pull-stream.html', 'Pull-Stream')}
76+ ${item(c, '/apis/pull-stream/core-sources.html', 'Source Functions')}
77+ ${item(c, '/apis/pull-stream/core-throughs.html', 'Through Functions')}
78+ ${item(c, '/apis/pull-stream/core-sinks.html', 'Sink Functions')}
79+ ${item(c, '/apis/pull-stream/pull-stream-to-stream.html', 'Pull-Stream-to-Stream')}
80+ ${item(c, '/apis/pull-stream/stream-to-pull-stream.html', 'Stream-to-Pull-Stream')}
81+ ${item(c, '/apis/pull-stream/pull-paramap.html', 'Pull-Paramap')}
82+ ${item(c, '/apis/pull-stream/pull-cat.html', 'Pull-Cat')}
83+ ${item(c, '/apis/pull-stream/pull-pushable.html', 'Pull-Pushable')}
84+ ${item(c, '/apis/pull-stream/pull-notify.html', 'Pull-Notify')}
85+ ${item(c, '/apis/pull-stream/pull-pause.html', 'Pull-Pause')}
86+ ${item(c, '/apis/pull-stream/pull-timeout.html', 'Pull-Timeout')}
87+ ${item(c, '/apis/pull-stream/pull-window.html', 'Pull-Window')}
88+ ${item(c, '/apis/pull-stream/pull-otherwise.html', 'Pull-Otherwise')}
7689 ${item(c, '/apis/pull-stream/pull-ws-server.html', 'Pull-WS-Server')}
77-`
90+</ul>`
7891
7992 module.exports['guides-concepts'] = (c) => `<ul class="nav">
8093 ${item(c, '/guides/concepts/intro.html', 'Intro')}
8194 </ul>`
tmpl/tabs.part.jsView
@@ -22,9 +22,9 @@
2222
2323 module.exports.apis = (c) => `<div class="tabs small">
2424 ${item(c, 'apis-scuttlebot', '/apis/scuttlebot/ssb.html', 'Scuttlebot')}
2525 ${item(c, 'apis-modules', '/apis/modules/ssb-client.html', 'Modules')}
26- ${item(c, 'apis-pull-stream', '/apis/pull-stream/ssb-client.html', 'Pull Stream')}
26+ ${item(c, 'apis-pull-stream', '/apis/pull-stream/pull-stream.html', 'Pull Stream')}
2727 ${item(c, 'apis-community', '/apis/community/ssbify.html', 'Community')}
2828 </div>`
2929
3030 module.exports.apps = (c) => `<div class="tabs small">

Built with git-ssb-web