ffc1185bd0baf0e5fe9fba17fe192602251e0abf184f96ff19908ded3810b09540ad8a9b6173de085a4da8df1fdb310343824d51bdb0e55bb0d850db291428 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769
  1. # minipass
  2. A _very_ minimal implementation of a [PassThrough
  3. stream](https://nodejs.org/api/stream.html#stream_class_stream_passthrough)
  4. [It's very
  5. fast](https://docs.google.com/spreadsheets/d/1K_HR5oh3r80b8WVMWCPPjfuWXUgfkmhlX7FGI6JJ8tY/edit?usp=sharing)
  6. for objects, strings, and buffers.
  7. Supports `pipe()`ing (including multi-`pipe()` and backpressure
  8. transmission), buffering data until either a `data` event handler
  9. or `pipe()` is added (so you don't lose the first chunk), and
  10. most other cases where PassThrough is a good idea.
  11. There is a `read()` method, but it's much more efficient to
  12. consume data from this stream via `'data'` events or by calling
  13. `pipe()` into some other stream. Calling `read()` requires the
  14. buffer to be flattened in some cases, which requires copying
  15. memory.
  16. If you set `objectMode: true` in the options, then whatever is
  17. written will be emitted. Otherwise, it'll do a minimal amount of
  18. Buffer copying to ensure proper Streams semantics when `read(n)`
  19. is called.
  20. `objectMode` can also be set by doing `stream.objectMode = true`,
  21. or by writing any non-string/non-buffer data. `objectMode` cannot
  22. be set to false once it is set.
  23. This is not a `through` or `through2` stream. It doesn't
  24. transform the data, it just passes it right through. If you want
  25. to transform the data, extend the class, and override the
  26. `write()` method. Once you're done transforming the data however
  27. you want, call `super.write()` with the transform output.
  28. For some examples of streams that extend Minipass in various
  29. ways, check out:
  30. - [minizlib](http://npm.im/minizlib)
  31. - [fs-minipass](http://npm.im/fs-minipass)
  32. - [tar](http://npm.im/tar)
  33. - [minipass-collect](http://npm.im/minipass-collect)
  34. - [minipass-flush](http://npm.im/minipass-flush)
  35. - [minipass-pipeline](http://npm.im/minipass-pipeline)
  36. - [tap](http://npm.im/tap)
  37. - [tap-parser](http://npm.im/tap-parser)
  38. - [treport](http://npm.im/treport)
  39. - [minipass-fetch](http://npm.im/minipass-fetch)
  40. - [pacote](http://npm.im/pacote)
  41. - [make-fetch-happen](http://npm.im/make-fetch-happen)
  42. - [cacache](http://npm.im/cacache)
  43. - [ssri](http://npm.im/ssri)
  44. - [npm-registry-fetch](http://npm.im/npm-registry-fetch)
  45. - [minipass-json-stream](http://npm.im/minipass-json-stream)
  46. - [minipass-sized](http://npm.im/minipass-sized)
  47. ## Differences from Node.js Streams
  48. There are several things that make Minipass streams different
  49. from (and in some ways superior to) Node.js core streams.
  50. Please read these caveats if you are familiar with node-core
  51. streams and intend to use Minipass streams in your programs.
  52. You can avoid most of these differences entirely (for a very
  53. small performance penalty) by setting `{async: true}` in the
  54. constructor options.
  55. ### Timing
  56. Minipass streams are designed to support synchronous use-cases.
  57. Thus, data is emitted as soon as it is available, always. It is
  58. buffered until read, but no longer. Another way to look at it is
  59. that Minipass streams are exactly as synchronous as the logic
  60. that writes into them.
  61. This can be surprising if your code relies on
  62. `PassThrough.write()` always providing data on the next tick
  63. rather than the current one, or being able to call `resume()` and
  64. not have the entire buffer disappear immediately.
  65. However, without this synchronicity guarantee, there would be no
  66. way for Minipass to achieve the speeds it does, or support the
  67. synchronous use cases that it does. Simply put, waiting takes
  68. time.
  69. This non-deferring approach makes Minipass streams much easier to
  70. reason about, especially in the context of Promises and other
  71. flow-control mechanisms.
  72. Example:
  73. ```js
  74. // hybrid module, either works
  75. import { Minipass } from 'minipass'
  76. // or:
  77. const { Minipass } = require('minipass')
  78. const stream = new Minipass()
  79. stream.on('data', () => console.log('data event'))
  80. console.log('before write')
  81. stream.write('hello')
  82. console.log('after write')
  83. // output:
  84. // before write
  85. // data event
  86. // after write
  87. ```
  88. ### Exception: Async Opt-In
  89. If you wish to have a Minipass stream with behavior that more
  90. closely mimics Node.js core streams, you can set the stream in
  91. async mode either by setting `async: true` in the constructor
  92. options, or by setting `stream.async = true` later on.
  93. ```js
  94. // hybrid module, either works
  95. import { Minipass } from 'minipass'
  96. // or:
  97. const { Minipass } = require('minipass')
  98. const asyncStream = new Minipass({ async: true })
  99. asyncStream.on('data', () => console.log('data event'))
  100. console.log('before write')
  101. asyncStream.write('hello')
  102. console.log('after write')
  103. // output:
  104. // before write
  105. // after write
  106. // data event <-- this is deferred until the next tick
  107. ```
  108. Switching _out_ of async mode is unsafe, as it could cause data
  109. corruption, and so is not enabled. Example:
  110. ```js
  111. import { Minipass } from 'minipass'
  112. const stream = new Minipass({ encoding: 'utf8' })
  113. stream.on('data', chunk => console.log(chunk))
  114. stream.async = true
  115. console.log('before writes')
  116. stream.write('hello')
  117. setStreamSyncAgainSomehow(stream) // <-- this doesn't actually exist!
  118. stream.write('world')
  119. console.log('after writes')
  120. // hypothetical output would be:
  121. // before writes
  122. // world
  123. // after writes
  124. // hello
  125. // NOT GOOD!
  126. ```
  127. To avoid this problem, once set into async mode, any attempt to
  128. make the stream sync again will be ignored.
  129. ```js
  130. const { Minipass } = require('minipass')
  131. const stream = new Minipass({ encoding: 'utf8' })
  132. stream.on('data', chunk => console.log(chunk))
  133. stream.async = true
  134. console.log('before writes')
  135. stream.write('hello')
  136. stream.async = false // <-- no-op, stream already async
  137. stream.write('world')
  138. console.log('after writes')
  139. // actual output:
  140. // before writes
  141. // after writes
  142. // hello
  143. // world
  144. ```
  145. ### No High/Low Water Marks
  146. Node.js core streams will optimistically fill up a buffer,
  147. returning `true` on all writes until the limit is hit, even if
  148. the data has nowhere to go. Then, they will not attempt to draw
  149. more data in until the buffer size dips below a minimum value.
  150. Minipass streams are much simpler. The `write()` method will
  151. return `true` if the data has somewhere to go (which is to say,
  152. given the timing guarantees, that the data is already there by
  153. the time `write()` returns).
  154. If the data has nowhere to go, then `write()` returns false, and
  155. the data sits in a buffer, to be drained out immediately as soon
  156. as anyone consumes it.
  157. Since nothing is ever buffered unnecessarily, there is much less
  158. copying data, and less bookkeeping about buffer capacity levels.
  159. ### Hazards of Buffering (or: Why Minipass Is So Fast)
  160. Since data written to a Minipass stream is immediately written
  161. all the way through the pipeline, and `write()` always returns
  162. true/false based on whether the data was fully flushed,
  163. backpressure is communicated immediately to the upstream caller.
  164. This minimizes buffering.
  165. Consider this case:
  166. ```js
  167. const { PassThrough } = require('stream')
  168. const p1 = new PassThrough({ highWaterMark: 1024 })
  169. const p2 = new PassThrough({ highWaterMark: 1024 })
  170. const p3 = new PassThrough({ highWaterMark: 1024 })
  171. const p4 = new PassThrough({ highWaterMark: 1024 })
  172. p1.pipe(p2).pipe(p3).pipe(p4)
  173. p4.on('data', () => console.log('made it through'))
  174. // this returns false and buffers, then writes to p2 on next tick (1)
  175. // p2 returns false and buffers, pausing p1, then writes to p3 on next tick (2)
  176. // p3 returns false and buffers, pausing p2, then writes to p4 on next tick (3)
  177. // p4 returns false and buffers, pausing p3, then emits 'data' and 'drain'
  178. // on next tick (4)
  179. // p3 sees p4's 'drain' event, and calls resume(), emitting 'resume' and
  180. // 'drain' on next tick (5)
  181. // p2 sees p3's 'drain', calls resume(), emits 'resume' and 'drain' on next tick (6)
  182. // p1 sees p2's 'drain', calls resume(), emits 'resume' and 'drain' on next
  183. // tick (7)
  184. p1.write(Buffer.alloc(2048)) // returns false
  185. ```
  186. Along the way, the data was buffered and deferred at each stage,
  187. and multiple event deferrals happened, for an unblocked pipeline
  188. where it was perfectly safe to write all the way through!
  189. Furthermore, setting a `highWaterMark` of `1024` might lead
  190. someone reading the code to think an advisory maximum of 1KiB is
  191. being set for the pipeline. However, the actual advisory
  192. buffering level is the _sum_ of `highWaterMark` values, since
  193. each one has its own bucket.
  194. Consider the Minipass case:
  195. ```js
  196. const m1 = new Minipass()
  197. const m2 = new Minipass()
  198. const m3 = new Minipass()
  199. const m4 = new Minipass()
  200. m1.pipe(m2).pipe(m3).pipe(m4)
  201. m4.on('data', () => console.log('made it through'))
  202. // m1 is flowing, so it writes the data to m2 immediately
  203. // m2 is flowing, so it writes the data to m3 immediately
  204. // m3 is flowing, so it writes the data to m4 immediately
  205. // m4 is flowing, so it fires the 'data' event immediately, returns true
  206. // m4's write returned true, so m3 is still flowing, returns true
  207. // m3's write returned true, so m2 is still flowing, returns true
  208. // m2's write returned true, so m1 is still flowing, returns true
  209. // No event deferrals or buffering along the way!
  210. m1.write(Buffer.alloc(2048)) // returns true
  211. ```
  212. It is extremely unlikely that you _don't_ want to buffer any data
  213. written, or _ever_ buffer data that can be flushed all the way
  214. through. Neither node-core streams nor Minipass ever fail to
  215. buffer written data, but node-core streams do a lot of
  216. unnecessary buffering and pausing.
  217. As always, the faster implementation is the one that does less
  218. stuff and waits less time to do it.
  219. ### Immediately emit `end` for empty streams (when not paused)
  220. If a stream is not paused, and `end()` is called before writing
  221. any data into it, then it will emit `end` immediately.
  222. If you have logic that occurs on the `end` event which you don't
  223. want to potentially happen immediately (for example, closing file
  224. descriptors, moving on to the next entry in an archive parse
  225. stream, etc.) then be sure to call `stream.pause()` on creation,
  226. and then `stream.resume()` once you are ready to respond to the
  227. `end` event.
  228. However, this is _usually_ not a problem because:
  229. ### Emit `end` When Asked
  230. One hazard of immediately emitting `'end'` is that you may not
  231. yet have had a chance to add a listener. In order to avoid this
  232. hazard, Minipass streams safely re-emit the `'end'` event if a
  233. new listener is added after `'end'` has been emitted.
  234. Ie, if you do `stream.on('end', someFunction)`, and the stream
  235. has already emitted `end`, then it will call the handler right
  236. away. (You can think of this somewhat like attaching a new
  237. `.then(fn)` to a previously-resolved Promise.)
  238. To prevent calling handlers multiple times who would not expect
  239. multiple ends to occur, all listeners are removed from the
  240. `'end'` event whenever it is emitted.
  241. ### Emit `error` When Asked
  242. The most recent error object passed to the `'error'` event is
  243. stored on the stream. If a new `'error'` event handler is added,
  244. and an error was previously emitted, then the event handler will
  245. be called immediately (or on `process.nextTick` in the case of
  246. async streams).
  247. This makes it much more difficult to end up trying to interact
  248. with a broken stream, if the error handler is added after an
  249. error was previously emitted.
  250. ### Impact of "immediate flow" on Tee-streams
  251. A "tee stream" is a stream piping to multiple destinations:
  252. ```js
  253. const tee = new Minipass()
  254. t.pipe(dest1)
  255. t.pipe(dest2)
  256. t.write('foo') // goes to both destinations
  257. ```
  258. Since Minipass streams _immediately_ process any pending data
  259. through the pipeline when a new pipe destination is added, this
  260. can have surprising effects, especially when a stream comes in
  261. from some other function and may or may not have data in its
  262. buffer.
  263. ```js
  264. // WARNING! WILL LOSE DATA!
  265. const src = new Minipass()
  266. src.write('foo')
  267. src.pipe(dest1) // 'foo' chunk flows to dest1 immediately, and is gone
  268. src.pipe(dest2) // gets nothing!
  269. ```
  270. One solution is to create a dedicated tee-stream junction that
  271. pipes to both locations, and then pipe to _that_ instead.
  272. ```js
  273. // Safe example: tee to both places
  274. const src = new Minipass()
  275. src.write('foo')
  276. const tee = new Minipass()
  277. tee.pipe(dest1)
  278. tee.pipe(dest2)
  279. src.pipe(tee) // tee gets 'foo', pipes to both locations
  280. ```
  281. The same caveat applies to `on('data')` event listeners. The
  282. first one added will _immediately_ receive all of the data,
  283. leaving nothing for the second:
  284. ```js
  285. // WARNING! WILL LOSE DATA!
  286. const src = new Minipass()
  287. src.write('foo')
  288. src.on('data', handler1) // receives 'foo' right away
  289. src.on('data', handler2) // nothing to see here!
  290. ```
  291. Using a dedicated tee-stream can be used in this case as well:
  292. ```js
  293. // Safe example: tee to both data handlers
  294. const src = new Minipass()
  295. src.write('foo')
  296. const tee = new Minipass()
  297. tee.on('data', handler1)
  298. tee.on('data', handler2)
  299. src.pipe(tee)
  300. ```
  301. All of the hazards in this section are avoided by setting `{
  302. async: true }` in the Minipass constructor, or by setting
  303. `stream.async = true` afterwards. Note that this does add some
  304. overhead, so should only be done in cases where you are willing
  305. to lose a bit of performance in order to avoid having to refactor
  306. program logic.
  307. ## USAGE
  308. It's a stream! Use it like a stream and it'll most likely do what
  309. you want.
  310. ```js
  311. import { Minipass } from 'minipass'
  312. const mp = new Minipass(options) // optional: { encoding, objectMode }
  313. mp.write('foo')
  314. mp.pipe(someOtherStream)
  315. mp.end('bar')
  316. ```
  317. ### OPTIONS
  318. - `encoding` How would you like the data coming _out_ of the
  319. stream to be encoded? Accepts any values that can be passed to
  320. `Buffer.toString()`.
  321. - `objectMode` Emit data exactly as it comes in. This will be
  322. flipped on by default if you write() something other than a
  323. string or Buffer at any point. Setting `objectMode: true` will
  324. prevent setting any encoding value.
  325. - `async` Defaults to `false`. Set to `true` to defer data
  326. emission until next tick. This reduces performance slightly,
  327. but makes Minipass streams use timing behavior closer to Node
  328. core streams. See [Timing](#timing) for more details.
  329. - `signal` An `AbortSignal` that will cause the stream to unhook
  330. itself from everything and become as inert as possible. Note
  331. that providing a `signal` parameter will make `'error'` events
  332. no longer throw if they are unhandled, but they will still be
  333. emitted to handlers if any are attached.
  334. ### API
  335. Implements the user-facing portions of Node.js's `Readable` and
  336. `Writable` streams.
  337. ### Methods
  338. - `write(chunk, [encoding], [callback])` - Put data in. (Note
  339. that, in the base Minipass class, the same data will come out.)
  340. Returns `false` if the stream will buffer the next write, or
  341. true if it's still in "flowing" mode.
  342. - `end([chunk, [encoding]], [callback])` - Signal that you have
  343. no more data to write. This will queue an `end` event to be
  344. fired when all the data has been consumed.
  345. - `setEncoding(encoding)` - Set the encoding for data coming of
  346. the stream. This can only be done once.
  347. - `pause()` - No more data for a while, please. This also
  348. prevents `end` from being emitted for empty streams until the
  349. stream is resumed.
  350. - `resume()` - Resume the stream. If there's data in the buffer,
  351. it is all discarded. Any buffered events are immediately
  352. emitted.
  353. - `pipe(dest)` - Send all output to the stream provided. When
  354. data is emitted, it is immediately written to any and all pipe
  355. destinations. (Or written on next tick in `async` mode.)
  356. - `unpipe(dest)` - Stop piping to the destination stream. This is
  357. immediate, meaning that any asynchronously queued data will
  358. _not_ make it to the destination when running in `async` mode.
  359. - `options.end` - Boolean, end the destination stream when the
  360. source stream ends. Default `true`.
  361. - `options.proxyErrors` - Boolean, proxy `error` events from
  362. the source stream to the destination stream. Note that errors
  363. are _not_ proxied after the pipeline terminates, either due
  364. to the source emitting `'end'` or manually unpiping with
  365. `src.unpipe(dest)`. Default `false`.
  366. - `on(ev, fn)`, `emit(ev, fn)` - Minipass streams are
  367. EventEmitters. Some events are given special treatment,
  368. however. (See below under "events".)
  369. - `promise()` - Returns a Promise that resolves when the stream
  370. emits `end`, or rejects if the stream emits `error`.
  371. - `collect()` - Return a Promise that resolves on `end` with an
  372. array containing each chunk of data that was emitted, or
  373. rejects if the stream emits `error`. Note that this consumes
  374. the stream data.
  375. - `concat()` - Same as `collect()`, but concatenates the data
  376. into a single Buffer object. Will reject the returned promise
  377. if the stream is in objectMode, or if it goes into objectMode
  378. by the end of the data.
  379. - `read(n)` - Consume `n` bytes of data out of the buffer. If `n`
  380. is not provided, then consume all of it. If `n` bytes are not
  381. available, then it returns null. **Note** consuming streams in
  382. this way is less efficient, and can lead to unnecessary Buffer
  383. copying.
  384. - `destroy([er])` - Destroy the stream. If an error is provided,
  385. then an `'error'` event is emitted. If the stream has a
  386. `close()` method, and has not emitted a `'close'` event yet,
  387. then `stream.close()` will be called. Any Promises returned by
  388. `.promise()`, `.collect()` or `.concat()` will be rejected.
  389. After being destroyed, writing to the stream will emit an
  390. error. No more data will be emitted if the stream is destroyed,
  391. even if it was previously buffered.
  392. ### Properties
  393. - `bufferLength` Read-only. Total number of bytes buffered, or in
  394. the case of objectMode, the total number of objects.
  395. - `encoding` The encoding that has been set. (Setting this is
  396. equivalent to calling `setEncoding(enc)` and has the same
  397. prohibition against setting multiple times.)
  398. - `flowing` Read-only. Boolean indicating whether a chunk written
  399. to the stream will be immediately emitted.
  400. - `emittedEnd` Read-only. Boolean indicating whether the end-ish
  401. events (ie, `end`, `prefinish`, `finish`) have been emitted.
  402. Note that listening on any end-ish event will immediateyl
  403. re-emit it if it has already been emitted.
  404. - `writable` Whether the stream is writable. Default `true`. Set
  405. to `false` when `end()`
  406. - `readable` Whether the stream is readable. Default `true`.
  407. - `pipes` An array of Pipe objects referencing streams that this
  408. stream is piping into.
  409. - `destroyed` A getter that indicates whether the stream was
  410. destroyed.
  411. - `paused` True if the stream has been explicitly paused,
  412. otherwise false.
  413. - `objectMode` Indicates whether the stream is in `objectMode`.
  414. Once set to `true`, it cannot be set to `false`.
  415. - `aborted` Readonly property set when the `AbortSignal`
  416. dispatches an `abort` event.
  417. ### Events
  418. - `data` Emitted when there's data to read. Argument is the data
  419. to read. This is never emitted while not flowing. If a listener
  420. is attached, that will resume the stream.
  421. - `end` Emitted when there's no more data to read. This will be
  422. emitted immediately for empty streams when `end()` is called.
  423. If a listener is attached, and `end` was already emitted, then
  424. it will be emitted again. All listeners are removed when `end`
  425. is emitted.
  426. - `prefinish` An end-ish event that follows the same logic as
  427. `end` and is emitted in the same conditions where `end` is
  428. emitted. Emitted after `'end'`.
  429. - `finish` An end-ish event that follows the same logic as `end`
  430. and is emitted in the same conditions where `end` is emitted.
  431. Emitted after `'prefinish'`.
  432. - `close` An indication that an underlying resource has been
  433. released. Minipass does not emit this event, but will defer it
  434. until after `end` has been emitted, since it throws off some
  435. stream libraries otherwise.
  436. - `drain` Emitted when the internal buffer empties, and it is
  437. again suitable to `write()` into the stream.
  438. - `readable` Emitted when data is buffered and ready to be read
  439. by a consumer.
  440. - `resume` Emitted when stream changes state from buffering to
  441. flowing mode. (Ie, when `resume` is called, `pipe` is called,
  442. or a `data` event listener is added.)
  443. ### Static Methods
  444. - `Minipass.isStream(stream)` Returns `true` if the argument is a
  445. stream, and false otherwise. To be considered a stream, the
  446. object must be either an instance of Minipass, or an
  447. EventEmitter that has either a `pipe()` method, or both
  448. `write()` and `end()` methods. (Pretty much any stream in
  449. node-land will return `true` for this.)
  450. ## EXAMPLES
  451. Here are some examples of things you can do with Minipass
  452. streams.
  453. ### simple "are you done yet" promise
  454. ```js
  455. mp.promise().then(
  456. () => {
  457. // stream is finished
  458. },
  459. er => {
  460. // stream emitted an error
  461. }
  462. )
  463. ```
  464. ### collecting
  465. ```js
  466. mp.collect().then(all => {
  467. // all is an array of all the data emitted
  468. // encoding is supported in this case, so
  469. // so the result will be a collection of strings if
  470. // an encoding is specified, or buffers/objects if not.
  471. //
  472. // In an async function, you may do
  473. // const data = await stream.collect()
  474. })
  475. ```
  476. ### collecting into a single blob
  477. This is a bit slower because it concatenates the data into one
  478. chunk for you, but if you're going to do it yourself anyway, it's
  479. convenient this way:
  480. ```js
  481. mp.concat().then(onebigchunk => {
  482. // onebigchunk is a string if the stream
  483. // had an encoding set, or a buffer otherwise.
  484. })
  485. ```
  486. ### iteration
  487. You can iterate over streams synchronously or asynchronously in
  488. platforms that support it.
  489. Synchronous iteration will end when the currently available data
  490. is consumed, even if the `end` event has not been reached. In
  491. string and buffer mode, the data is concatenated, so unless
  492. multiple writes are occurring in the same tick as the `read()`,
  493. sync iteration loops will generally only have a single iteration.
  494. To consume chunks in this way exactly as they have been written,
  495. with no flattening, create the stream with the `{ objectMode:
  496. true }` option.
  497. ```js
  498. const mp = new Minipass({ objectMode: true })
  499. mp.write('a')
  500. mp.write('b')
  501. for (let letter of mp) {
  502. console.log(letter) // a, b
  503. }
  504. mp.write('c')
  505. mp.write('d')
  506. for (let letter of mp) {
  507. console.log(letter) // c, d
  508. }
  509. mp.write('e')
  510. mp.end()
  511. for (let letter of mp) {
  512. console.log(letter) // e
  513. }
  514. for (let letter of mp) {
  515. console.log(letter) // nothing
  516. }
  517. ```
  518. Asynchronous iteration will continue until the end event is reached,
  519. consuming all of the data.
  520. ```js
  521. const mp = new Minipass({ encoding: 'utf8' })
  522. // some source of some data
  523. let i = 5
  524. const inter = setInterval(() => {
  525. if (i-- > 0) mp.write(Buffer.from('foo\n', 'utf8'))
  526. else {
  527. mp.end()
  528. clearInterval(inter)
  529. }
  530. }, 100)
  531. // consume the data with asynchronous iteration
  532. async function consume() {
  533. for await (let chunk of mp) {
  534. console.log(chunk)
  535. }
  536. return 'ok'
  537. }
  538. consume().then(res => console.log(res))
  539. // logs `foo\n` 5 times, and then `ok`
  540. ```
  541. ### subclass that `console.log()`s everything written into it
  542. ```js
  543. class Logger extends Minipass {
  544. write(chunk, encoding, callback) {
  545. console.log('WRITE', chunk, encoding)
  546. return super.write(chunk, encoding, callback)
  547. }
  548. end(chunk, encoding, callback) {
  549. console.log('END', chunk, encoding)
  550. return super.end(chunk, encoding, callback)
  551. }
  552. }
  553. someSource.pipe(new Logger()).pipe(someDest)
  554. ```
  555. ### same thing, but using an inline anonymous class
  556. ```js
  557. // js classes are fun
  558. someSource
  559. .pipe(
  560. new (class extends Minipass {
  561. emit(ev, ...data) {
  562. // let's also log events, because debugging some weird thing
  563. console.log('EMIT', ev)
  564. return super.emit(ev, ...data)
  565. }
  566. write(chunk, encoding, callback) {
  567. console.log('WRITE', chunk, encoding)
  568. return super.write(chunk, encoding, callback)
  569. }
  570. end(chunk, encoding, callback) {
  571. console.log('END', chunk, encoding)
  572. return super.end(chunk, encoding, callback)
  573. }
  574. })()
  575. )
  576. .pipe(someDest)
  577. ```
  578. ### subclass that defers 'end' for some reason
  579. ```js
  580. class SlowEnd extends Minipass {
  581. emit(ev, ...args) {
  582. if (ev === 'end') {
  583. console.log('going to end, hold on a sec')
  584. setTimeout(() => {
  585. console.log('ok, ready to end now')
  586. super.emit('end', ...args)
  587. }, 100)
  588. } else {
  589. return super.emit(ev, ...args)
  590. }
  591. }
  592. }
  593. ```
  594. ### transform that creates newline-delimited JSON
  595. ```js
  596. class NDJSONEncode extends Minipass {
  597. write(obj, cb) {
  598. try {
  599. // JSON.stringify can throw, emit an error on that
  600. return super.write(JSON.stringify(obj) + '\n', 'utf8', cb)
  601. } catch (er) {
  602. this.emit('error', er)
  603. }
  604. }
  605. end(obj, cb) {
  606. if (typeof obj === 'function') {
  607. cb = obj
  608. obj = undefined
  609. }
  610. if (obj !== undefined) {
  611. this.write(obj)
  612. }
  613. return super.end(cb)
  614. }
  615. }
  616. ```
  617. ### transform that parses newline-delimited JSON
  618. ```js
  619. class NDJSONDecode extends Minipass {
  620. constructor (options) {
  621. // always be in object mode, as far as Minipass is concerned
  622. super({ objectMode: true })
  623. this._jsonBuffer = ''
  624. }
  625. write (chunk, encoding, cb) {
  626. if (typeof chunk === 'string' &&
  627. typeof encoding === 'string' &&
  628. encoding !== 'utf8') {
  629. chunk = Buffer.from(chunk, encoding).toString()
  630. } else if (Buffer.isBuffer(chunk)) {
  631. chunk = chunk.toString()
  632. }
  633. if (typeof encoding === 'function') {
  634. cb = encoding
  635. }
  636. const jsonData = (this._jsonBuffer + chunk).split('\n')
  637. this._jsonBuffer = jsonData.pop()
  638. for (let i = 0; i < jsonData.length; i++) {
  639. try {
  640. // JSON.parse can throw, emit an error on that
  641. super.write(JSON.parse(jsonData[i]))
  642. } catch (er) {
  643. this.emit('error', er)
  644. continue
  645. }
  646. }
  647. if (cb)
  648. cb()
  649. }
  650. }
  651. ```