-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
12 changed files
with
364 additions
and
0 deletions.
There are no files selected for viewing
1 change: 1 addition & 0 deletions
1
06-coding-with-streams/20-unordered-limited-parallel-execution/.gitignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| results.txt |
18 changes: 18 additions & 0 deletions
18
06-coding-with-streams/20-unordered-limited-parallel-execution/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| # 20-unordered-limited-parallel-execution | ||
|
|
||
| This example shows how to create a limited parallel execution flow using | ||
| streams. | ||
|
|
||
| ## Run | ||
|
|
||
| To run the example: | ||
|
|
||
| ```bash | ||
| node check-urls.js urls.txt | ||
| ``` | ||
|
|
||
| Then check the content of the results file with: | ||
|
|
||
| ```bash | ||
| cat results.txt | ||
| ``` |
29 changes: 29 additions & 0 deletions
29
06-coding-with-streams/20-unordered-limited-parallel-execution/check-urls.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| import { createReadStream, createWriteStream } from 'node:fs' | ||
| import { createInterface } from 'node:readline' | ||
| import { pipeline } from 'node:stream/promises' | ||
| import { LimitedParallelStream } from './limited-parallel-stream.js' | ||
|
|
||
| const inputFile = createReadStream(process.argv[2]) | ||
| const fileLines = createInterface({ | ||
| input: inputFile, | ||
| }) | ||
| const checkUrls = new LimitedParallelStream( | ||
| 8, | ||
| async (url, _enc, push, done) => { | ||
| if (!url) { | ||
| return done() | ||
| } | ||
| try { | ||
| await fetch(url, { method: 'HEAD', timeout: 5 * 1000 }) | ||
| push(`${url} is up\n`) | ||
| } catch (err) { | ||
| push(`${url} is down: ${err}\n`) | ||
| } | ||
| done() | ||
| } | ||
| ) | ||
| const outputFile = createWriteStream('results.txt') | ||
|
|
||
| await pipeline(fileLines, checkUrls, outputFile) | ||
|
|
||
| console.log('All urls have been checked') |
48 changes: 48 additions & 0 deletions
48
06-coding-with-streams/20-unordered-limited-parallel-execution/limited-parallel-stream.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| import { Transform } from 'node:stream' | ||
|
|
||
| export class LimitedParallelStream extends Transform { | ||
| constructor(concurrency, userTransform, opts) { | ||
| super({ objectMode: true, ...opts }) | ||
| this.concurrency = concurrency | ||
| this.userTransform = userTransform | ||
| this.running = 0 | ||
| this.continueCb = null | ||
| this.terminateCb = null | ||
| } | ||
|
|
||
| _transform(chunk, enc, done) { | ||
| this.running++ | ||
| this.userTransform( | ||
| chunk, | ||
| enc, | ||
| this.push.bind(this), | ||
| this._onComplete.bind(this) | ||
| ) | ||
| if (this.running < this.concurrency) { | ||
| done() | ||
| } else { | ||
| this.continueCb = done | ||
| } | ||
| } | ||
|
|
||
| _flush(done) { | ||
| if (this.running > 0) { | ||
| this.terminateCb = done | ||
| } else { | ||
| done() | ||
| } | ||
| } | ||
|
|
||
| _onComplete(err) { | ||
| this.running-- | ||
| if (err) { | ||
| return this.emit('error', err) | ||
| } | ||
| const tmpCb = this.continueCb | ||
| this.continueCb = null | ||
| tmpCb?.() | ||
| if (this.running === 0) { | ||
| this.terminateCb?.() | ||
| } | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
06-coding-with-streams/20-unordered-limited-parallel-execution/package.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| { | ||
| "name": "20-unordered-limited-parallel-execution", | ||
| "version": "1.0.0", | ||
| "description": "This example shows how to create a limited parallel execution flow using streams.", | ||
| "type": "module", | ||
| "scripts": {}, | ||
| "engines": { | ||
| "node": ">=22" | ||
| }, | ||
| "engineStrict": true, | ||
| "keywords": [], | ||
| "author": "Luciano Mammino and Mario Casciaro", | ||
| "license": "MIT", | ||
| "dependencies": {} | ||
| } |
50 changes: 50 additions & 0 deletions
50
06-coding-with-streams/20-unordered-limited-parallel-execution/urls.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| https://mario.fyi | ||
| https://loige.co | ||
| http://thiswillbedownforsure.com | ||
| https://google.com | ||
| https://youtube.com | ||
| https://facebook.com | ||
| https://twitter.com | ||
| https://instagram.com | ||
| https://linkedin.com | ||
| https://wikipedia.org | ||
| https://amazon.com | ||
| https://apple.com | ||
| https://whatsapp.com | ||
| https://tiktok.com | ||
| https://yahoo.com | ||
| https://microsoft.com | ||
| https://reddit.com | ||
| https://netflix.com | ||
| https://zoom.us | ||
| https://ebay.com | ||
| https://baidu.com | ||
| https://vk.com | ||
| https://aliexpress.com | ||
| https://bing.com | ||
| https://nytimes.com | ||
| https://cnn.com | ||
| https://bbc.com | ||
| https://spotify.com | ||
| https://adobe.com | ||
| https://wordpress.com | ||
| https://weather.com | ||
| https://tripadvisor.com | ||
| https://pinterest.com | ||
| https://github.com | ||
| https://salesforce.com | ||
| https://medium.com | ||
| https://etsy.com | ||
| https://shopify.com | ||
| https://canva.com | ||
| https://fandom.com | ||
| https://patreon.com | ||
| https://playstation.com | ||
| https://yelp.com | ||
| https://snapchat.com | ||
| https://telegram.org | ||
| https://dropbox.com | ||
| https://stackoverflow.com | ||
| https://cloudflare.com | ||
| https://weibo.com | ||
| https://duckduckgo.com |
1 change: 1 addition & 0 deletions
1
06-coding-with-streams/21-ordered-limited-parallel-execution/.gitignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| results.txt |
28 changes: 28 additions & 0 deletions
28
06-coding-with-streams/21-ordered-limited-parallel-execution/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # 21-ordered-parallel-execution | ||
|
|
||
| This example shows how to create a limited parallel execution flow using | ||
| streams. | ||
|
|
||
| ## Dependencies | ||
|
|
||
| Install all necessary dependencies with: | ||
|
|
||
| ```bash | ||
| pnpm install | ||
| ``` | ||
|
|
||
| ## Run | ||
|
|
||
| To run the example: | ||
|
|
||
| ```bash | ||
| node check-urls.js urls.txt | ||
| ``` | ||
|
|
||
| For a long list of urls you can alternatively use the `500urls.txt` file! | ||
|
|
||
| Then check the content of the results file with: | ||
|
|
||
| ```bash | ||
| cat results.txt | ||
| ``` |
26 changes: 26 additions & 0 deletions
26
06-coding-with-streams/21-ordered-limited-parallel-execution/check-urls.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| import { createReadStream, createWriteStream } from 'node:fs' | ||
| import { createInterface } from 'node:readline' | ||
| import { pipeline } from 'node:stream/promises' | ||
| import parallelTransform from 'parallel-transform' // v1.2.0 | ||
|
|
||
| const inputFile = createReadStream(process.argv[2]) | ||
| const fileLines = createInterface({ | ||
| input: inputFile, | ||
| }) | ||
| const checkUrls = parallelTransform(8, async function (url, done) { | ||
| if (!url) { | ||
| return done() | ||
| } | ||
| try { | ||
| await fetch(url, { method: 'HEAD', timeout: 5 * 1000 }) | ||
| this.push(`${url} is up\n`) | ||
| } catch (err) { | ||
| this.push(`${url} is down: ${err}\n`) | ||
| } | ||
| done() | ||
| }) | ||
| const outputFile = createWriteStream('results.txt') | ||
|
|
||
| await pipeline(fileLines, checkUrls, outputFile) | ||
|
|
||
| console.log('All urls have been checked') |
17 changes: 17 additions & 0 deletions
17
06-coding-with-streams/21-ordered-limited-parallel-execution/package.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| { | ||
| "name": "21-ordered-parallel-execution", | ||
| "version": "1.0.0", | ||
| "description": "This example shows how to create a limited parallel execution flow using streams.", | ||
| "type": "module", | ||
| "scripts": {}, | ||
| "engines": { | ||
| "node": ">=22" | ||
| }, | ||
| "engineStrict": true, | ||
| "keywords": [], | ||
| "author": "Luciano Mammino and Mario Casciaro", | ||
| "license": "MIT", | ||
| "dependencies": { | ||
| "parallel-transform": "^1.2.0" | ||
| } | ||
| } |
81 changes: 81 additions & 0 deletions
81
06-coding-with-streams/21-ordered-limited-parallel-execution/pnpm-lock.yaml
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
50 changes: 50 additions & 0 deletions
50
06-coding-with-streams/21-ordered-limited-parallel-execution/urls.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| https://mario.fyi | ||
| https://loige.co | ||
| http://thiswillbedownforsure.com | ||
| https://google.com | ||
| https://youtube.com | ||
| https://facebook.com | ||
| https://twitter.com | ||
| https://instagram.com | ||
| https://linkedin.com | ||
| https://wikipedia.org | ||
| https://amazon.com | ||
| https://apple.com | ||
| https://whatsapp.com | ||
| https://tiktok.com | ||
| https://yahoo.com | ||
| https://microsoft.com | ||
| https://reddit.com | ||
| https://netflix.com | ||
| https://zoom.us | ||
| https://ebay.com | ||
| https://baidu.com | ||
| https://vk.com | ||
| https://aliexpress.com | ||
| https://bing.com | ||
| https://nytimes.com | ||
| https://cnn.com | ||
| https://bbc.com | ||
| https://spotify.com | ||
| https://adobe.com | ||
| https://wordpress.com | ||
| https://weather.com | ||
| https://tripadvisor.com | ||
| https://pinterest.com | ||
| https://github.com | ||
| https://salesforce.com | ||
| https://medium.com | ||
| https://etsy.com | ||
| https://shopify.com | ||
| https://canva.com | ||
| https://fandom.com | ||
| https://patreon.com | ||
| https://playstation.com | ||
| https://yelp.com | ||
| https://snapchat.com | ||
| https://telegram.org | ||
| https://dropbox.com | ||
| https://stackoverflow.com | ||
| https://cloudflare.com | ||
| https://weibo.com | ||
| https://duckduckgo.com |