docs: test code snipets, generate output for surface_flows, fix hyd…

…ro-project#814

docs/docs/hydroflow/syntax/surface_flows.md → docs/docs/hydroflow/syntax/surface_flows.mdx

@@ -1,22 +1,25 @@
 ---
 sidebar_position: 2
 ---
+import CodeBlock from '@theme/CodeBlock';
+import example1Code from '!!raw-loader!../../../../hydroflow/examples/example_surface_flows_1_basic.rs';
+import example2Code from '!!raw-loader!../../../../hydroflow/examples/example_surface_flows_2_varname.rs';
+import example3Code from '!!raw-loader!../../../../hydroflow/examples/example_surface_flows_3_ports.rs';
+import example3Out from '!!raw-loader!../../../../hydroflow/tests/snapshots/surface_examples__example_surface_flows_3_ports.snap';
+import example4Code from '!!raw-loader!../../../../hydroflow/examples/example_surface_flows_4_context.rs';
+import example4Out from '!!raw-loader!../../../../hydroflow/tests/snapshots/surface_examples__example_surface_flows_4_context.snap';
+import { getLines, extractMermaid, extractOutput } from '../../../src/util';
 
 # Flow Syntax
 Flows consist of named _operators_ that are connected via flow _edges_ denoted by `->`. The example below
 uses the [`source_iter`](./surface_ops_gen.md#source_iter) operator to generate two strings from a Rust `vec`, the
 [`map`](./surface_ops_gen.md#map) operator to apply some Rust code to uppercase each string, and the [`for_each`](./surface_ops_gen.md#for_each)
 operator to print each string to stdout.
-```rust,ignore
-source_iter(vec!["Hello", "world"])
-    -> map(|x| x.to_uppercase()) -> for_each(|x| println!("{}", x));
-```
+<CodeBlock language="rust">{getLines(example1Code, 5, 6)}</CodeBlock>
 
 Flows can be assigned to variable names for convenience. E.g, the above can be rewritten as follows:
-```rust,ignore
-source_iter(vec!["Hello", "world"]) -> upper_print;
-upper_print = map(|x| x.to_uppercase()) -> for_each(|x| println!("{}", x));
-```
+<CodeBlock language="rust">{getLines(example2Code, 5, 6)}</CodeBlock>
+
 Note that the order of the statements (lines) doesn't matter. In this example, `upper_print` is
 referenced before it is assigned, and that is completely OK and better matches the flow of
 data, making the program more understandable.
@@ -35,49 +38,12 @@ allow you to choose arbitrary strings, which help you make your code and dataflo
 (e.g. `my_tee[print]` and `my_tee[continue]`).
 
 Here is an example that tees one flow into two, handles each separately, and then unions them to print out the contents in both lowercase and uppercase:
-```rust,ignore
-my_tee = source_iter(vec!["Hello", "world"]) -> tee();
-my_tee -> map(|x| x.to_uppercase()) -> [low_road]my_union;
-my_tee -> map(|x| x.to_lowercase()) -> [high_road]my_union;
-my_union = union() -> for_each(|x| println!("{}", x));
-```
+<CodeBlock language="rust">{getLines(example3Code, 5, 8)}</CodeBlock>
+
 Here is a visualization of the flow that was generated. Note that the outbound labels to `my_tee` 
 were auto-generated, but the inbound labels to `my_union` were specified by the code above:
-```mermaid
-%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%%
-flowchart TD
-classDef pullClass fill:#02f,color:#fff,stroke:#000
-classDef pushClass fill:#ff0,stroke:#000
-linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em;
-subgraph sg_1v1 ["sg_1v1 stratum 0"]
-    1v1[\"(1v1) <tt>source_iter(vec! [&quot;Hello&quot;, &quot;world&quot;])</tt>"/]:::pullClass
-    2v1[/"(2v1) <tt>tee()</tt>"\]:::pushClass
-    1v1--->2v1
-    subgraph sg_1v1_var_my_tee ["var <tt>my_tee</tt>"]
-        1v1
-        2v1
-    end
-end
-subgraph sg_2v1 ["sg_2v1 stratum 0"]
-    3v1[\"(3v1) <tt>map(| x : &amp; str | x.to_uppercase())</tt>"/]:::pullClass
-    4v1[\"(4v1) <tt>map(| x : &amp; str | x.to_lowercase())</tt>"/]:::pullClass
-    5v1[\"(5v1) <tt>union()</tt>"/]:::pullClass
-    6v1[/"(6v1) <tt>for_each(| x | println! (&quot;{}&quot;, x))</tt>"\]:::pushClass
-    3v1--low road--->5v1
-    4v1--high road--->5v1
-    5v1--->6v1
-    subgraph sg_2v1_var_my_union ["var <tt>my_union</tt>"]
-        5v1
-        6v1
-    end
-end
-2v1--0--->7v1
-2v1--1--->8v1
-7v1["(7v1) <tt>handoff</tt>"]:::otherClass
-7v1--->3v1
-8v1["(8v1) <tt>handoff</tt>"]:::otherClass
-8v1--->4v1
-```
+<mermaid value={extractMermaid(example3Out)}></mermaid>
+
 Hydroflow compiled this flow into two subgraphs called _compiled components_, connected by _handoffs_. You can ignore
 these details unless you are interested in low-level performance tuning; they are explained in the discussion
 of [in-out trees](../architecture/in-out_trees.md).
@@ -87,9 +53,7 @@ Closures inside surface syntax operators have access to a special `context` obje
 access to scheduling, timing, and state APIs. The object is accessible as a shared reference
 (`&Context`) via the special name `context`.
 [Here is the full API documentation for `Context`](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/context/struct.Context.html).
+<CodeBlock language="rust">{getLines(example4Code, 5, 6)}</CodeBlock>
 
-```rust,ignore
-source_iter([()])
-    -> for_each(|()| println!("Current tick: {}, stratum: {}", context.current_tick(), context.current_stratum()));
-// Current tick: 0, stratum: 0
-```
+Output:
+<CodeBlock language="console">{extractOutput(example4Out, true)}</CodeBlock>

docs/src/util.ts

@@ -1,15 +1,23 @@
 export function getLines(str: string, lineStart: number, lineEnd?: number): string {
-    return str.split('\n').slice(lineStart - 1, lineEnd || lineStart).join('\n');
+    let lines = str.split('\n').slice(lineStart - 1, lineEnd || lineStart);
+    const leadingWhitespace = Math.min(...lines.map(line => line.search(/\S/)).map(Number).filter(n => 0 !== n));
+    if (0 < leadingWhitespace) {
+        lines = lines.map(line => line.slice(leadingWhitespace));
+    }
+    return lines.join('\n');
 }
 
-export function extractOutput(output: string): string {
+export function extractOutput(output: string, short = false): string {
     const outputLines = output.replace(/\n$/, '').split('\n');
     outputLines.splice(0, 4);
     if (outputLines[0].startsWith('%%')) {
         const count = outputLines.findIndex(line => 0 === line.length);
         outputLines.splice(0, count + 1, '<graph output>');
     }
     const stdOut = outputLines.join('\n');
+    if (short) {
+        return stdOut;
+    }
     return `#shell-command-next-line
 cargo run
 <build output>

hydroflow/examples/example_surface_flows_1_basic.rs

@@ -0,0 +1,9 @@
+use hydroflow::hydroflow_syntax;
+
+pub fn main() {
+    let mut flow = hydroflow_syntax! {
+        source_iter(vec!["Hello", "world"])
+            -> map(|x| x.to_uppercase()) -> for_each(|x| println!("{}", x));
+    };
+    flow.run_available();
+}

hydroflow/examples/example_surface_flows_2_varname.rs

@@ -0,0 +1,9 @@
+use hydroflow::hydroflow_syntax;
+
+pub fn main() {
+    let mut flow = hydroflow_syntax! {
+        source_iter(vec!["Hello", "world"]) -> upper_print;
+        upper_print = map(|x| x.to_uppercase()) -> for_each(|x| println!("{}", x));
+    };
+    flow.run_available();
+}

hydroflow/examples/example_surface_flows_3_ports.rs

@@ -0,0 +1,17 @@
+use hydroflow::hydroflow_syntax;
+
+pub fn main() {
+    let mut flow = hydroflow_syntax! {
+        my_tee = source_iter(vec!["Hello", "world"]) -> tee();
+        my_tee -> map(|x| x.to_uppercase()) -> [low_road]my_union;
+        my_tee -> map(|x| x.to_lowercase()) -> [high_road]my_union;
+        my_union = union() -> for_each(|x| println!("{}", x));
+    };
+    println!(
+        "{}",
+        flow.meta_graph()
+            .expect("No graph found, maybe failed to parse.")
+            .to_mermaid(&Default::default())
+    );
+    flow.run_available();
+}

hydroflow/examples/example_surface_flows_4_context.rs

@@ -0,0 +1,9 @@
+use hydroflow::hydroflow_syntax;
+
+pub fn main() {
+    let mut flow = hydroflow_syntax! {
+        source_iter([()])
+            -> for_each(|()| println!("Current tick: {}, stratum: {}", context.current_tick(), context.current_stratum()));
+    };
+    flow.run_available();
+}

hydroflow/tests/snapshots/surface_examples__example_surface_flows_1_basic.snap

@@ -0,0 +1,7 @@
+---
+source: hydroflow/tests/surface_examples.rs
+expression: output
+---
+HELLO
+WORLD
+

hydroflow/tests/snapshots/surface_examples__example_surface_flows_2_varname.snap

@@ -0,0 +1,7 @@
+---
+source: hydroflow/tests/surface_examples.rs
+expression: output
+---
+HELLO
+WORLD
+

hydroflow/tests/snapshots/surface_examples__example_surface_flows_3_ports.snap

@@ -0,0 +1,50 @@
+---
+source: hydroflow/tests/surface_examples.rs
+expression: output
+---
+%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%%
+flowchart TD
+classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre
+classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre
+classDef otherClass fill:#fdc,stroke:#000,text-align:left,white-space:pre
+linkStyle default stroke:#aaa
+1v1[\"(1v1) <code>source_iter(vec![&quot;Hello&quot;, &quot;world&quot;])</code>"/]:::pullClass
+2v1[/"(2v1) <code>tee()</code>"\]:::pushClass
+3v1[\"(3v1) <code>map(|x| x.to_uppercase())</code>"/]:::pullClass
+4v1[\"(4v1) <code>map(|x| x.to_lowercase())</code>"/]:::pullClass
+5v1[\"(5v1) <code>union()</code>"/]:::pullClass
+6v1[/"(6v1) <code>for_each(|x| println!(&quot;{}&quot;, x))</code>"\]:::pushClass
+7v1["(7v1) <code>handoff</code>"]:::otherClass
+8v1["(8v1) <code>handoff</code>"]:::otherClass
+1v1-->2v1
+3v1-->|low_road|5v1
+2v1-->7v1
+4v1-->|high_road|5v1
+2v1-->8v1
+5v1-->6v1
+7v1-->3v1
+8v1-->4v1
+subgraph sg_1v1 ["sg_1v1 stratum 0"]
+    1v1
+    2v1
+    subgraph sg_1v1_var_my_tee ["var <tt>my_tee</tt>"]
+        1v1
+        2v1
+    end
+end
+subgraph sg_2v1 ["sg_2v1 stratum 0"]
+    3v1
+    4v1
+    5v1
+    6v1
+    subgraph sg_2v1_var_my_union ["var <tt>my_union</tt>"]
+        5v1
+        6v1
+    end
+end
+
+hello
+world
+HELLO
+WORLD
+

hydroflow/tests/snapshots/surface_examples__example_surface_flows_4_context.snap

@@ -0,0 +1,6 @@
+---
+source: hydroflow/tests/surface_examples.rs
+expression: output
+---
+Current tick: [0], stratum: 0
+

hydroflow_lang/src/graph/ops/demux.rs

@@ -25,7 +25,7 @@ use crate::pretty_span::PrettySpan;
 /// > Note: The [`Pusherator`](https://hydro-project.github.io/hydroflow/doc/pusherator/trait.Pusherator.html)
 /// > trait is automatically imported to enable the [`.give(...)` method](https://hydro-project.github.io/hydroflow/doc/pusherator/trait.Pusherator.html#tymethod.give).
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// my_demux = source_iter(1..=100) -> demux(|v, var_args!(fzbz, fizz, buzz, rest)|

hydroflow_lang/src/graph/ops/filter.rs

@@ -12,7 +12,7 @@ use super::{
 /// not modify or take ownership of the values. If you need to modify the values while filtering
 /// use [`filter_map`](#filter_map) instead.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// source_iter(vec!["hello", "world"]) -> filter(|x| x.starts_with('w'))

hydroflow_lang/src/graph/ops/filter_map.rs

@@ -9,7 +9,7 @@ use super::{
 ///
 /// An operator that both filters and maps. It yields only the items for which the supplied closure returns `Some(value)`.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// source_iter(vec!["1", "hello", "world", "2"])

hydroflow_lang/src/graph/ops/flat_map.rs

@@ -12,7 +12,7 @@ use super::{
 /// For each item `i` passed in, treat `i` as an iterator and map the closure to that
 /// iterator to produce items one by one. The type of the input items must be iterable.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// // should print out each character of each word on a separate line

hydroflow_lang/src/graph/ops/fold.rs

@@ -17,7 +17,7 @@ use crate::diagnostic::{Diagnostic, Level};
 /// operator, except that it takes the accumulator by `&mut` instead of by value. Folds every item
 /// into an accumulator by applying a closure, returning the final result.
 ///
-/// > Note: The closures have access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closures have access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// `fold` can also be provided with one generic lifetime persistence argument, either
 /// `'tick` or `'static`, to specify how data persists. With `'tick`, Items will only be collected

hydroflow_lang/src/graph/ops/fold_keyed.rs

@@ -19,7 +19,7 @@ use super::{
 /// is partitioned into groups by the first field ("keys"), and for each group the values in the second
 /// field are accumulated via the closures in the arguments.
 ///
-/// > Note: The closures have access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closures have access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// source_iter([("toy", 1), ("toy", 2), ("shoe", 11), ("shoe", 35), ("haberdashery", 7)])

hydroflow_lang/src/graph/ops/for_each.rs

@@ -12,7 +12,7 @@ use super::{
 /// Iterates through a stream passing each element to the closure in the
 /// argument.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 ///     source_iter(vec!["Hello", "World"])

hydroflow_lang/src/graph/ops/inspect.rs

@@ -12,7 +12,7 @@ use super::{
 /// mainly useful for debugging as in the example below, and it is generally an
 /// anti-pattern to provide a closure with side effects.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// source_iter([1, 2, 3, 4])

hydroflow_lang/src/graph/ops/map.rs

@@ -13,7 +13,7 @@ use super::{
 /// If you do not want to modify the item stream and instead only want to view
 /// each item use the [`inspect`](#inspect) operator instead.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// source_iter(vec!["hello", "world"]) -> map(|x| x.to_uppercase())

hydroflow_lang/src/graph/ops/partition.rs

@@ -19,7 +19,7 @@ use crate::pretty_span::PrettySpan;
 /// > Arguments: A Rust closure, the first argument is a reference to the item and the second
 /// argument corresponds to one of two modes, either named or indexed.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// # Named mode
 /// With named ports, the closure's second argument must be a Rust 'slice pattern' of names, such as

hydroflow_lang/src/graph/ops/reduce.rs

@@ -19,7 +19,7 @@ use crate::diagnostic::{Diagnostic, Level};
 /// operator, except that it takes the accumulator by `&mut` instead of by value. Reduces every
 /// item into an accumulator by applying a closure, returning the final result.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// `reduce` can also be provided with one generic lifetime persistence argument, either
 /// `'tick` or `'static`, to specify how data persists. With `'tick`, values will only be collected

hydroflow_lang/src/graph/ops/reduce_keyed.rs

@@ -19,7 +19,7 @@ use crate::diagnostic::{Diagnostic, Level};
 /// is partitioned into groups by the first field, and for each group the values in the second
 /// field are accumulated via the closures in the arguments.
 ///
-/// > Note: The closures have access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closures have access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// `reduce_keyed` can also be provided with one generic lifetime persistence argument, either
 /// `'tick` or `'static`, to specify how data persists. With `'tick`, values will only be collected

hydroflow_lang/src/graph/ops/sort_by_key.rs

@@ -8,7 +8,7 @@ use super::{
 /// Like sort, takes a stream as input and produces a version of the stream as output.
 /// This operator sorts according to the key extracted by the closure.
 ///
-/// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object).
+/// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object).
 ///
 /// ```hydroflow
 /// source_iter(vec![(2, 'y'), (3, 'x'), (1, 'z')])