diff --git a/Project.toml b/Project.toml index ce1ee1b..470b2dd 100644 --- a/Project.toml +++ b/Project.toml @@ -6,12 +6,9 @@ version = "0.1.0" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -Cairo = "159f3aea-2a34-519c-b102-8c37f9878175" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" Dagger = "d58978e5-989f-55fb-8d15-ea34adc7bf54" -DaggerWebDash = "cfc5aa84-1a2a-41ab-b391-ede92ecae40c" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" EzXML = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" @@ -21,18 +18,14 @@ LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d" MetaGraphs = "626554b9-1ddb-594c-aa3c-2596fe9399a5" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" -TimespanLogging = "a526e669-04d3-4846-9525-c66122c55f63" [compat] Aqua = "0.8" ArgParse = "1.2" BenchmarkTools = "1.5" -Cairo = "1.0" Colors = "0.12" Dagger = "0.18.11" -DaggerWebDash = "0.1.3" DataFrames = "1.6" -Dates = "1.10" Distributed = "1.10" EzXML = "1.2" FileIO = "1.16" @@ -44,7 +37,6 @@ MetaGraphs = "0.7" Plots = "1.40" Printf = "1.10" Test = "1.10" -TimespanLogging = "0.1.0" julia = "1.10" [extras] diff --git a/bin/schedule.jl b/bin/schedule.jl index 96401d1..50f75a1 100644 --- a/bin/schedule.jl +++ b/bin/schedule.jl @@ -39,9 +39,11 @@ end function main() args = parse_args() - if !isnothing(args["dot-trace"]) - @info "Enabled logging" + logging_required = !isnothing(args["dot-trace"]) + + if logging_required FrameworkDemo.configure_LocalEventLog() + @info "Enabled logging" end graph = FrameworkDemo.parse_graphml(args["data-flow"]) @@ -53,12 +55,10 @@ function main() event_count = event_count, max_concurrent = max_concurrent, fast = fast) - - if !isnothing(args["dot-trace"]) - logs = Dagger.fetch_logs!() - open(args["dot-trace"], "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, logs, :graphviz_simple) - @info "Written logs dot graph to $(args["dot-trace"])" + if logging_required + logs = FrameworkDemo.fetch_logs!() + if !isnothing(args["dot-trace"]) + FrameworkDemo.save_logs_dot(logs, args["dot-trace"]) end end end diff --git a/docs/add_show_logs_impl.png b/docs/add_show_logs_impl.png deleted file mode 100644 index e56df1e..0000000 Binary files a/docs/add_show_logs_impl.png and /dev/null differ diff --git a/docs/example_graphs/DAG.dot b/docs/example_graphs/DAG.dot deleted file mode 100644 index 2217541..0000000 --- a/docs/example_graphs/DAG.dot +++ /dev/null @@ -1,50 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_17 [label="print_func -1.034 s",color="#000000",shape="ellipse",penwidth=5]; -n_18 [label="print_func -1.015 s",color="#000000",shape="ellipse",penwidth=5]; -n_19 [label="print_func -1.015 s",color="#000000",shape="ellipse",penwidth=5]; -n_20 [label="mock_func -1.029 s",color="#000000",shape="ellipse",penwidth=5]; -n_11 [label="print_func -1.034 s",color="#FFFF74",shape="ellipse",penwidth=5]; -n_5 [label="print_func -1.016 s",color="#FFFF74",shape="ellipse",penwidth=5]; -n_12 [label="print_func -1.016 s",color="#FFFF74",shape="ellipse",penwidth=5]; -n_6 [label="print_func -1.004 s",color="#FFFF74",shape="ellipse",penwidth=5]; -n_13 [label="print_func -1.031 s",color="#FFFF74",shape="ellipse",penwidth=5]; -n_7 [label="print_func -1.003 s",color="#FFFF74",shape="ellipse",penwidth=5]; -n_14 [label="print_func -1.034 s",color="#FF9BFF",shape="ellipse",penwidth=5]; -n_8 [label="print_func -1.016 s",color="#FF9BFF",shape="ellipse",penwidth=5]; -n_15 [label="print_func -1.017 s",color="#FF9BFF",shape="ellipse",penwidth=5]; -n_9 [label="print_func -1.004 s",color="#FF9BFF",shape="ellipse",penwidth=5]; -n_16 [label="print_func -1.031 s",color="#FF9BFF",shape="ellipse",penwidth=5]; -n_10 [label="print_func -1.003 s",color="#FF9BFF",shape="ellipse",penwidth=5]; -n_17 -> n_18 [label="Move: 59.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_17 -> n_19 [label="Move: 18.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_18 -> n_19 [label="Move: 5.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_11 -> n_12 [label="Move: 71.3 us",color="#FFFF74;0.5:#FFFF74",penwidth=2]; -n_5 -> n_6 [label="Move: 32.4 us",color="#FFFF74;0.5:#FFFF74",penwidth=2]; -n_11 -> n_13 [label="Move: 25.8 us",color="#FFFF74;0.5:#FFFF74",penwidth=2]; -n_12 -> n_13 [label="Move: 8.2 us",color="#FFFF74;0.5:#FFFF74",penwidth=2]; -n_5 -> n_7 [label="Move: 25.1 us",color="#FFFF74;0.5:#FFFF74",penwidth=2]; -n_6 -> n_7 [label="Move: 10.1 us",color="#FFFF74;0.5:#FFFF74",penwidth=2]; -n_14 -> n_15 [label="Move: 48.5 us",color="#FF9BFF;0.5:#FF9BFF",penwidth=2]; -n_8 -> n_9 [label="Move: 28.8 us",color="#FF9BFF;0.5:#FF9BFF",penwidth=2]; -n_14 -> n_16 [label="Move: 22.1 us",color="#FF9BFF;0.5:#FF9BFF",penwidth=2]; -n_15 -> n_16 [label="Move: 6.0 us",color="#FF9BFF;0.5:#FF9BFF",penwidth=2]; -n_8 -> n_10 [label="Move: 25.1 us",color="#FF9BFF;0.5:#FF9BFF",penwidth=2]; -n_9 -> n_10 [label="Move: 9.2 us",color="#FF9BFF;0.5:#FF9BFF",penwidth=2]; -} diff --git a/docs/example_graphs/DAG.png b/docs/example_graphs/DAG.png deleted file mode 100644 index ad3f240..0000000 Binary files a/docs/example_graphs/DAG.png and /dev/null differ diff --git a/docs/example_graphs/DAG_complex_example.png b/docs/example_graphs/DAG_complex_example.png deleted file mode 100644 index ee8b16b..0000000 Binary files a/docs/example_graphs/DAG_complex_example.png and /dev/null differ diff --git a/docs/example_graphs/DAG_example.dot b/docs/example_graphs/DAG_example.dot deleted file mode 100644 index 8086612..0000000 --- a/docs/example_graphs/DAG_example.dot +++ /dev/null @@ -1,25 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_29 [label="print_func -1.011 s",color="#000000",shape="ellipse",penwidth=5]; -n_30 [label="print_func -1.018 s",color="#000000",shape="ellipse",penwidth=5]; -n_31 [label="print_func -1.013 s",color="#000000",shape="ellipse",penwidth=5]; -n_32 [label="print_func -1.003 s",color="#000000",shape="ellipse",penwidth=5]; -n_33 [label="print_func -1.016 s",color="#000000",shape="ellipse",penwidth=5]; -n_34 [label="mock_func -1.03 s",color="#000000",shape="ellipse",penwidth=5]; -n_29 -> n_30 [label="Move: 12.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_29 -> n_31 [label="Move: 16.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_30 -> n_31 [label="Move: 3.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_29 -> n_32 [label="Move: 16.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_30 -> n_32 [label="Move: 4.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_31 -> n_32 [label="Move: 4.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_29 -> n_33 [label="Move: 16.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_30 -> n_33 [label="Move: 4.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_31 -> n_33 [label="Move: 3.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_32 -> n_33 [label="Move: 3.9 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/docs/example_graphs/DAG_example.png b/docs/example_graphs/DAG_example.png deleted file mode 100644 index 6d1c2c2..0000000 Binary files a/docs/example_graphs/DAG_example.png and /dev/null differ diff --git a/docs/example_graphs/DAG_example_advanced_12_procs.png b/docs/example_graphs/DAG_example_advanced_12_procs.png deleted file mode 100644 index a31c23a..0000000 Binary files a/docs/example_graphs/DAG_example_advanced_12_procs.png and /dev/null differ diff --git a/docs/example_graphs/DAG_example_advanced_single_proc.dot b/docs/example_graphs/DAG_example_advanced_single_proc.dot deleted file mode 100644 index 3e75372..0000000 --- a/docs/example_graphs/DAG_example_advanced_single_proc.dot +++ /dev/null @@ -1,181 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_49 [label="mock_Gaudi_algorithm -1.027 s",color="#000000",shape="ellipse",penwidth=5]; -n_5 [label="mock_Gaudi_algorithm -1.003 s",color="#000000",shape="ellipse",penwidth=5]; -n_19 [label="mock_Gaudi_algorithm -1.008 s",color="#000000",shape="ellipse",penwidth=5]; -n_50 [label="dataobject_algorithm -151.194 ms",color="#000000",shape="ellipse",penwidth=5]; -n_6 [label="dataobject_algorithm -106.03 ms",color="#000000",shape="ellipse",penwidth=5]; -n_20 [label="dataobject_algorithm -104.386 ms",color="#000000",shape="ellipse",penwidth=5]; -n_51 [label="mock_Gaudi_algorithm -1.054 s",color="#000000",shape="ellipse",penwidth=5]; -n_7 [label="mock_Gaudi_algorithm -1.015 s",color="#000000",shape="ellipse",penwidth=5]; -n_21 [label="mock_Gaudi_algorithm -1.011 s",color="#000000",shape="ellipse",penwidth=5]; -n_52 [label="dataobject_algorithm -108.05 ms",color="#000000",shape="ellipse",penwidth=5]; -n_8 [label="dataobject_algorithm -109.371 ms",color="#000000",shape="ellipse",penwidth=5]; -n_22 [label="dataobject_algorithm -107.926 ms",color="#000000",shape="ellipse",penwidth=5]; -n_53 [label="mock_Gaudi_algorithm -1.011 s",color="#000000",shape="ellipse",penwidth=5]; -n_17 [label="mock_Gaudi_algorithm -1.012 s",color="#000000",shape="ellipse",penwidth=5]; -n_23 [label="mock_Gaudi_algorithm -1.015 s",color="#000000",shape="ellipse",penwidth=5]; -n_54 [label="dataobject_algorithm -106.642 ms",color="#000000",shape="ellipse",penwidth=5]; -n_18 [label="dataobject_algorithm -109.126 ms",color="#000000",shape="ellipse",penwidth=5]; -n_24 [label="dataobject_algorithm -107.827 ms",color="#000000",shape="ellipse",penwidth=5]; -n_55 [label="mock_Gaudi_algorithm -1.015 s",color="#000000",shape="ellipse",penwidth=5]; -n_31 [label="mock_Gaudi_algorithm -1.014 s",color="#000000",shape="ellipse",penwidth=5]; -n_25 [label="mock_Gaudi_algorithm -1.013 s",color="#000000",shape="ellipse",penwidth=5]; -n_56 [label="dataobject_algorithm -107.6 ms",color="#000000",shape="ellipse",penwidth=5]; -n_32 [label="dataobject_algorithm -108.326 ms",color="#000000",shape="ellipse",penwidth=5]; -n_26 [label="dataobject_algorithm -110.496 ms",color="#000000",shape="ellipse",penwidth=5]; -n_57 [label="mock_Gaudi_algorithm -1.003 s",color="#000000",shape="ellipse",penwidth=5]; -n_59 [label="mock_Gaudi_algorithm -1.01 s",color="#000000",shape="ellipse",penwidth=5]; -n_27 [label="mock_Gaudi_algorithm -1.017 s",color="#000000",shape="ellipse",penwidth=5]; -n_58 [label="dataobject_algorithm -107.782 ms",color="#000000",shape="ellipse",penwidth=5]; -n_60 [label="dataobject_algorithm -109.446 ms",color="#000000",shape="ellipse",penwidth=5]; -n_28 [label="dataobject_algorithm -110.961 ms",color="#000000",shape="ellipse",penwidth=5]; -n_61 [label="mock_Gaudi_algorithm -1.014 s",color="#000000",shape="ellipse",penwidth=5]; -n_63 [label="mock_Gaudi_algorithm -1.009 s",color="#000000",shape="ellipse",penwidth=5]; -n_29 [label="mock_Gaudi_algorithm -1.009 s",color="#000000",shape="ellipse",penwidth=5]; -n_62 [label="dataobject_algorithm -108.793 ms",color="#000000",shape="ellipse",penwidth=5]; -n_64 [label="dataobject_algorithm -109.249 ms",color="#000000",shape="ellipse",penwidth=5]; -n_30 [label="dataobject_algorithm -109.295 ms",color="#000000",shape="ellipse",penwidth=5]; -n_9 [label="mock_Gaudi_algorithm -1.01 s",color="#000000",shape="ellipse",penwidth=5]; -n_33 [label="mock_Gaudi_algorithm -1.014 s",color="#000000",shape="ellipse",penwidth=5]; -n_10 [label="dataobject_algorithm -108.617 ms",color="#000000",shape="ellipse",penwidth=5]; -n_34 [label="dataobject_algorithm -108.44 ms",color="#000000",shape="ellipse",penwidth=5]; -n_11 [label="mock_Gaudi_algorithm -1.018 s",color="#000000",shape="ellipse",penwidth=5]; -n_35 [label="mock_Gaudi_algorithm -1.007 s",color="#000000",shape="ellipse",penwidth=5]; -n_12 [label="dataobject_algorithm -107.967 ms",color="#000000",shape="ellipse",penwidth=5]; -n_36 [label="dataobject_algorithm -107.845 ms",color="#000000",shape="ellipse",penwidth=5]; -n_13 [label="mock_Gaudi_algorithm -1.015 s",color="#000000",shape="ellipse",penwidth=5]; -n_37 [label="mock_Gaudi_algorithm -1.009 s",color="#000000",shape="ellipse",penwidth=5]; -n_14 [label="dataobject_algorithm -107.834 ms",color="#000000",shape="ellipse",penwidth=5]; -n_38 [label="dataobject_algorithm -107.375 ms",color="#000000",shape="ellipse",penwidth=5]; -n_15 [label="mock_Gaudi_algorithm -1.015 s",color="#000000",shape="ellipse",penwidth=5]; -n_39 [label="mock_Gaudi_algorithm -1.009 s",color="#000000",shape="ellipse",penwidth=5]; -n_16 [label="dataobject_algorithm -108.024 ms",color="#000000",shape="ellipse",penwidth=5]; -n_40 [label="dataobject_algorithm -108.107 ms",color="#000000",shape="ellipse",penwidth=5]; -n_45 [label="mock_Gaudi_algorithm -1.014 s",color="#000000",shape="ellipse",penwidth=5]; -n_47 [label="mock_Gaudi_algorithm -1.014 s",color="#000000",shape="ellipse",penwidth=5]; -n_41 [label="mock_Gaudi_algorithm -1.01 s",color="#000000",shape="ellipse",penwidth=5]; -n_46 [label="dataobject_algorithm -107.875 ms",color="#000000",shape="ellipse",penwidth=5]; -n_48 [label="dataobject_algorithm -109.547 ms",color="#000000",shape="ellipse",penwidth=5]; -n_42 [label="dataobject_algorithm -108.848 ms",color="#000000",shape="ellipse",penwidth=5]; -n_43 [label="mock_Gaudi_algorithm -1.014 s",color="#000000",shape="ellipse",penwidth=5]; -n_44 [label="dataobject_algorithm -108.455 ms",color="#000000",shape="ellipse",penwidth=5]; -n_65 [label="mock_func -1.025 s",color="#000000",shape="ellipse",penwidth=5]; -n_49 -> n_50 [label="Move: 17.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_6 [label="Move: 12.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_19 -> n_20 [label="Move: 16.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_50 -> n_51 [label="Move: 9.797 ms",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_7 [label="Move: 13.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_20 -> n_21 [label="Move: 9.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_51 -> n_52 [label="Move: 18.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_8 [label="Move: 15.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_21 -> n_22 [label="Move: 4.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_52 -> n_53 [label="Move: 19.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_17 [label="Move: 11.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_22 -> n_23 [label="Move: 33.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_53 -> n_54 [label="Move: 16.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_17 -> n_18 [label="Move: 16.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_23 -> n_24 [label="Move: 15.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_54 -> n_55 [label="Move: 16.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_22 -> n_31 [label="Move: 14.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_24 -> n_25 [label="Move: 16.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_55 -> n_56 [label="Move: 9.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_31 -> n_32 [label="Move: 13.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_25 -> n_26 [label="Move: 16.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_56 -> n_57 [label="Move: 17.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_56 -> n_59 [label="Move: 15.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_26 -> n_27 [label="Move: 9.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_57 -> n_58 [label="Move: 15.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_59 -> n_60 [label="Move: 16.4 us",color="#000000;0.5:#000000",penwidth=2]; -n_27 -> n_28 [label="Move: 15.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_56 -> n_61 [label="Move: 27.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_56 -> n_63 [label="Move: 11.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_28 -> n_29 [label="Move: 11.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_61 -> n_62 [label="Move: 10.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_63 -> n_64 [label="Move: 16.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_29 -> n_30 [label="Move: 14.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_9 [label="Move: 13.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_9 -> n_10 [label="Move: 16.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_33 -> n_34 [label="Move: 8.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_10 -> n_11 [label="Move: 18.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_34 -> n_35 [label="Move: 5.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_11 -> n_12 [label="Move: 10.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_35 -> n_36 [label="Move: 9.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_12 -> n_13 [label="Move: 23.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_36 -> n_37 [label="Move: 5.4 us",color="#000000;0.5:#000000",penwidth=2]; -n_13 -> n_14 [label="Move: 10.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_37 -> n_38 [label="Move: 8.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_14 -> n_15 [label="Move: 11.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_38 -> n_39 [label="Move: 9.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_15 -> n_16 [label="Move: 11.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_39 -> n_40 [label="Move: 7.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_40 -> n_45 [label="Move: 6.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_40 -> n_47 [label="Move: 14.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_40 -> n_41 [label="Move: 12.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_45 -> n_46 [label="Move: 5.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_47 -> n_48 [label="Move: 11.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_41 -> n_42 [label="Move: 14.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_40 -> n_43 [label="Move: 6.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_43 -> n_44 [label="Move: 9.3 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/docs/example_graphs/DAG_example_advanced_single_proc.png b/docs/example_graphs/DAG_example_advanced_single_proc.png deleted file mode 100644 index f094a5f..0000000 Binary files a/docs/example_graphs/DAG_example_advanced_single_proc.png and /dev/null differ diff --git a/docs/plot_gantt.png b/docs/plot_gantt.png deleted file mode 100644 index 93475f3..0000000 Binary files a/docs/plot_gantt.png and /dev/null differ diff --git a/docs/plot_gantt_ps.png b/docs/plot_gantt_ps.png deleted file mode 100644 index 8cd75bf..0000000 Binary files a/docs/plot_gantt_ps.png and /dev/null differ diff --git a/docs/show_logs_impl.png b/docs/show_logs_impl.png deleted file mode 100644 index 8800797..0000000 Binary files a/docs/show_logs_impl.png and /dev/null differ diff --git a/examples/dummy_tasks.jl b/examples/dummy_tasks.jl deleted file mode 100644 index 0177526..0000000 --- a/examples/dummy_tasks.jl +++ /dev/null @@ -1,83 +0,0 @@ -import Dagger - -function taskA(time_to_sleep, id, x...) - println("In taskA!") - sleep(time_to_sleep) - for (i, val) in enumerate(x) - println("From taskA, with id: $id, arg $i: " * string(val)) - end - - return "Result of task A with id: $id" -end - -function taskB(time_to_sleep, id, x...) - println("In taskB!") - sleep(time_to_sleep) - for (i, val) in enumerate(x) - println("From taskB, with id: $id, arg $i: " * string(val)) - end - - return "Result of task B with id: $id" -end - -function taskC(time_to_sleep, id, x...) - println("In taskC!") - sleep(time_to_sleep) - for (i, val) in enumerate(x) - println("From taskC, with id: $id, arg $i: " * string(val)) - end - - return "Result of task C with id: $id" -end - -function taskD(time_to_sleep, id, x...) - println("In taskD!") - sleep(time_to_sleep) - for (i, val) in enumerate(x) - println("From taskD, with id: $id, arg $i: " * string(val)) - end - - return "Result of task D with id: $id" -end - -function taskE(time_to_sleep, id, x...) - println("In taskE!") - sleep(time_to_sleep) - for (i, val) in enumerate(x) - println("From taskE, with id: $id, arg $i: " * string(val)) - end - - return "Result of task E with id: $id" -end - -function taskF(time_to_sleep, id, x...) - println("In taskF!") - sleep(time_to_sleep) - for (i, val) in enumerate(x) - println("From taskF, with id: $id, arg $i: " * string(val)) - end - - return "Result of task F with id: $id" -end - -function oldAPI_graph_setup(time_to_sleep) - a = Dagger.delayed(taskA)(time_to_sleep, 1) - b = Dagger.delayed(taskB)(time_to_sleep, 2, a) - c = Dagger.delayed(taskC)(time_to_sleep, 3, a, b) - d = Dagger.delayed(taskD)(time_to_sleep, 4, a, b, c) - e = Dagger.delayed(taskE)(time_to_sleep, 5, a) - f = Dagger.delayed(taskF)(time_to_sleep, 6, a, b, c, d, e) - - return f -end - -function modernAPI_graph_setup(time_to_sleep) - a = Dagger.@spawn (taskA)(time_to_sleep, 1) - b = Dagger.@spawn (taskB)(time_to_sleep, 2, a) - c = Dagger.@spawn (taskC)(time_to_sleep, 3, a, b) - d = Dagger.@spawn (taskD)(time_to_sleep, 4, a, b, c) - e = Dagger.@spawn (taskE)(time_to_sleep, 5, a) - f = Dagger.@spawn (taskF)(time_to_sleep, 6, a, b, c, d, e) - - return f -end diff --git a/examples/examples_results/enable_logging/viz_enable_logging_modernAPI_example_result.dot b/examples/examples_results/enable_logging/viz_enable_logging_modernAPI_example_result.dot deleted file mode 100644 index 181b9fd..0000000 --- a/examples/examples_results/enable_logging/viz_enable_logging_modernAPI_example_result.dot +++ /dev/null @@ -1,51 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_5 [label="taskA -1.051 s",color="#000000",shape="ellipse",penwidth=5]; -n_9 [label="taskE -1.02 s",color="#000000",shape="ellipse",penwidth=5]; -n_6 [label="taskB -1.018 s",color="#000000",shape="ellipse",penwidth=5]; -n_7 [label="taskC -1.03 s",color="#000000",shape="ellipse",penwidth=5]; -n_8 [label="taskD -1.042 s",color="#000000",shape="ellipse",penwidth=5]; -n_10 [label="taskF -1.035 s",color="#000000",shape="ellipse",penwidth=5]; -n_arg_1_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_5 -> n_5 [label="Move: 6.992 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_5 -> n_5 [label="Move: 37.0 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_9 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_9 -> n_9 [label="Move: 6.989 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_9 [label="5",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_9 -> n_9 [label="Move: 30.2 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_6 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_6 -> n_6 [label="Move: 9.011 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_6 [label="2",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_6 -> n_6 [label="Move: 39.7 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_7 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_7 -> n_7 [label="Move: 8.619 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_7 [label="3",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_7 -> n_7 [label="Move: 31.6 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_8 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_8 -> n_8 [label="Move: 7.278 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_8 [label="4",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_8 -> n_8 [label="Move: 37.8 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_10 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_10 -> n_10 [label="Move: 7.985 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_10 [label="6",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_10 -> n_10 [label="Move: 30.0 us",color="black;0.5:#000000",penwidth=2]; -n_5 -> n_9 [label="Move: 44.072 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_6 [label="Move: 7.083 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_7 [label="Move: 6.017 ms",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_7 [label="Move: 94.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_8 [label="Move: 6.215 ms",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_8 [label="Move: 165.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_8 [label="Move: 54.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_10 [label="Move: 5.649 ms",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_10 [label="Move: 110.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_10 [label="Move: 24.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_10 [label="Move: 19.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_9 -> n_10 [label="Move: 19.6 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/examples/examples_results/enable_logging/viz_enable_logging_modernAPI_example_result.png b/examples/examples_results/enable_logging/viz_enable_logging_modernAPI_example_result.png deleted file mode 100644 index d748841..0000000 Binary files a/examples/examples_results/enable_logging/viz_enable_logging_modernAPI_example_result.png and /dev/null differ diff --git a/examples/examples_results/enable_logging/viz_enable_logging_oldAPI_example_result.dot b/examples/examples_results/enable_logging/viz_enable_logging_oldAPI_example_result.dot deleted file mode 100644 index fcc404e..0000000 --- a/examples/examples_results/enable_logging/viz_enable_logging_oldAPI_example_result.dot +++ /dev/null @@ -1,51 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_4 [label="taskA -1.202 s",color="#000000",shape="ellipse",penwidth=5]; -n_8 [label="taskE -1.028 s",color="#000000",shape="ellipse",penwidth=5]; -n_5 [label="taskB -1.016 s",color="#000000",shape="ellipse",penwidth=5]; -n_6 [label="taskC -1.057 s",color="#000000",shape="ellipse",penwidth=5]; -n_7 [label="taskD -1.041 s",color="#000000",shape="ellipse",penwidth=5]; -n_9 [label="taskF -1.04 s",color="#000000",shape="ellipse",penwidth=5]; -n_arg_1_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_5 -> n_5 [label="Move: 8.647 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_5 [label="2",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_5 -> n_5 [label="Move: 22.3 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_4 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_4 -> n_4 [label="Move: 6.854 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_4 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_4 -> n_4 [label="Move: 18.7 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_6 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_6 -> n_6 [label="Move: 7.676 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_6 [label="3",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_6 -> n_6 [label="Move: 19.6 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_7 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_7 -> n_7 [label="Move: 7.666 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_7 [label="4",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_7 -> n_7 [label="Move: 20.7 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_9 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_9 -> n_9 [label="Move: 9.121 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_9 [label="6",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_9 -> n_9 [label="Move: 26.7 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_8 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_8 -> n_8 [label="Move: 6.955 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_8 [label="5",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_8 -> n_8 [label="Move: 20.3 us",color="black;0.5:#000000",penwidth=2]; -n_4 -> n_8 [label="Move: 115.486 ms",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_5 [label="Move: 6.136 ms",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_6 [label="Move: 6.148 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_6 [label="Move: 77.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_7 [label="Move: 5.932 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_7 [label="Move: 114.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_7 [label="Move: 22.4 us",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_9 [label="Move: 6.407 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_9 [label="Move: 96.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_9 [label="Move: 16.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_9 [label="Move: 12.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_9 [label="Move: 12.6 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/examples/examples_results/enable_logging/viz_enable_logging_oldAPI_example_result.png b/examples/examples_results/enable_logging/viz_enable_logging_oldAPI_example_result.png deleted file mode 100644 index f4a2cec..0000000 Binary files a/examples/examples_results/enable_logging/viz_enable_logging_oldAPI_example_result.png and /dev/null differ diff --git a/examples/examples_results/log_sink/viz_modernAPI_localeventlog_example_result.dot b/examples/examples_results/log_sink/viz_modernAPI_localeventlog_example_result.dot deleted file mode 100644 index 80ea7b4..0000000 --- a/examples/examples_results/log_sink/viz_modernAPI_localeventlog_example_result.dot +++ /dev/null @@ -1,51 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_5 [label="taskA -1.06 s",color="#000000",shape="ellipse",penwidth=5]; -n_9 [label="taskE -1.022 s",color="#000000",shape="ellipse",penwidth=5]; -n_6 [label="taskB -1.028 s",color="#000000",shape="ellipse",penwidth=5]; -n_7 [label="taskC -1.04 s",color="#000000",shape="ellipse",penwidth=5]; -n_8 [label="taskD -1.044 s",color="#000000",shape="ellipse",penwidth=5]; -n_10 [label="taskF -1.033 s",color="#000000",shape="ellipse",penwidth=5]; -n_arg_1_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_5 -> n_5 [label="Move: 3.5 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_5 -> n_5 [label="Move: 1.9 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_9 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_9 -> n_9 [label="Move: 2.1 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_9 [label="5",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_9 -> n_9 [label="Move: 1.7 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_6 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_6 -> n_6 [label="Move: 3.1 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_6 [label="2",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_6 -> n_6 [label="Move: 2.6 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_7 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_7 -> n_7 [label="Move: 3.1 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_7 [label="3",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_7 -> n_7 [label="Move: 2.4 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_8 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_8 -> n_8 [label="Move: 3.0 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_8 [label="4",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_8 -> n_8 [label="Move: 2.5 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_10 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_10 -> n_10 [label="Move: 3.3 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_10 [label="6",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_10 -> n_10 [label="Move: 2.6 us",color="black;0.5:#000000",penwidth=2]; -n_5 -> n_9 [label="Move: 8.882 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_6 [label="Move: 18.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_7 [label="Move: 15.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_7 [label="Move: 4.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_8 [label="Move: 13.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_8 [label="Move: 3.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_8 [label="Move: 3.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_10 [label="Move: 13.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_10 [label="Move: 4.3 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_10 [label="Move: 4.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_10 [label="Move: 3.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_9 -> n_10 [label="Move: 4.5 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/examples/examples_results/log_sink/viz_modernAPI_localeventlog_example_result.png b/examples/examples_results/log_sink/viz_modernAPI_localeventlog_example_result.png deleted file mode 100644 index 78746ba..0000000 Binary files a/examples/examples_results/log_sink/viz_modernAPI_localeventlog_example_result.png and /dev/null differ diff --git a/examples/examples_results/log_sink/viz_modernAPI_multieventlog_example_result.dot b/examples/examples_results/log_sink/viz_modernAPI_multieventlog_example_result.dot deleted file mode 100644 index 3669419..0000000 --- a/examples/examples_results/log_sink/viz_modernAPI_multieventlog_example_result.dot +++ /dev/null @@ -1,51 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_5 [label="taskA -1.058 s",color="#000000",shape="ellipse",penwidth=5]; -n_9 [label="taskE -1.032 s",color="#000000",shape="ellipse",penwidth=5]; -n_6 [label="taskB -1.024 s",color="#000000",shape="ellipse",penwidth=5]; -n_7 [label="taskC -1.044 s",color="#000000",shape="ellipse",penwidth=5]; -n_8 [label="taskD -1.031 s",color="#000000",shape="ellipse",penwidth=5]; -n_10 [label="taskF -1.031 s",color="#000000",shape="ellipse",penwidth=5]; -n_arg_1_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_5 -> n_5 [label="Move: 7.639 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_5 -> n_5 [label="Move: 40.6 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_9 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_9 -> n_9 [label="Move: 7.044 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_9 [label="5",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_9 -> n_9 [label="Move: 46.1 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_6 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_6 -> n_6 [label="Move: 8.662 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_6 [label="2",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_6 -> n_6 [label="Move: 46.4 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_7 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_7 -> n_7 [label="Move: 8.475 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_7 [label="3",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_7 -> n_7 [label="Move: 39.5 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_8 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_8 -> n_8 [label="Move: 7.795 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_8 [label="4",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_8 -> n_8 [label="Move: 41.5 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_10 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_10 -> n_10 [label="Move: 7.477 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_10 [label="6",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_10 -> n_10 [label="Move: 43.8 us",color="black;0.5:#000000",penwidth=2]; -n_5 -> n_9 [label="Move: 47.882 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_6 [label="Move: 6.215 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_7 [label="Move: 6.068 ms",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_7 [label="Move: 40.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_8 [label="Move: 5.636 ms",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_8 [label="Move: 41.9 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_8 [label="Move: 19.1 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_10 [label="Move: 5.835 ms",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_10 [label="Move: 43.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_10 [label="Move: 19.4 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_10 [label="Move: 25.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_9 -> n_10 [label="Move: 18.0 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/examples/examples_results/log_sink/viz_modernAPI_multieventlog_example_result.png b/examples/examples_results/log_sink/viz_modernAPI_multieventlog_example_result.png deleted file mode 100644 index 1c03659..0000000 Binary files a/examples/examples_results/log_sink/viz_modernAPI_multieventlog_example_result.png and /dev/null differ diff --git a/examples/examples_results/log_sink/viz_oldAPI_localeventlog_example_result.dot b/examples/examples_results/log_sink/viz_oldAPI_localeventlog_example_result.dot deleted file mode 100644 index 4ee54bf..0000000 --- a/examples/examples_results/log_sink/viz_oldAPI_localeventlog_example_result.dot +++ /dev/null @@ -1,51 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_4 [label="taskA -1.06 s",color="#000000",shape="ellipse",penwidth=5]; -n_8 [label="taskE -1.024 s",color="#000000",shape="ellipse",penwidth=5]; -n_5 [label="taskB -1.021 s",color="#000000",shape="ellipse",penwidth=5]; -n_6 [label="taskC -1.04 s",color="#000000",shape="ellipse",penwidth=5]; -n_7 [label="taskD -1.042 s",color="#000000",shape="ellipse",penwidth=5]; -n_9 [label="taskF -1.047 s",color="#000000",shape="ellipse",penwidth=5]; -n_arg_1_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_5 -> n_5 [label="Move: 16.5 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_5 [label="2",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_5 -> n_5 [label="Move: 5.7 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_4 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_4 -> n_4 [label="Move: 5.1 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_4 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_4 -> n_4 [label="Move: 2.2 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_6 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_6 -> n_6 [label="Move: 2.9 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_6 [label="3",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_6 -> n_6 [label="Move: 2.9 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_7 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_7 -> n_7 [label="Move: 2.8 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_7 [label="4",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_7 -> n_7 [label="Move: 2.4 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_9 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_9 -> n_9 [label="Move: 2.7 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_9 [label="6",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_9 -> n_9 [label="Move: 2.6 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_8 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_8 -> n_8 [label="Move: 2.1 us",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_8 [label="5",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_8 -> n_8 [label="Move: 2.1 us",color="black;0.5:#000000",penwidth=2]; -n_4 -> n_8 [label="Move: 9.596 ms",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_5 [label="Move: 31.4 us",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_6 [label="Move: 12.4 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_6 [label="Move: 4.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_7 [label="Move: 14.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_7 [label="Move: 4.5 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_7 [label="Move: 3.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_9 [label="Move: 12.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_9 [label="Move: 4.4 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_9 [label="Move: 4.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_9 [label="Move: 3.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_9 [label="Move: 4.0 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/examples/examples_results/log_sink/viz_oldAPI_localeventlog_example_result.png b/examples/examples_results/log_sink/viz_oldAPI_localeventlog_example_result.png deleted file mode 100644 index 9776b03..0000000 Binary files a/examples/examples_results/log_sink/viz_oldAPI_localeventlog_example_result.png and /dev/null differ diff --git a/examples/examples_results/log_sink/viz_oldAPI_multieventlog_example_result.dot b/examples/examples_results/log_sink/viz_oldAPI_multieventlog_example_result.dot deleted file mode 100644 index 91cfd05..0000000 --- a/examples/examples_results/log_sink/viz_oldAPI_multieventlog_example_result.dot +++ /dev/null @@ -1,51 +0,0 @@ -strict digraph { -graph [layout=dot,rankdir=LR]; -n_4 [label="taskA -1.212 s",color="#000000",shape="ellipse",penwidth=5]; -n_8 [label="taskE -1.026 s",color="#000000",shape="ellipse",penwidth=5]; -n_5 [label="taskB -1.028 s",color="#000000",shape="ellipse",penwidth=5]; -n_6 [label="taskC -1.043 s",color="#000000",shape="ellipse",penwidth=5]; -n_7 [label="taskD -1.033 s",color="#000000",shape="ellipse",penwidth=5]; -n_9 [label="taskF -1.043 s",color="#000000",shape="ellipse",penwidth=5]; -n_arg_1_to_5 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_5 -> n_5 [label="Move: 7.137 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_5 [label="2",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_5 -> n_5 [label="Move: 26.9 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_4 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_4 -> n_4 [label="Move: 6.94 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_4 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_4 -> n_4 [label="Move: 28.2 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_6 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_6 -> n_6 [label="Move: 7.225 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_6 [label="3",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_6 -> n_6 [label="Move: 30.3 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_7 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_7 -> n_7 [label="Move: 7.487 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_7 [label="4",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_7 -> n_7 [label="Move: 32.9 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_9 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_9 -> n_9 [label="Move: 9.547 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_9 [label="6",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_9 -> n_9 [label="Move: 32.2 us",color="black;0.5:#000000",penwidth=2]; -n_arg_1_to_8 [label="1",color="black",shape="ellipse",penwidth=5]; -n_arg_1_to_8 -> n_8 [label="Move: 6.95 ms",color="black;0.5:#000000",penwidth=2]; -n_arg_2_to_8 [label="5",color="black",shape="ellipse",penwidth=5]; -n_arg_2_to_8 -> n_8 [label="Move: 29.8 us",color="black;0.5:#000000",penwidth=2]; -n_4 -> n_8 [label="Move: 48.82 ms",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_5 [label="Move: 6.144 ms",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_6 [label="Move: 6.785 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_6 [label="Move: 32.0 us",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_7 [label="Move: 5.817 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_7 [label="Move: 30.8 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_7 [label="Move: 11.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_4 -> n_9 [label="Move: 7.055 ms",color="#000000;0.5:#000000",penwidth=2]; -n_5 -> n_9 [label="Move: 30.6 us",color="#000000;0.5:#000000",penwidth=2]; -n_6 -> n_9 [label="Move: 13.2 us",color="#000000;0.5:#000000",penwidth=2]; -n_7 -> n_9 [label="Move: 10.7 us",color="#000000;0.5:#000000",penwidth=2]; -n_8 -> n_9 [label="Move: 10.6 us",color="#000000;0.5:#000000",penwidth=2]; -} diff --git a/examples/examples_results/log_sink/viz_oldAPI_multieventlog_example_result.png b/examples/examples_results/log_sink/viz_oldAPI_multieventlog_example_result.png deleted file mode 100644 index 4c6971c..0000000 Binary files a/examples/examples_results/log_sink/viz_oldAPI_multieventlog_example_result.png and /dev/null differ diff --git a/examples/examples_results/render_logs/render_graphviz_complex_example_result.png b/examples/examples_results/render_logs/render_graphviz_complex_example_result.png deleted file mode 100644 index 35b89f7..0000000 Binary files a/examples/examples_results/render_logs/render_graphviz_complex_example_result.png and /dev/null differ diff --git a/examples/examples_results/render_logs/render_graphviz_simple_example_result.png b/examples/examples_results/render_logs/render_graphviz_simple_example_result.png deleted file mode 100644 index 75d2bb5..0000000 Binary files a/examples/examples_results/render_logs/render_graphviz_simple_example_result.png and /dev/null differ diff --git a/examples/logging/README.md b/examples/logging/README.md deleted file mode 100644 index 2ae3362..0000000 --- a/examples/logging/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# Logging in Dagger - -Dagger documentation mentions several ways to get and use scheduler logs. In this directory you can find the relevant examples. - -**IMPORTANT** - -Essential problem of logging in Dagger is that it may work differently for old and new API. - -The examples and the herein text are relevant as of the Dagger 0.18.11 (unless otherwise stated) - -## Getting logs - -Currently there are essentially two ways to get logs: low-level usage of log_sinks (which define how the logs will be collected and processed) and high-level interface (`enable_logging!()`, `fetch_logs!()` functions). The corresponding sections will describe each in more detail and provide some examples of their usage. - -### log_sink - -Log_sink is a mechanism used to determine, how the logs are collected and processed. There are two types of log_sinks (there are also other ones, but this are most widely used): straightforward LocalEventLog, which saves logs directly into the arrays contained in each worker, and the more modern MultiEventLog, which documentation recommends to use, and which comprises of consumers processing events (consumer may be responsible for some specific type of info) and aggregators. - -Importantly, `LocalEventLog` stores raw events (`Dagger.TimespanLogging.Event`), whereas `MultiEventLog` processes them with consumers functions first, putting their results to the corresponding to this consumer array. - -The folder "getting_logs/log_sink" contains the examples of directly getting logs using the log_sink parameter of the Dagger.Context. - -On `get_logs!()` both the `LocalEventLog` and `MultiEventLog` gather what they have stored on all workers so far and return that as a result of this function (by default, `LocalEventLog` additionally transforms events into timespans combining start_events and finish_events). Note that all the returned logs are cleaned up, so, for example, immediately calling `get_logs()` second time after the first one will return nothing. - -Another function, which can be used to get logs is `fetch_logs!()` (currently it is just alias for the `TimespanLogging.get_logs!(Dagger.Sch.eager_context())`) - -### enable_logging - -A new way to get logs, added recently is the `enable_logging!()`. It is a function to activate preconfigured logging mechanism (it uses `MultiEventLog` under the hood anyway). Use `fetch_logs!()` to retrieve the logs then. Check the corresponding example for more details. - -## Processing logs - -The focus here is to get the result in a `.dot` format to interpret it as a graph. This can be achieved via `Dagger.show_logs()` function. Essentially it gets the vector of timespans (of events), transforms them and writes result (`.dot` format) to an io object. As mentioned earlier, `LocalEventLog` stores raw events; however, `get_logs!(::LocalEventLog; ...)` transforms them into the timespans, which are the inputs to a `Dagger.show_logs()`. All of that happens under the hood, however, this is not the case for `MultiEventLog`, as it does not store raw events. Therefore, by default, only the `LocalEventLog` can generate logs suitable to use for graph plotting. - -!!! Important - -Due to a known issues, currently (as of Dagger 0.18.11) `show_logs()` function contains a few bugs, which make the correct visualization not possible. Check [this](https://github.com/JuliaParallel/Dagger.jl/pull/531) pull request for the details. For now, examples use modified version of `GraphVizSimpleExt.jl` contained in the project and resolving this issue. - -### Show logs - -The `show_logs` function may be used to return the DAG representation in a `.dot` format with some metadata. - -`show_logs` function is essentially defined only if `Colors` package is imported/used - you will get a "Not implemented" error else. That is the code implementing it: -![show_logs implementation](../../docs/show_logs_impl.png) - -If `Colors` package is defined, the following definitions of `show_logs` are added: -![additional show logs implementation](../../docs/add_show_logs_impl.png) - -#### viz_oldAPI_localeventlog.jl - -Here, the `LocalEventLog` type of `log_sink` is used with the old API of Dagger (`Dagger.delayed`). This results in logs being written in the corresponding log file in the `.dot` format (it is important to import `Colors` package, as it also enables the relevant Dagger function) and visualized in another `.png` file. - -#### viz_oldAPI_multieventlog.jl - -This file contains the combination of old Dagger API and `MultiEventLog` as the log_sink. However, only `LocalEventLog` can be used directly with the `show_logs()`, due to how `Event`s are processed in `MultiEventLog`. In order to eventually write the logs to a file in a `.dot` format, one can use `Dagger.TimespanLogging.Events.FullMetrics` consumer which does nothing to the input event and return it as is, which is identical to `LocalEventLog` behaviour. Then, the workflow for the latter one can be reproduced. - -#### viz_modernAPI_localeventlog.jl - -Modern Dagger API and `LocalEventLog` `log_sink` are combined here. - -#### viz_modernAPI_multieventlog.jl - -Modern Dagger API used with `MultiEventLog`. The same kind of workaround is applied, as earlier (viz_oldAPI_multieventlog.jl). - -#### viz_raw_enable_logging_oldAPI.jl - -This example contains an attempt to use `enable_logging!()` with `show_logs()`, which causes an error. This is intentionally left example to show that some kind of workaround is needed. - -#### viz_enable_logging_oldAPI - -This example is similar to the previous one, except that now additional processing is applied to logs (as mentioned earlier, `MultiEventLog` is used under the hood in `enable_logging!()`, so the same workaround has been applied here). - -#### viz_enable_logging_modernAPI.jl - -The same as the previous one, but with the modern API. Should not work as of Dagger 0.18.11 (but works using modified `GraphVizSimpleExt`), however, should be fixed soon. - -## Visualizing logs - -### Custom logs vizualization - -Such vizualization can be seen in the examples for the previous section. - -### Webdash - -Webdash does not seem to work as of Dagger 0.18.11. - -### Render Logs - -The corresponding folder contains examples of the `render_logs()` function use. - -`:graphviz` does not seem to generate graphs properly for now. Also sometimes throws `KeyError`. - -`:plots_gantt_ps` displays the following result: - -![gantt_ps](../../docs/plot_gantt_ps.png) - -`:plots_gantt`: - -![gantt](../../docs/plot_gantt.png) \ No newline at end of file diff --git a/examples/logging/getting_logs/enable_logging.jl b/examples/logging/getting_logs/enable_logging.jl deleted file mode 100644 index b2b507a..0000000 --- a/examples/logging/getting_logs/enable_logging.jl +++ /dev/null @@ -1,13 +0,0 @@ -include("../../dummy_tasks.jl") # here Dagger is imported as well - -Dagger.enable_logging!(tasknames = true, # Here we can choose the consumers to use. Check source code for more. - taskdeps = true, - taskargs = true, - taskargmoves = true) - -t = modernAPI_graph_setup(1) # Can also be the old API -fetch(t) - -logs = Dagger.fetch_logs!() -println("\n\nRaw logs:") -println(logs) # <- That should be a Dictionary of the form: {worker_id -> {consumer_symbol -> [list of the events on that worker after being processed by a consumer corresponding to this symbol]}} diff --git a/examples/logging/getting_logs/log_sink/localeventlog.jl b/examples/logging/getting_logs/log_sink/localeventlog.jl deleted file mode 100644 index f54d23b..0000000 --- a/examples/logging/getting_logs/log_sink/localeventlog.jl +++ /dev/null @@ -1,11 +0,0 @@ -include("../../../dummy_tasks.jl") # here Dagger is imported as well - -ctx = Dagger.Sch.eager_context() -ctx.log_sink = Dagger.TimespanLogging.LocalEventLog() # By default, NoOpLog, which does nothing, is set - -t = modernAPI_graph_setup(1) # Can also be the old API -fetch(t) - -logs = Dagger.TimespanLogging.get_logs!(ctx) -println("\n\nRaw logs:") -println(logs) # <- That should be a Dictionary of the form: {worker_id -> [list of the events on that worker]} diff --git a/examples/logging/getting_logs/log_sink/multieventlog.jl b/examples/logging/getting_logs/log_sink/multieventlog.jl deleted file mode 100644 index a672f6c..0000000 --- a/examples/logging/getting_logs/log_sink/multieventlog.jl +++ /dev/null @@ -1,27 +0,0 @@ -include("../../../dummy_tasks.jl") # here Dagger is imported as well - -TimespanLogging = Dagger.TimespanLogging -ctx = Dagger.Sch.eager_context() -ml = TimespanLogging.MultiEventLog() # By default, NoOpLog, which does nothing, is set -ctx.log_sink = ml - -# For now, MultiEventLog is unconfigured and will not log anything useful. -# Let's configure it: -ml[:core] = TimespanLogging.Events.CoreMetrics() -ml[:id] = TimespanLogging.Events.IDMetrics() -# ml[:profile] = DaggerWebDash.ProfileMetrics() -ml[:wsat] = Dagger.Events.WorkerSaturation() -ml[:loadavg] = TimespanLogging.Events.CPULoadAverages() -ml[:bytes] = Dagger.Events.BytesAllocd() -ml[:mem] = TimespanLogging.Events.MemoryFree() -ml[:esat] = TimespanLogging.Events.EventSaturation() -ml[:psat] = Dagger.Events.ProcessorSaturation() -ml[:timeline] = TimespanLogging.Events.TimelineMetrics() -# There may be other consumers. In general, they can be defined by user as well to create some custom metrics. - -t = modernAPI_graph_setup(1) # Can also be the old API -fetch(t) - -logs = Dagger.TimespanLogging.get_logs!(ctx) -println("\n\nRaw logs:") -println(logs) # <- That should be a Dictionary of the form: {worker_id -> {consumer_symbol -> [list of the events on that worker after being processed by a consumer corresponding to this symbol]}} diff --git a/examples/logging/processing_and_visualizing_logs/enable_logging/viz_enable_logging_modernAPI.jl b/examples/logging/processing_and_visualizing_logs/enable_logging/viz_enable_logging_modernAPI.jl deleted file mode 100644 index 13d1b0b..0000000 --- a/examples/logging/processing_and_visualizing_logs/enable_logging/viz_enable_logging_modernAPI.jl +++ /dev/null @@ -1,53 +0,0 @@ -import Colors -using Dagger -using TimespanLogging -using DaggerWebDash -using FrameworkDemo -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) -FILENAME_TEMPLATE = "$output_dir/viz_enable_logging_modernAPI" - -function workaround() - logs = Dagger.fetch_logs!() # Fetch all the logs - events_logs = Dict() - for (key, value) in logs - events_logs[key] = value[:full] # Leave only the events logs - end - spans = Dagger.TimespanLogging.build_timespans(vcat(values(events_logs)...)).completed # Form the timespans - convert(Vector{Dagger.TimespanLogging.Timespan}, spans) -end - -function configure_MultiEventLog() - Dagger.enable_logging!(timeline = true, # Some example configuration - tasknames = true, - taskdeps = true, - taskargs = true, - taskargmoves = true) - Dagger.Sch.eager_context().log_sink[:full] = Dagger.TimespanLogging.Events.FullMetrics() -end - -# Use MultiEventLog sink: -configure_MultiEventLog() -# Or use a Local Event Log sink: -# configure_LocalEventLog() - -graph_thunk = modernAPI_graph_setup(1) - -println(fetch(graph_thunk)) # Wait for the graph execution and fetch the results - -# Use MultiEventLog sink: -logs = workaround() -# Or use a Local Event Log sink: -# logs = Dagger.TimespanLogging.get_logs!(Dagger.Sch.eager_context()) - -log_file_name = FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".dot" -open(log_file_name, "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, logs, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, logs, :graphviz_simple) after the bug fix & update in the package - # FrameworkDemo.Dagger.show_logs(graph_thunk, logs, :graphviz_simple) # Returns the string representation of the graph -end - -FrameworkDemo.dot_to_png(log_file_name, - FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".png", 2000, - 2000) diff --git a/examples/logging/processing_and_visualizing_logs/enable_logging/viz_enable_logging_oldAPI.jl b/examples/logging/processing_and_visualizing_logs/enable_logging/viz_enable_logging_oldAPI.jl deleted file mode 100644 index 2f368d3..0000000 --- a/examples/logging/processing_and_visualizing_logs/enable_logging/viz_enable_logging_oldAPI.jl +++ /dev/null @@ -1,55 +0,0 @@ -using Distributed -using Colors -using GraphViz -using Cairo -using Dagger -using FrameworkDemo - -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) -FILENAME_TEMPLATE = "$output_dir/viz_enable_logging_oldAPI" - -function workaround() - logs = Dagger.fetch_logs!() # Fetch all the logs - events_logs = Dict() - for (key, value) in logs - events_logs[key] = value[:full] # Leave only the events logs - end - spans = Dagger.TimespanLogging.build_timespans(vcat(values(events_logs)...)).completed # Form the timespans - convert(Vector{Dagger.TimespanLogging.Timespan}, spans) -end - -function configure_MultiEventLog() - Dagger.enable_logging!(timeline = true, # Some example configuration - tasknames = true, - taskdeps = true, - taskargs = true, - taskargmoves = true) - Dagger.Sch.eager_context().log_sink[:full] = Dagger.TimespanLogging.Events.FullMetrics() -end - -# Use MultiEventLog sink: -configure_MultiEventLog() -# Or use a Local Event Log sink: -# configure_LocalEventLog() - -graph_thunk = oldAPI_graph_setup(1) - -println(collect(Dagger.Sch.eager_context(), graph_thunk)) # Wait for the graph execution and fetch the results - -# Use MultiEventLog sink: -logs = workaround() -# Or use a Local Event Log sink: -# logs = Dagger.TimespanLogging.get_logs!(Dagger.Sch.eager_context()) - -log_file_name = FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".dot" -open(log_file_name, "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, logs, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, logs, :graphviz_simple) after the bug fix in the package - # FrameworkDemo.Dagger.show_logs(graph_thunk, logs, :graphviz_simple) # Returns the string representation of the graph -end - -FrameworkDemo.dot_to_png(log_file_name, - FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".png", 2000, - 2000) diff --git a/examples/logging/processing_and_visualizing_logs/enable_logging/viz_raw_enable_logging_oldAPI.jl b/examples/logging/processing_and_visualizing_logs/enable_logging/viz_raw_enable_logging_oldAPI.jl deleted file mode 100644 index 5c58081..0000000 --- a/examples/logging/processing_and_visualizing_logs/enable_logging/viz_raw_enable_logging_oldAPI.jl +++ /dev/null @@ -1,29 +0,0 @@ -# TODO: push "import Dagger: istask, dependents" to graphsimpleviz_ext -import Colors -using Distributed -using Dagger -using FrameworkDemo -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) - -FILENAME_TEMPLATE = "$output_dir/viz_raw_enable_logging_oldAPI" - -Dagger.enable_logging!() - -graph_thunk = oldAPI_graph_setup(1) - -ctx = Dagger.Sch.eager_context() -println(collect(ctx, graph_thunk)) # Wait for the graph execution and fetch the results - -log_file_name = FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".dot" -open(log_file_name, "w") do io - #FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, Dagger.fetch_logs!(), :graphviz_simple) - FrameworkDemo.ModGraphVizSimple.show_logs(graph_thunk, Dagger.fetch_logs!(), - :graphviz_simple) -end - -FrameworkDemo.dot_to_png(log_file_name, - FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".png", 700, - 700) diff --git a/examples/logging/processing_and_visualizing_logs/log_sink/viz_modernAPI_localeventlog.jl b/examples/logging/processing_and_visualizing_logs/log_sink/viz_modernAPI_localeventlog.jl deleted file mode 100644 index 068cfaf..0000000 --- a/examples/logging/processing_and_visualizing_logs/log_sink/viz_modernAPI_localeventlog.jl +++ /dev/null @@ -1,29 +0,0 @@ -import Colors # Writing logs into a file would fail without this import -using FrameworkDemo -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) - -FILENAME_TEMPLATE = "./examples/examples_results/log_sink/viz_modernAPI_localeventlog" - -ctx = Dagger.Sch.eager_context() - -# Setup the log sink (that basically chooses the behaviour of the logging mechanism) -# LocalEventLog -ctx.log_sink = Dagger.TimespanLogging.LocalEventLog() - -graph_thunk = modernAPI_graph_setup(1) -println(fetch(graph_thunk)) - -logs = Dagger.TimespanLogging.get_logs!(ctx) -log_file_name = FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".dot" -open(log_file_name, "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, logs, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, logs, :graphviz_simple) after the bug fix in the package - # or FrameworkDemo.ModGraphVizSimple.show_logs(io, logs, :graphviz_simple) # Dagger.show_logs(io, logs, :graphviz_simple) after the bug fix in the package - # or FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, :graphviz_simple) after the bug fix in the package -end - -FrameworkDemo.dot_to_png(log_file_name, - FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".png", 2000, - 2000) diff --git a/examples/logging/processing_and_visualizing_logs/log_sink/viz_modernAPI_multieventlog.jl b/examples/logging/processing_and_visualizing_logs/log_sink/viz_modernAPI_multieventlog.jl deleted file mode 100644 index 190d647..0000000 --- a/examples/logging/processing_and_visualizing_logs/log_sink/viz_modernAPI_multieventlog.jl +++ /dev/null @@ -1,51 +0,0 @@ -import Colors # Writing logs into a file would fail without this import -using FrameworkDemo -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) - -FILENAME_TEMPLATE = "$output_dir/viz_modernAPI_multieventlog" - -ctx = Dagger.Sch.eager_context() - -# Setup the log sink (that basically chooses the behaviour of the logging mechanism) -# MultiEventLog -ml = Dagger.TimespanLogging.MultiEventLog() - -# MultiEventLog configuration -ml[:core] = Dagger.TimespanLogging.Events.CoreMetrics() -ml[:id] = Dagger.TimespanLogging.Events.IDMetrics() -ml[:full] = Dagger.TimespanLogging.Events.FullMetrics() -ml[:timeline] = Dagger.TimespanLogging.Events.TimelineMetrics() -ml[:wsat] = Dagger.Events.WorkerSaturation() -ml[:loadavg] = Dagger.TimespanLogging.Events.CPULoadAverages() -ml[:bytes] = Dagger.Events.BytesAllocd() -ml[:mem] = Dagger.TimespanLogging.Events.MemoryFree() -ml[:esat] = Dagger.TimespanLogging.Events.EventSaturation() -ml[:psat] = Dagger.Events.ProcessorSaturation() - -ctx.log_sink = ml - -graph_thunk = modernAPI_graph_setup(1) -println(fetch(graph_thunk)) - -# Workaround to use show_logs() with the MultiEventLog -logs = Dagger.TimespanLogging.get_logs!(ctx) # Get all kind of logs on all the workers -events_logs = Dict() -for (key, value) in logs - events_logs[key] = value[:full] # Leave only the events logs -end -spans = Dagger.TimespanLogging.build_timespans(vcat(values(events_logs)...)).completed # Form the timespans -timespan_logs = convert(Vector{Dagger.TimespanLogging.Timespan}, spans) -log_file_name = FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".dot" -open(log_file_name, "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, timespan_logs, - :graphviz_simple) # Dagger.show_logs(io, graph_thunk, timespan_logs, :graphviz_simple) after the bug fix in the package - # or FrameworkDemo.ModGraphVizSimpleExt.show_logs(io, timespan_logs, :graphviz_simple) # Dagger.show_logs(io, timespan_logs, :graphviz_simple) after the bug fix in the package - # or FrameworkDemo.ModGraphVizSimpleExt.show_logs(io, graph_thunk, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, :graphviz_simple) after the bug fix in the package -end - -FrameworkDemo.dot_to_png(log_file_name, - FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".png", 2000, - 2000) diff --git a/examples/logging/processing_and_visualizing_logs/log_sink/viz_oldAPI_localeventlog.jl b/examples/logging/processing_and_visualizing_logs/log_sink/viz_oldAPI_localeventlog.jl deleted file mode 100644 index b9fdb7f..0000000 --- a/examples/logging/processing_and_visualizing_logs/log_sink/viz_oldAPI_localeventlog.jl +++ /dev/null @@ -1,29 +0,0 @@ -import Colors # Writing logs into a file would fail without this import -using FrameworkDemo -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) - -FILENAME_TEMPLATE = "$output_dir/viz_oldAPI_localeventlog" - -ctx = Dagger.Sch.eager_context() - -# Setup the log sink (that basically chooses the behaviour of the logging mechanism) -# LocalEventLog -ctx.log_sink = Dagger.TimespanLogging.LocalEventLog() - -graph_thunk = oldAPI_graph_setup(1) -println(collect(ctx, graph_thunk)) - -logs = Dagger.TimespanLogging.get_logs!(ctx) -log_file_name = FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".dot" -open(log_file_name, "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, logs, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, logs, :graphviz_simple) after the bug fix in the package - # or ModGraphVizSimpleExt.show_logs(io, logs, :graphviz_simple) # Dagger.show_logs(io, logs, :graphviz_simple) after the bug fix in the package - # or ModGraphVizSimpleExt.show_logs(io, graph_thunk, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, :graphviz_simple) after the bug fix in the package -end - -FrameworkDemo.dot_to_png(log_file_name, - FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".png", 2000, - 2000) diff --git a/examples/logging/processing_and_visualizing_logs/log_sink/viz_oldAPI_multieventlog.jl b/examples/logging/processing_and_visualizing_logs/log_sink/viz_oldAPI_multieventlog.jl deleted file mode 100644 index 66f68f6..0000000 --- a/examples/logging/processing_and_visualizing_logs/log_sink/viz_oldAPI_multieventlog.jl +++ /dev/null @@ -1,51 +0,0 @@ -import Colors # Writing logs into a file would fail without this import -using FrameworkDemo -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) - -FILENAME_TEMPLATE = "$output_dir/viz_oldAPI_multieventlog" - -ctx = Dagger.Sch.eager_context() - -# Setup the log sink (that basically chooses the behaviour of the logging mechanism) -# MultiEventLog -ml = Dagger.TimespanLogging.MultiEventLog() - -# MultiEventLog configuration -ml[:core] = Dagger.TimespanLogging.Events.CoreMetrics() -ml[:id] = Dagger.TimespanLogging.Events.IDMetrics() -ml[:full] = Dagger.TimespanLogging.Events.FullMetrics() -ml[:timeline] = Dagger.TimespanLogging.Events.TimelineMetrics() -ml[:wsat] = Dagger.Events.WorkerSaturation() -ml[:loadavg] = Dagger.TimespanLogging.Events.CPULoadAverages() -ml[:bytes] = Dagger.Events.BytesAllocd() -ml[:mem] = Dagger.TimespanLogging.Events.MemoryFree() -ml[:esat] = Dagger.TimespanLogging.Events.EventSaturation() -ml[:psat] = Dagger.Events.ProcessorSaturation() - -ctx.log_sink = ml - -graph_thunk = oldAPI_graph_setup(1) -println(collect(ctx, graph_thunk)) - -# Workaround to use show_logs() with the MultiEventLog -logs = Dagger.TimespanLogging.get_logs!(ctx) # Get all kind of logs on all the workers -events_logs = Dict() -for (key, value) in logs - events_logs[key] = value[:full] # Leave only the events logs -end -spans = Dagger.TimespanLogging.build_timespans(vcat(values(events_logs)...)).completed # Form the timespans -timespan_logs = convert(Vector{Dagger.TimespanLogging.Timespan}, spans) -log_file_name = FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".dot" -open(log_file_name, "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, graph_thunk, timespan_logs, - :graphviz_simple) # Dagger.show_logs(io, graph_thunk, timespan_logs, :graphviz_simple) after the bug fix in the package - # or ModGraphVizSimpleExt.show_logs(io, timespan_logs, :graphviz_simple) # Dagger.show_logs(io, timespan_logs, :graphviz_simple) after the bug fix in the package - # or ModGraphVizSimpleExt.show_logs(io, graph_thunk, :graphviz_simple) # Dagger.show_logs(io, graph_thunk, :graphviz_simple) after the bug fix in the package -end - -FrameworkDemo.dot_to_png(log_file_name, - FrameworkDemo.timestamp_string(FILENAME_TEMPLATE) * ".png", 2000, - 2000) diff --git a/examples/logging/processing_and_visualizing_logs/render_logs/render_graphviz_complex.jl b/examples/logging/processing_and_visualizing_logs/render_logs/render_graphviz_complex.jl deleted file mode 100644 index af761d7..0000000 --- a/examples/logging/processing_and_visualizing_logs/render_logs/render_graphviz_complex.jl +++ /dev/null @@ -1,93 +0,0 @@ -using Distributed -new_procs = addprocs(4) # Set the number of workers -using Colors -using GraphViz -using Cairo -using Dagger -using DaggerWebDash -using Graphs -using MetaGraphs -using FrameworkDemo - -# Defining constants -graph1_path = "./data/demo/sequencer/df.graphml" -graph2_path = "./data/demo/parallel/df.graphml" - -output_dir = "examples/results" -mkpath(output_dir) -LOGS_FILE = FrameworkDemo.timestamp_string("$output_dir/out") * ".dot" -GRAPH_IMAGE_PATH = FrameworkDemo.timestamp_string("$output_dir/DAG") * ".png" - -OUTPUT_GRAPH_PATH = "$output_dir/" -OUTPUT_GRAPH_IMAGE_PATH = "$output_dir/" - -MAX_GRAPHS_RUN = 3 - -function execution(graphs_map) - graphs_being_run = Set{Int}() - graphs_dict = Dict{Int, String}() - graphs_tasks = Dict{Int, Dagger.DTask}() - graphs = FrameworkDemo.parse_graphs(graphs_map, OUTPUT_GRAPH_PATH, - OUTPUT_GRAPH_IMAGE_PATH) - notifications = RemoteChannel(() -> Channel{Int}(32)) - # notifications = Channel{Int}(32) - for (i, (g_name, g)) in enumerate(graphs) - graphs_dict[i] = g_name - while !(length(graphs_being_run) < MAX_GRAPHS_RUN) - finished_graph_id = take!(notifications) - delete!(graphs_being_run, finished_graph_id) - delete!(graphs_tasks, i) - println("Dispatcher: graph finished - $finished_graph_id: $(graphs_dict[finished_graph_id])") - end - graphs_tasks[i] = FrameworkDemo.schedule_graph_with_notify(g, notifications, g_name, - i) - push!(graphs_being_run, i) - println("Dispatcher: scheduled graph $i: $g_name") - end - - results = [] - for (g_name, g) in graphs - g_map = Dict{Int, Any}() - for vertex_id in Graphs.vertices(g) - future = get_prop(g, vertex_id, :res_data) - g_map[vertex_id] = fetch(future) - end - push!(results, (g_name, g_map)) - end - - for (g_name, res) in results - for (id, value) in res - println("Graph: $g_name, Final result for vertex $id: $value") - end - end - for (_, task) in graphs_tasks - wait(task) - end -end - -function main(graphs_map) - Dagger.enable_logging!(tasknames = true, - taskdeps = true, - taskargs = true, - taskargmoves = true) - - @time execution(graphs_map) - - graph = Dagger.render_logs(Dagger.fetch_logs!(), :graphviz, disconnected = true, - color_by = :proc) - surface = Cairo.CairoSVGSurface(IOBuffer(), 7000, 2000) - context = Cairo.CairoContext(surface) - GraphViz.render(context, graph) - img_name = FrameworkDemo.timestamp_string("$output_dir/render_graphviz_complex") * - ".png" - write_to_png(surface, img_name) -end - -graphs_map = Dict{String, String}("graph1" => graph1_path, - "graph2" => graph2_path, - "graph3" => graph1_path, - "graph4" => graph2_path) - -main(graphs_map) -rmprocs!(Dagger.Sch.eager_context(), workers()) -rmprocs(workers()) diff --git a/examples/logging/processing_and_visualizing_logs/render_logs/render_graphviz_simple.jl b/examples/logging/processing_and_visualizing_logs/render_logs/render_graphviz_simple.jl deleted file mode 100644 index bd3befd..0000000 --- a/examples/logging/processing_and_visualizing_logs/render_logs/render_graphviz_simple.jl +++ /dev/null @@ -1,31 +0,0 @@ -using Distributed -new_procs = addprocs(4) # Set the number of workers -using Colors -using GraphViz -using Cairo -using Dagger -using FrameworkDemo -include("../../../dummy_tasks.jl") - -output_dir = "examples/results" -mkpath(output_dir) - -Dagger.enable_logging!(tasknames = true, - taskdeps = true, - taskargs = true, - taskargmoves = true) - -a = modernAPI_graph_setup(0.1) - -ctx = Dagger.Sch.eager_context() -println(fetch(a)) - -graph = Dagger.render_logs(Dagger.fetch_logs!(), :graphviz, disconnected = true, - color_by = :proc) - -surface = Cairo.CairoSVGSurface(IOBuffer(), 7000, 2000) -context = Cairo.CairoContext(surface) - -GraphViz.render(context, graph) -img_name = FrameworkDemo.timestamp_string("$output_dir/render_graphviz_simple") * ".png" -FrameworkDemo.write_to_png(surface, img_name) diff --git a/examples/logging/processing_and_visualizing_logs/render_logs/render_plots_gantt.jl b/examples/logging/processing_and_visualizing_logs/render_logs/render_plots_gantt.jl deleted file mode 100644 index 60aa285..0000000 --- a/examples/logging/processing_and_visualizing_logs/render_logs/render_plots_gantt.jl +++ /dev/null @@ -1,89 +0,0 @@ -using Distributed -new_procs = addprocs(4) # Set the number of workers -using Colors -using GraphViz -using Cairo -using DataFrames -using Plots -using Dagger -using DaggerWebDash -using Graphs -using MetaGraphs -using FrameworkDemo - -# Defining constants -graph1_path = "./data/demo/sequencer/df.graphml" -graph2_path = "./data/demo/parallel/df.graphml" - -output_dir = "examples/results" -mkpath(output_dir) -LOGS_FILE = FrameworkDemo.timestamp_string("$output_dir/out") * ".dot" -GRAPH_IMAGE_PATH = FrameworkDemo.timestamp_string("output_dir/DAG") * ".png" - -OUTPUT_GRAPH_PATH = "$output_dir/" -OUTPUT_GRAPH_IMAGE_PATH = "$output_dir/" - -MAX_GRAPHS_RUN = 3 - -function execution(graphs_map) - graphs_being_run = Set{Int}() - graphs_dict = Dict{Int, String}() - graphs_tasks = Dict{Int, Dagger.DTask}() - graphs = FrameworkDemo.parse_graphs(graphs_map, OUTPUT_GRAPH_PATH, - OUTPUT_GRAPH_IMAGE_PATH) - notifications = RemoteChannel(() -> Channel{Int}(32)) - # notifications = Channel{Int}(32) - for (i, (g_name, g)) in enumerate(graphs) - graphs_dict[i] = g_name - while !(length(graphs_being_run) < MAX_GRAPHS_RUN) - finished_graph_id = take!(notifications) - delete!(graphs_being_run, finished_graph_id) - delete!(graphs_tasks, i) - println("Dispatcher: graph finished - $finished_graph_id: $(graphs_dict[finished_graph_id])") - end - graphs_tasks[i] = FrameworkDemo.schedule_graph_with_notify(g, notifications, g_name, - i) - push!(graphs_being_run, i) - println("Dispatcher: scheduled graph $i: $g_name") - end - - results = [] - for (g_name, g) in graphs - g_map = Dict{Int, Any}() - for vertex_id in Graphs.vertices(g) - future = get_prop(g, vertex_id, :res_data) - g_map[vertex_id] = fetch(future) - end - push!(results, (g_name, g_map)) - end - - for (g_name, res) in results - for (id, value) in res - println("Graph: $g_name, Final result for vertex $id: $value") - end - end - for (_, task) in graphs_tasks - wait(task) - end -end - -function main(graphs_map) - Dagger.enable_logging!(tasknames = true, - taskdeps = true, - taskargs = true, - taskargmoves = true) - - @time execution(graphs_map) - - plot = Dagger.render_logs(Dagger.fetch_logs!(), :plots_gantt) - display(plot) -end - -graphs_map = Dict{String, String}("graph1" => graph1_path, - "graph2" => graph2_path, - "graph3" => graph1_path, - "graph4" => graph2_path) - -main(graphs_map) -rmprocs!(Dagger.Sch.eager_context(), workers()) -rmprocs(workers()) diff --git a/examples/logging/processing_and_visualizing_logs/render_logs/render_plots_gantt_ps.jl b/examples/logging/processing_and_visualizing_logs/render_logs/render_plots_gantt_ps.jl deleted file mode 100644 index ccc6f63..0000000 --- a/examples/logging/processing_and_visualizing_logs/render_logs/render_plots_gantt_ps.jl +++ /dev/null @@ -1,89 +0,0 @@ -using Distributed -new_procs = addprocs(4) # Set the number of workers -using Colors -using GraphViz -using Cairo -using Dagger -using DataFrames -using Plots -using DaggerWebDash -using Graphs -using MetaGraphs -using FrameworkDemo - -# Defining constants -graph1_path = "./data/demo/sequencer/df.graphml" -graph2_path = "./data/demo/parallel/df.graphml" - -output_dir = "examples/results" -mkpath(output_dir) -LOGS_FILE = FrameworkDemo.timestamp_string("$output_dir/out") * ".dot" -GRAPH_IMAGE_PATH = FrameworkDemo.timestamp_string("$output_dir/DAG") * ".png" - -OUTPUT_GRAPH_PATH = "$output_dir/" -OUTPUT_GRAPH_IMAGE_PATH = "$output_dir/" - -MAX_GRAPHS_RUN = 3 - -function execution(graphs_map) - graphs_being_run = Set{Int}() - graphs_dict = Dict{Int, String}() - graphs_tasks = Dict{Int, Dagger.DTask}() - graphs = FrameworkDemo.parse_graphs(graphs_map, OUTPUT_GRAPH_PATH, - OUTPUT_GRAPH_IMAGE_PATH) - notifications = RemoteChannel(() -> Channel{Int}(32)) - # notifications = Channel{Int}(32) - for (i, (g_name, g)) in enumerate(graphs) - graphs_dict[i] = g_name - while !(length(graphs_being_run) < MAX_GRAPHS_RUN) - finished_graph_id = take!(notifications) - delete!(graphs_being_run, finished_graph_id) - delete!(graphs_tasks, i) - println("Dispatcher: graph finished - $finished_graph_id: $(graphs_dict[finished_graph_id])") - end - graphs_tasks[i] = FrameworkDemo.schedule_graph_with_notify(g, notifications, g_name, - i) - push!(graphs_being_run, i) - println("Dispatcher: scheduled graph $i: $g_name") - end - - results = [] - for (g_name, g) in graphs - g_map = Dict{Int, Any}() - for vertex_id in Graphs.vertices(g) - future = get_prop(g, vertex_id, :res_data) - g_map[vertex_id] = fetch(future) - end - push!(results, (g_name, g_map)) - end - - for (g_name, res) in results - for (id, value) in res - println("Graph: $g_name, Final result for vertex $id: $value") - end - end - for (_, task) in graphs_tasks - wait(task) - end -end - -function main(graphs_map) - Dagger.enable_logging!(tasknames = true, - taskdeps = true, - taskargs = true, - taskargmoves = true) - - @time execution(graphs_map) - - plot = Dagger.render_logs(Dagger.fetch_logs!(), :plots_gantt_ps) - display(plot) -end - -graphs_map = Dict{String, String}("graph1" => graph1_path, - "graph2" => graph2_path, - "graph3" => graph1_path, - "graph4" => graph2_path) - -main(graphs_map) -rmprocs!(Dagger.Sch.eager_context(), workers()) -rmprocs(workers()) diff --git a/examples/logging/processing_and_visualizing_logs/webdash.jl b/examples/logging/processing_and_visualizing_logs/webdash.jl deleted file mode 100644 index be38044..0000000 --- a/examples/logging/processing_and_visualizing_logs/webdash.jl +++ /dev/null @@ -1,57 +0,0 @@ -using Dagger, DaggerWebDash, TimespanLogging - -ctx = Context() # or `ctx = Dagger.Sch.eager_context()` for eager API usage -ml = TimespanLogging.MultiEventLog() - -## Add some logging events of interest - -ml[:core] = TimespanLogging.Events.CoreMetrics() -ml[:id] = TimespanLogging.Events.IDMetrics() -ml[:timeline] = TimespanLogging.Events.TimelineMetrics() -# ... - -# (Optional) Enable profile flamegraph generation with ProfileSVG -ml[:profile] = DaggerWebDash.ProfileMetrics() -ctx.profile = true - -# Create a LogWindow; necessary for real-time event updates -lw = TimespanLogging.Events.LogWindow(20 * 10^9, :core) -ml.aggregators[:logwindow] = lw - -# Create the D3Renderer server on port 8080 -d3r = DaggerWebDash.D3Renderer(8080) - -## Add some plots! Rendered top-down in order - -# Show an overview of all generated events as a Gantt chart -push!(d3r, DaggerWebDash.GanttPlot(:core, :id, :esat, :psat; title = "Overview")) - -# Show various numerical events as line plots over time -push!(d3r, DaggerWebDash.LinePlot(:core, :wsat, "Worker Saturation", "Running Tasks")) -push!(d3r, - DaggerWebDash.LinePlot(:core, :loadavg, "CPU Load Average", - "Average Running Threads")) -push!(d3r, DaggerWebDash.LinePlot(:core, :bytes, "Allocated Bytes", "Bytes")) -push!(d3r, DaggerWebDash.LinePlot(:core, :mem, "Available Memory", "% Free")) - -# Show a graph rendering of compute tasks and data movement between them -# Note: Profile events are ignored if absent from the log -push!(d3r, DaggerWebDash.GraphPlot(:core, :id, :timeline, :profile, "DAG")) - -# TODO: Not yet functional -#push!(d3r, DaggerWebDash.ProfileViewer(:core, :profile, "Profile Viewer")) - -# Add the D3Renderer as a consumer of special events generated by LogWindow -push!(lw.creation_handlers, d3r) -push!(lw.deletion_handlers, d3r) - -# D3Renderer is also an aggregator -ml.aggregators[:d3r] = d3r - -ctx.log_sink = ml -# ... use `ctx` - -include("../../dummy_tasks.jl") - -t = modernAPI_graph_setup(50) -println(fetch(t)) diff --git a/examples/parsing_graphs.jl b/examples/parsing_graphs.jl deleted file mode 100644 index 3459ede..0000000 --- a/examples/parsing_graphs.jl +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env julia - -using FrameworkDemo - -function main() - G = FrameworkDemo.parse_graphml("./data/demo/sequencer/df.graphml") - G_copy = deepcopy(G) - FrameworkDemo.show_graph(G_copy) -end - -if abspath(PROGRAM_FILE) == @__FILE__ - main() -end diff --git a/examples/processors_example.jl b/examples/processors_example.jl deleted file mode 100644 index ea29576..0000000 --- a/examples/processors_example.jl +++ /dev/null @@ -1,25 +0,0 @@ -# This example was taken from https://juliaparallel.org/Dagger.jl/v0.18/processors/ - -using Distributed - -ps1 = addprocs(2, exeflags = "--project") -@everywhere using Distributed, Dagger - -# Dummy task to wait for 0.5 seconds and then return the id of the worker -ts = delayed(vcat)((delayed(i -> (sleep(0.5); myid()))(i) for i in 1:20)...) - -ctx = Context() -# Scheduler is blocking, so we need a new task to add workers while it runs -job = @async collect(ctx, ts) - -# Lets fire up some new workers -ps2 = addprocs(2, exeflags = "--project") -@everywhere ps2 using Distributed, Dagger -# New workers are not available until we do this -addprocs!(ctx, ps2) - -# Lets hope the job didn't complete before workers were added :) -@show fetch(job) |> unique - -# and cleanup after ourselves... -workers() |> rmprocs diff --git a/examples/schedule.jl b/examples/schedule.jl deleted file mode 100644 index 3387af6..0000000 --- a/examples/schedule.jl +++ /dev/null @@ -1,92 +0,0 @@ -using Distributed - -if abspath(PROGRAM_FILE) == @__FILE__ - new_procs = addprocs(12) # Set the number of workers -end - -using Dagger -using Graphs -using MetaGraphs -using FrameworkDemo -using FrameworkDemo.ModGraphVizSimple # This is a workaround to make visualization work until the bugs are fixed in the package. - -# Defining constants -output_dir = "results" -graph1_path = "./data/demo/sequencer/df.graphml" -graph2_path = "./data/demo/parallel/df.graphml" - -LOGS_FILE = FrameworkDemo.timestamp_string("$output_dir/out") * ".dot" -GRAPH_IMAGE_PATH = FrameworkDemo.timestamp_string("$output_dir/DAG") * ".png" - -OUTPUT_GRAPH_PATH = "$output_dir/" -OUTPUT_GRAPH_IMAGE_PATH = "$output_dir/" - -MAX_GRAPHS_RUN = 3 - -function execution(graphs_map) - graphs_being_run = Set{Int}() - graphs_dict = Dict{Int, String}() - graphs_tasks = Dict{Int, Dagger.DTask}() - graphs = FrameworkDemo.parse_graphs(graphs_map, OUTPUT_GRAPH_PATH, - OUTPUT_GRAPH_IMAGE_PATH) - notifications = RemoteChannel(() -> Channel{Int}(32)) - # notifications = Channel{Int}(32) - coefficients = Dagger.@shard FrameworkDemo.calculate_coefficients() - - for (i, (g_name, g)) in enumerate(graphs) - graphs_dict[i] = g_name - while !(length(graphs_being_run) < MAX_GRAPHS_RUN) - finished_graph_id = take!(notifications) - delete!(graphs_being_run, finished_graph_id) - delete!(graphs_tasks, i) - println("Dispatcher: graph finished - $finished_graph_id: $(graphs_dict[finished_graph_id])") - end - graphs_tasks[i] = FrameworkDemo.schedule_graph_with_notify(g, notifications, g_name, - i, coefficients) - push!(graphs_being_run, i) - println("Dispatcher: scheduled graph $i: $g_name") - end - results = [] - for (g_name, g) in graphs - g_map = Dict{Int, Any}() - data_vertices = MetaGraphs.filter_vertices(g, :type, "DataObject") - for vertex_id in data_vertices - future = get_prop(g, vertex_id, :res_data) - g_map[vertex_id] = fetch(future) - end - push!(results, (g_name, g_map)) - end - for (g_name, res) in results - for (id, value) in res - println("Graph: $g_name, Final result for vertex $id: $value") - end - end - for (_, task) in graphs_tasks - wait(task) - end -end - -function main(graphs_map) - FrameworkDemo.configure_LocalEventLog() - - @time execution(graphs_map) - - ctx = Dagger.Sch.eager_context() - logs = Dagger.TimespanLogging.get_logs!(ctx) - open(LOGS_FILE, "w") do io - FrameworkDemo.ModGraphVizSimple.show_logs(io, logs, :graphviz_simple) - end - FrameworkDemo.dot_to_png(LOGS_FILE, GRAPH_IMAGE_PATH, 7000, 8000) # adjust picture size, if needed (optional param) -end - -graphs_map = Dict{String, String}("graph1" => graph1_path, - "graph2" => graph2_path, - "graph3" => graph1_path, - "graph4" => graph2_path) - -if abspath(PROGRAM_FILE) == @__FILE__ - mkpath(output_dir) - main(graphs_map) - rmprocs!(Dagger.Sch.eager_context(), workers()) - rmprocs(workers()) -end diff --git a/src/logging.jl b/src/logging.jl index cd4d973..8ad4ed4 100644 --- a/src/logging.jl +++ b/src/logging.jl @@ -1,103 +1,13 @@ using Dagger -using DaggerWebDash -using TimespanLogging -using Dates - -function configure_webdash_multievent() - ctx = Dagger.Sch.eager_context() - ml = Dagger.TimespanLogging.MultiEventLog() - - TimespanLogging = Dagger.TimespanLogging - ## Add some logging events of interest - - ml[:core] = TimespanLogging.Events.CoreMetrics() - ml[:id] = TimespanLogging.Events.IDMetrics() - ml[:timeline] = TimespanLogging.Events.TimelineMetrics() - # ... - - # (Optional) Enable profile flamegraph generation with ProfileSVG - ml[:profile] = DaggerWebDash.ProfileMetrics() - ctx.profile = true - - # Create a LogWindow; necessary for real-time event updates - lw = TimespanLogging.Events.LogWindow(20 * 10^9, :core) - ml.aggregators[:logwindow] = lw - - # Create the D3Renderer server on port 8080 - d3r = DaggerWebDash.D3Renderer(8080) - - ## Add some plots! Rendered top-down in order - - # Show an overview of all generated events as a Gantt chart - push!(d3r, DaggerWebDash.GanttPlot(:core, :id, :esat, :psat; title = "Overview")) - - # Show various numerical events as line plots over time - push!(d3r, DaggerWebDash.LinePlot(:core, :wsat, "Worker Saturation", "Running Tasks")) - push!(d3r, - DaggerWebDash.LinePlot(:core, :loadavg, "CPU Load Average", - "Average Running Threads")) - push!(d3r, DaggerWebDash.LinePlot(:core, :bytes, "Allocated Bytes", "Bytes")) - push!(d3r, DaggerWebDash.LinePlot(:core, :mem, "Available Memory", "% Free")) - - # Show a graph rendering of compute tasks and data movement between them - # Note: Profile events are ignored if absent from the log - push!(d3r, DaggerWebDash.GraphPlot(:core, :id, :timeline, :profile, "DAG")) - - # TODO: Not yet functional - #push!(d3r, DaggerWebDash.ProfileViewer(:core, :profile, "Profile Viewer")) - # Add the D3Renderer as a consumer of special events generated by LogWindow - push!(lw.creation_handlers, d3r) - push!(lw.deletion_handlers, d3r) - - # D3Renderer is also an aggregator - ml.aggregators[:d3r] = d3r - - ctx.log_sink = ml -end function configure_LocalEventLog() - TimespanLogging = Dagger.TimespanLogging ctx = Dagger.Sch.eager_context() - log = TimespanLogging.LocalEventLog() + log = Dagger.TimespanLogging.LocalEventLog() ctx.log_sink = log end -function timestamp_string(str) - dt = Dates.now() - timestamp = Dates.format(dt, "yyyy-mm-dd HH-MM-SS") - return str * " " * timestamp -end - -function fetch_LocalEventLog() - ctx = Dagger.Sch.eager_context() - logs = Dagger.TimespanLogging.get_logs!(ctx.log_sink) - # str = Dagger.show_plan() - doesn't work (exist) - return logs -end - -function my_show_plan(io::IO, logs::Vector{Dagger.TimespanLogging.Timespan}, t = nothing) - println(io, """strict digraph { - graph [layout=dot,rankdir=LR];""") - ModGraphVizSimple.write_dag(io, t, logs) - println(io, "}") -end - -function flush_logs_to_file(log_file) - open(log_file, "w") do io - my_show_plan(io, Dagger.fetch_logs!(), nothing) # Writes graph to a file - end -end - -function flush_logs_to_file(log_file, t::Thunk) - open(log_file, "w") do io - Dagger.show_logs(io, t, Dagger.fetch_logs!(), :graphviz_simple) # Writes graph to a file - end -end - -function save_logs(log_file, logs) - open(log_file, "w") do io - write(io, logs) - end +function fetch_logs!() + return Dagger.fetch_logs!() end function dispatch_begin_msg(index) diff --git a/src/parsing.jl b/src/parsing.jl index 5ad89de..6231ad5 100644 --- a/src/parsing.jl +++ b/src/parsing.jl @@ -3,30 +3,5 @@ using MetaGraphs include("../deps/GraphMLReader.jl/src/GraphMLReader.jl") function parse_graphml(filename::String)::MetaDiGraph - GraphMLReader.loadgraphml(filename, "G") -end - -function show_graph(G) - for v in Graphs.vertices(G) - println("Node: ") - print("Node type: ") - println(get_prop(G, v, :type)) - if has_prop(G, v, :class) - print("Node class: ") - println(get_prop(G, v, :class)) - end - if has_prop(G, v, :runtime_average_s) - print("Average runtime [s]: ") - println(get_prop(G, v, :runtime_average_s)) - end - if has_prop(G, v, :size_average_B) - print("Average size [B]: ") - println(get_prop(G, v, :size_average_B)) - end - print("Original name: ") - println(get_prop(G, v, :original_id)) - print("Node name: ") - println(get_prop(G, v, :node_id)) - println() - end + return GraphMLReader.loadgraphml(filename, "G") end diff --git a/src/visualization.jl b/src/visualization.jl index 489f5e7..a167c6b 100644 --- a/src/visualization.jl +++ b/src/visualization.jl @@ -1,15 +1,22 @@ -using GraphViz -using FileIO -using Cairo +import Colors +import GraphViz +import FileIO +import DataFrames +import Plots -function dot_to_png(in, out, width = 7000, height = 2000) - dot_code = read(in, String) - graph = GraphViz.load(IOBuffer(dot_code)) - GraphViz.layout!(graph) - - surface = Cairo.CairoSVGSurface(IOBuffer(), width, height) - context = Cairo.CairoContext(surface) - - GraphViz.render(context, graph) - write_to_png(surface, out) +function save_logs_dot(logs, path::String) + if splitext(path)[2] == ".dot" + open(path, "w") do io + ModGraphVizSimple.show_logs(io, logs, :graphviz_simple) + @info "Written logs dot graph to $path" + end + else + buffer = IOBuffer() + ModGraphVizSimple.show_logs(buffer, logs, :graphviz_simple) + dot = String(take!(buffer)) + graphviz = GraphViz.Graph(dot) + GraphViz.layout!(graphviz; engine = "dot") + FileIO.save(path, graphviz) + @info "Written logs graph to $path" + end end diff --git a/test/Aqua.jl b/test/Aqua.jl index a44accd..62481e8 100644 --- a/test/Aqua.jl +++ b/test/Aqua.jl @@ -7,7 +7,5 @@ using Aqua ambiguities = false, stale_deps = (; ignore = [:ArgParse, # bin/ - :BenchmarkTools, # benchmarks.jl - :Plots, # benchmarks.jl, Dagger ext - :DataFrames])) + :BenchmarkTools])) end