Update microbenchmarks

Experiments/Scripts/coloring_methods.jl

@@ -11,7 +11,6 @@ partitioning_schemes = [
                         [(Hash, 64)],
                         [(Degree, 8), (QuasiStable, 32), (NeighborNodeLabels, 24)],
                         [(Degree, 8), (NeighborNodeLabels, 24), (QuasiStable, 32)]]
-partitioning_schemes = [[(QuasiStable, 64)]]
 experiment_params = Vector{ExperimentParams}()
 for dataset in datasets
     for scheme in partitioning_schemes

Experiments/Scripts/comparison_exps.jl

@@ -4,6 +4,7 @@ include("../Experiments.jl")
 
 #datasets = [human, aids, lubm80, yeast, hprd, dblp, youtube, eu2005, patents, wordnet]
 datasets = [human, aids, lubm80, yeast, dblp, youtube, eu2005, patents]
+#datasets = [aids]
 
 experiment_params = Vector{ExperimentParams}()
 for dataset in datasets
@@ -26,7 +27,7 @@ for dataset in datasets
     push!(experiment_params, ExperimentParams(deg_stats_type=MaxDegStats,
                                                 dataset=dataset,
                                                 partitioning_scheme=[(Hash, 64)],
-                                                description = "MaxH64"))
+                                                description = "BSK"))
 
     push!(experiment_params, ExperimentParams(deg_stats_type=AvgDegStats,
                                                 dataset=dataset,
@@ -35,7 +36,7 @@ for dataset in datasets
                                                 description = "IndEst"))
 end
 
-#build_experiments(experiment_params)
+build_experiments(experiment_params)
 
 #run_estimation_experiments(experiment_params)
 
@@ -51,28 +52,40 @@ graph_grouped_boxplot_with_comparison_methods(experiment_params;
 
 graph_grouped_boxplot_with_comparison_methods(experiment_params;
                                                 ylims=[10^-21, 10^21],
-                                                y_ticks=[10^-20, 10^-15, 10^-5, 10^-2, 10^0, 10^2, 10^5, 10^10, 10^15, 10^20],
+                                                y_ticks=[10^-20, 10^-15, 10^-10, 10^-5, 10^-2, 10^0, 10^2, 10^5, 10^10, 10^15, 10^20],
                                                 y_type = estimate_error,
                                                 grouping=description,
                                                 dimensions = (1450, 550),
                                                 legend_pos=:bottomleft,
                                                 y_label="Relative Error (10^)",
                                                 filename="error")
 
+
+graph_grouped_boxplot_with_comparison_methods(experiment_params;
+                                                ylims=[10^-21, 10^21],
+                                                x_type = query_size,
+                                                y_ticks=[10^-20, 10^-15, 10^-10, 10^-5, 10^-2, 10^0, 10^2, 10^5, 10^10, 10^15, 10^20],
+                                                y_type = estimate_error,
+                                                grouping=description,
+                                                dimensions = (1450, 550),
+                                                legend_pos=:bottomleft,
+                                                y_label="Relative Error (10^)",
+                                                filename="error-query-size")
+
 graph_grouped_bar_plot(experiment_params;
                         grouping=description,
                         y_type=memory_footprint,
-                        y_lims=[0, 30],
+                        ylims=[0, 30],
                         y_ticks = [5, 10, 15, 20, 25, 30],
-                        dimensions = (900, 550),
+                        dimensions = (1000, 550),
                         y_label="Memory (MBs)",
                         filename="memory")
 
 graph_grouped_bar_plot(experiment_params;
                         grouping=description,
                         y_type=build_time,
-                        y_lims=[0, 720],
-                        y_ticks = [0, 100, 200, 300, 400, 500, 600, 700],
-                        dimensions = (900, 550),
+                        ylims=[0, 100],
+                        y_ticks = [20, 40, 60, 80, 100],
+                        dimensions = (1000, 550),
                         y_label="Build Time (s)",
                         filename="build_time")

Experiments/Scripts/just-edge-updates-experiments.jl

@@ -3,19 +3,19 @@ using Graphs
 using Random
 include("../Experiments.jl")
 
-datasets::Vector{DATASET} = [yeast]
+datasets = [aids]
 # datasets::Vector{DATASET} = [aids, human, yeast, wordnet, youtube, dblp, patents]
 # datasets::Vector{DATASET} = [aids, human, lubm80, yeast, hprd, wordnet, dblp, youtube, eu2005, patents]
 max_cycles = 6
-proportions_updated = [0, 0.2, 0.4, 0.6, 0.8, 1.0]
+proportions_updated = [0, 0.2, 0.4, 0.6, 0.8, .9, 1.0]
 # To test deletion, we will add a random node / edge and then delete them...
 # proportion_not_updated = 0.5
 
 experiment_params_list::Vector{ExperimentParams} = [ExperimentParams(dataset=current_dataset, proportion_updated=current_proportion)
                                                     for current_dataset in datasets for current_proportion in proportions_updated]
 println("started building")
 shuffled_edges = Dict()
-#= for experiment_params in experiment_params_list
+for experiment_params in experiment_params_list
     build_times = [("Dataset", "Partitioner", "NumColors", "BuildPhase", "BuildTime", "MemoryFootprint")]
     dataset = experiment_params.dataset
     summary_params = experiment_params.summary_params
@@ -85,16 +85,16 @@ shuffled_edges = Dict()
     results_filename = params_to_results_filename(experiment_params)
     result_file_location = "Experiments/Results/Build_" * results_filename
     writedlm(result_file_location, build_times, ",")
-end =#
+end
 
 println("started estimating")
-#run_estimation_experiments(experiment_params_list)
+run_estimation_experiments(experiment_params_list)
 println("started graphing")
 # compare how overall accuracy is affected by summary updates
 # graph_grouped_box_plot(experiment_params_list, x_type=dataset, y_type=estimate_error, grouping=proportion_not_updated, filename="overall-accuracy-and-updates")
 # compare how cycle stat accuracies are affected by summary updates
 # graph_grouped_box_plot(experiment_params_list, x_type=proportion_deleted, y_type=estimate_error, x_label="proportion added then deleted", y_label="accuracy", grouping=cycle_size, filename="deletion-experiment")
-graph_grouped_bar_plot(experiment_params_list, x_type=dataset, y_type=build_time, y_lims=[0, 1], y_ticks = [0,.2, .4, .6, .8, 1], x_label="Proportion Updated", y_label="Build Time (S)", grouping=proportion_updated, filename="just-edge-updates-build")
-graph_grouped_box_plot(experiment_params_list, x_type=dataset, y_type=estimate_error,y_lims=[10^-20, 10^15],  x_label="Proportion Updated", y_label="Estimate Error", grouping=proportion_updated, filename="just-edge-updates-error")
+graph_grouped_bar_plot(experiment_params_list, x_type=dataset, y_type=build_time, y_lims=[0, 10], y_ticks = [0, 2, 4 ,6 ,8, 10], legend_pos=:topright, x_label="Proportion Updated", y_label="Build Time (S)", grouping=proportion_updated, filename="just-edge-updates-build")
+graph_grouped_box_plot(experiment_params_list, x_type=dataset, y_type=estimate_error,y_lims=[-20, 15],  x_label="Proportion Updated", y_label="Estimate Error", grouping=proportion_updated, filename="just-edge-updates-error")
 graph_grouped_box_plot(experiment_params_list, x_type=dataset, y_type=runtime, y_lims=[10^-5, 10], y_ticks = [10^-5, 10^-4, 10^-3, 10^-2, 10^-1, 1, 10], x_label="Proportion Updated", y_label="Runtime (S)", grouping=proportion_updated, filename="just-edge-updates-runtime")
 graph_grouped_bar_plot(experiment_params_list, x_type=dataset, y_type=memory_footprint, y_lims=[0, 20], y_ticks = [0, 5, 10, 15, 20], x_label="Proportion Updated", y_label="Memory (MB)", grouping=proportion_updated, filename="just-edge-updates-memory")

Experiments/Scripts/num_colors_exps.jl

@@ -0,0 +1,54 @@
+
+using Profile
+include("../Experiments.jl")
+
+datasets = [yeast]
+partitioning_schemes = [
+    [(QuasiStable, 1)],
+    [(QuasiStable, 4)],
+    [(QuasiStable, 16)],
+    [(QuasiStable, 32)],
+    [(QuasiStable, 64)],
+    [(QuasiStable, 128)],
+    [(QuasiStable, 256)],
+    [(QuasiStable, 512)],
+]
+experiment_params = Vector{ExperimentParams}()
+for dataset in datasets
+    for scheme in partitioning_schemes
+            push!(experiment_params, ExperimentParams(dataset=dataset, partitioning_scheme=scheme))
+    end
+end
+
+#build_experiments(experiment_params)
+
+#run_estimation_experiments(experiment_params)
+
+graph_grouped_box_plot(experiment_params;
+                        ylims=[10^-20, 10^15],
+                        y_ticks =[10^-20,10^-15, 10^-10, 10^-5, 1, 10^5, 10^10, 10^15],
+                        grouping=technique,
+                        y_label="Relative Error 10^ (s)",
+                        dimensions = (1000, 600),
+                        legend_pos = :topright,
+                        filename="num_colors_error")
+
+graph_grouped_box_plot(experiment_params;
+                        ylims=[10^-4.5, 10^1.5],
+                        y_ticks =[10^-4, 10^-3, 10^-2, 10^-1, 1, 10],
+                        grouping=technique,
+                        y_type=runtime,
+                        y_label="Runtime 10^ (s)",
+                        dimensions = (1000, 600),
+                        legend_pos = :topleft,
+                        filename="num_colors_runtime")
+
+graph_grouped_bar_plot(experiment_params,
+                        y_type=memory_footprint,
+                        ylims=[0, 30],
+                        y_ticks =[5 ,10, 15, 20, 25, 30],
+                        y_label="Memory Footprint (MB)",
+                        dimensions = (600, 400),
+                        grouping=technique,
+                        legend_pos = :topleft,
+                        filename="num_colors_memory")

Experiments/Scripts/run-cycle-experiments.jl

@@ -2,16 +2,51 @@ using Plots.PlotMeasures
 include("../Experiments.jl")
 
 # datasets::Vector{DATASET} = [aids, human, lubm80, yago, yeast, hprd, wordnet, dblp, youtube, eu2005, patents]
-datasets::Vector{DATASET} = [aids]
+data = [yeast]
 max_cycles = 6
-experiment_params_list::Vector{ExperimentParams} = [ExperimentParams(dataset=current_dataset, partitioner=QuasiStable, max_cycle_size=current_size) for current_dataset in datasets for current_size in 2:max_cycles]
+experiment_params_list::Vector{ExperimentParams} = [ExperimentParams(dataset=current_dataset, max_cycle_size=current_size) for current_dataset in data for current_size in 2:max_cycles]
 
 # println("started building")
-# build_experiments(experiment_params_list)
+#build_experiments(experiment_params_list)
 # println("started estimating")
-# run_estimation_experiments(experiment_params_list)
+#run_estimation_experiments(experiment_params_list)
 println("started graphing")
-graph_grouped_bar_plot(experiment_params_list, x_type=cycle_size, y_type=build_time,  x_label="Size of Cycles for Stored Probability", y_label="Build Time", grouping=dataset, filename="cycle-build-aids")
-graph_grouped_box_plot(experiment_params_list, x_type=cycle_size, y_type=estimate_error, x_label="Size of Cycles for Stored Probability", y_label="Estimate Error", grouping=dataset, filename="cycle-error-aids")
-graph_grouped_bar_plot(experiment_params_list, x_type=cycle_size, y_type=runtime, y_lims=[0, 0.02], x_label="Size of Cycles for Stored Probability", y_label="Runtime", grouping=dataset, filename="cycle-runtime-aids")
-graph_grouped_bar_plot(experiment_params_list, x_type=cycle_size, y_type=memory_footprint, y_lims=[0, 4], x_label="Size of Cycles for Stored Probability", y_label="Memory Footprint", grouping=dataset, filename="cycle-memory-aids")
+
+graph_grouped_box_plot(experiment_params_list;
+                        x_type=dataset,
+                        ylims=[10^-20, 10^15],
+                        y_ticks =[10^-20,10^-15, 10^-10, 10^-5, 1, 10^5, 10^10, 10^15],
+                        grouping=cycle_size,
+                        y_type=estimate_error,
+                        y_label="Relative Error 10^ (s)",
+                        filename="cycles_size_error")
+
+graph_grouped_box_plot(experiment_params_list;
+                        x_type=dataset,
+                        ylims=[10^-3, 10^1],
+                        y_ticks =[10^-3, 10^-2, 10^-1, 1, 10],
+                        grouping=cycle_size,
+                        y_type=runtime,
+                        y_label="Seconds 10^ (s)",
+                        filename="cycles_size_runtime")
+
+graph_grouped_bar_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=memory_footprint,
+                        ylims=[0, 15.5],
+                        y_ticks =[3, 6, 9, 12, 15],
+                        dimensions = (600, 400),
+                        y_label="Memory Footprint (MB)",
+                        grouping=cycle_size,
+                        filename="cycles_size_memory")
+
+graph_grouped_bar_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=build_time,
+                        ylims = [0, 4],
+                        y_ticks = [1, 2, 3, 4],
+                        dimensions = (600, 400),
+                        legend_pos = :topright,
+                        y_label="Build Time (s)",
+                        grouping=cycle_size,
+                        filename="cycles_size_build_time")

Experiments/Scripts/run-inference-sampling-experiments.jl

@@ -2,17 +2,34 @@ using Plots.PlotMeasures
 include("../Experiments.jl")
 
 # datasets::Vector{DATASET} = [aids, human, lubm80, yago, yeast, hprd, wordnet, dblp, youtube, eu2005, patents]
-datasets::Vector{DATASET} = [aids]
-max_paths = 100
-experiment_params_list::Vector{ExperimentParams} = [ExperimentParams(dataset=current_dataset, partitioner=QuasiStable, inference_max_paths=current_paths) for current_dataset in datasets for current_paths in 20:20:max_paths]
-
+data::Vector{DATASET} = [yeast]
+max_paths = [10, 100, 1000, 10000]
+experiment_params_list::Vector{ExperimentParams} = [ExperimentParams(dataset=current_dataset, inference_max_paths=current_paths) for current_dataset in data for current_paths in max_paths]
 println("started building")
-build_experiments(experiment_params_list)
+#build_experiments(experiment_params_list)
 println("started estimating")
-run_estimation_experiments(experiment_params_list)
+#run_estimation_experiments(experiment_params_list)
 println("started graphing")
-# graph_grouped_box_plot(experiment_params_list, x_type=dataset, y_type=estimate_error, x_label="dataset", y_label="accuracy", grouping=inference_paths, filename="inferencesampling")
-graph_grouped_bar_plot(experiment_params_list, x_type=inference_paths, y_type=build_time, x_label="Maximum Inference Paths", y_label="Build Time", grouping=dataset, filename="inference-build-aids")
-graph_grouped_box_plot(experiment_params_list, x_type=inference_paths, y_type=estimate_error, x_label="Maximum Inference Paths", y_label="Estimate Error", grouping=dataset, filename="inference-error-aids")
-graph_grouped_bar_plot(experiment_params_list, x_type=inference_paths, y_type=runtime, y_lims=[0, 0.02], x_label="Maximum Inference Paths", y_label="Runtime", grouping=dataset, filename="inference-runtime-aids")
-graph_grouped_bar_plot(experiment_params_list, x_type=inference_paths, y_type=memory_footprint, y_lims=[0, 4], x_label="Maximum Inference Paths", y_label="Memory Footprint", grouping=dataset, filename="inference-memory-aids")
+graph_grouped_box_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=estimate_error,
+                        ylims=[10^-20, 10^20],
+                        y_ticks=[10^-20, 10^-15, 10^-10, 10^-5, 1, 10^5, 10^10, 10^15],
+                        dimensions = (600, 550),
+                        legend_pos = :topleft,
+                        x_label="Maximum Inference Paths",
+                        y_label="Estimate Error 10^",
+                        grouping=inference_paths,
+                        filename="inference-paths-error")
+
+graph_grouped_box_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=runtime,
+                        ylims=[.0001, 100],
+                        y_ticks=[.001, .01, .1, 1, 10, 100],
+                        dimensions = (600, 550),
+                        legend_pos = :topleft,
+                        x_label="Maximum Inference Paths",
+                        y_label="Runtime 10^ (s)",
+                        grouping=inference_paths,
+                        filename="inference-paths-runtime")

Experiments/Scripts/run-summary-sampling-experiments.jl

@@ -3,19 +3,49 @@ include("../Experiments.jl")
 
 
 # datasets::Vector{DATASET} = [aids, human, lubm80, yago, yeast, hprd, wordnet, dblp, youtube, eu2005, patents]
-datasets::Vector{DATASET} = [aids, human, dblp]
+datasets::Vector{DATASET} = [yeast]
 # max_paths = [32, 64, 128, 256, 512, 1024, 2048, 20000, 100000]
-max_paths = [32, 64, 128, 256, 512, 1024, 2048]
+max_paths = [64, 256, 1024, 4096, 8192, 8192*4, 8192*4*4]
 
 experiment_params_list::Vector{ExperimentParams} = [ExperimentParams(dataset=current_dataset, summary_max_paths=current_paths) for current_dataset in datasets for current_paths in max_paths]
 
 println("started building")
-build_experiments(experiment_params_list)
+#build_experiments(experiment_params_list)
 println("started estimating")
-run_estimation_experiments(experiment_params_list)
+#run_estimation_experiments(experiment_params_list)
 println("started graphing")
-# graph_grouped_box_plot(experiment_params_list, x_type=dataset, y_type=estimate_error, x_label="dataset", y_label="accuracy", grouping=summary_paths, filename="summarysamples")
-graph_grouped_bar_plot(experiment_params_list, x_type=dataset, y_type=build_time, y_lims=[0, 100], x_label="Maximum Paths During Summary Building", y_label="Build Time", grouping=summary_paths, filename="summary-build-multi")
-graph_grouped_box_plot(experiment_params_list, x_type=dataset, y_type=estimate_error, x_label="Maximum Paths During Summary Building", y_label="Estimate Error", grouping=summary_paths, filename="summary-error-multi")
-graph_grouped_bar_plot(experiment_params_list, x_type=dataset, y_type=runtime, y_lims=[0, 0.5], x_label="Maximum Paths During Summary Building", y_label="Runtime", grouping=summary_paths, filename="summary-runtime-multi")
-graph_grouped_bar_plot(experiment_params_list, x_type=dataset, y_type=memory_footprint, y_lims=[0, 20], x_label="Maximum Paths During Summary Building", y_label="Memory Footprint", grouping=summary_paths, filename="summary-memory-multi")
+
+graph_grouped_bar_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=build_time,
+                        ylims=[0, 200],
+                        y_ticks=[50, 100, 150, 200],
+                        y_label="Build Time",
+                        grouping=summary_paths,
+                        filename="summary-build-multi")
+
+graph_grouped_box_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=estimate_error,
+                        ylims=[10^-20, 10^15],
+                        y_ticks =[10^-20, 10^-15, 10^-10, 10^-5, 1, 10^5, 10^10, 10^15],
+                        y_label="Estimate Error", grouping=summary_paths,
+                        filename="summary-error-multi")
+
+graph_grouped_box_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=runtime,
+                        ylims=[.0001, 10],
+                        y_ticks =[.0001, .001, .01, 1, 10],
+                        y_label="Runtime",
+                        grouping=summary_paths,
+                        filename="summary-runtime-multi")
+
+graph_grouped_bar_plot(experiment_params_list,
+                        x_type=dataset,
+                        y_type=memory_footprint,
+                        ylims=[0, 30],
+                        y_ticks =[5 ,10, 15, 20, 25, 30],
+                        y_label="Memory Footprint",
+                        grouping=summary_paths,
+                        filename="summary-memory-multi")