hamsterster.yml

tag: dev  # Required
#datastore:  # All properties required
#  - config:
#      hostname: !ENV ${MYSQL_HOST}
#      username: !ENV ${MYSQL_USERNAME}
#      password: !ENV ${MYSQL_PASSWORD}
#      db_name: !ENV ${MYSQL_DB_NAME}
#      port: 3306
#    type: mysql
spark:
  - config:
      spark.master: local[*]  # Required
      spark.submit.deployMode: client  # Required
      spark_warehouse_folder: data/spark-warehouse  # Required
      spark.ui.port: 4040
      spark.driver.cores: 5
      spark.driver.memory: 8g
      spark.driver.memoryOverhead: 4096
      spark.driver.maxResultSize: 0
      spark.executor.instances: 2
      spark.executor.cores: 3
      spark.executor.memory: 4g
      spark.executor.memoryOverhead: 4096
      spark.sql.broadcastTimeout: 3600
      spark.sql.autoBroadcastJoinThreshold: -1
      spark.sql.shuffle.partitions: 4
      spark.default.parallelism: 4
      spark.network.timeout: 3600s
    dirs:
      df_data_folder: data/dataframes
      spark_warehouse_folder: data/spark-warehouse
      checkpoints_folder: data/checkpoints
      communities_csv_folder: data/csv_data
input:  # All properties required
  - config:
      name: Hamsterster
      nodes:
        path: data/input_graphs/Hamsterster/nodes
        has_header: true
        delimiter: '|'
        encoding: ISO-8859-1
        feature_names:  # You can set custom names here
          - id
          - name
          - joined
          - species
          - coloring
          - gender
          - birthday
          - age
          - hometown
          - favorite_toy
          - favorite_activity
          - favorite_foo
      edges:
        path: data/input_graphs/Hamsterster/edges
        has_header: true
        has_weights: false
        delimiter: ' '
    type: local
run_options:  # All properties required
  - config:
      feature_min_avg: 0.33
      r_lvl1_thres: 0.50
      r_lvl2_thres: 0.85
      max_edge_weight: 0.50
      betweenness_thres: 10
      max_sp_length: 2
      min_comp_size: 100
      max_steps: 5000
      features_to_check:
        - id
        - species
        - coloring
        - hometown
output:  # All properties required
  - config:
      logs_folder: data/logs
      plots_folder: data/plots
      cached_init_step: false
      save_communities_to_csvs: false
      plot_steps:
#        - 0
        -
          -
        - -1  # The Last step