test

uche-madu · Oct 24, 2023 · bd5ab6b · bd5ab6b
1 parent abde0d4
commit bd5ab6b
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 2 deletions.
diff --git a/dags/user_analytics.py b/dags/user_analytics.py
@@ -237,11 +237,34 @@ def dataproc_tasks() -> None:
         and deletion of the Dataproc cluster.
         """
 
+
+        # VIRTUAL_CLUSTER_CONFIG = {
+        #     "kubernetes_cluster_config": {
+        #         "gke_cluster_config": {
+        #             "gke_cluster_target": f"projects/{PROJECT_ID}/locations/{REGION}/clusters/{CLUSTER_NAME}",
+        #             "node_pool_target": [
+        #                 {
+        #                     "node_pool": f"projects/{PROJECT_ID}/locations/{REGION}/clusters/{CLUSTER_NAME}/nodePools/dp",  # noqa
+        #                     "roles": ["DEFAULT"],
+        #                     "node_pool_config": {
+        #                         "config": {
+        #                             "preemptible": True,
+        #                         }
+        #                     },
+        #                 }
+        #             ],
+        #         },
+        #         "kubernetes_software_config": {"component_version": {"SPARK": b"3"}},
+        #     },
+        #     "staging_bucket": "test-staging-bucket",
+        # }
+
+
         CLUSTER_GENERATOR_CONFIG = ClusterGenerator(
             project_id=PROJECT_ID,
             master_machine_type="n2-standard-4",
             master_disk_size=32,
-            worker_machine_type="n2-standard-2",
+            worker_machine_type="n2-standard-4",
             worker_disk_size=32,
             num_workers=2,
             storage_bucket=BUCKET_NAME,

diff --git a/requirements.txt b/requirements.txt
@@ -2,4 +2,4 @@ apache-airflow-providers-google==10.9.0
 apache-airflow-providers-postgres==5.6.1
 astronomer-cosmos[dbt-bigquery]==1.1.1
 airflow_provider_great_expectations==0.2.6
-google-cloud-secret-manager==2.16.4
+# google-cloud-secret-manager==2.16.4