From bd5ab6b2a0feff691353dfbe1247e6ae735b33cf Mon Sep 17 00:00:00 2001
From: Uche Madu <dreemer6@yahoo.com>
Date: Tue, 24 Oct 2023 01:37:59 +0100
Subject: [PATCH] test

---
 dags/user_analytics.py | 25 ++++++++++++++++++++++++-
 requirements.txt       |  2 +-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/dags/user_analytics.py b/dags/user_analytics.py
index bbd9d48..d5b4ceb 100644
--- a/dags/user_analytics.py
+++ b/dags/user_analytics.py
@@ -237,11 +237,34 @@ def dataproc_tasks() -> None:
         and deletion of the Dataproc cluster.
         """
 
+        
+        # VIRTUAL_CLUSTER_CONFIG = {
+        #     "kubernetes_cluster_config": {
+        #         "gke_cluster_config": {
+        #             "gke_cluster_target": f"projects/{PROJECT_ID}/locations/{REGION}/clusters/{CLUSTER_NAME}",
+        #             "node_pool_target": [
+        #                 {
+        #                     "node_pool": f"projects/{PROJECT_ID}/locations/{REGION}/clusters/{CLUSTER_NAME}/nodePools/dp",  # noqa
+        #                     "roles": ["DEFAULT"],
+        #                     "node_pool_config": {
+        #                         "config": {
+        #                             "preemptible": True,
+        #                         }
+        #                     },
+        #                 }
+        #             ],
+        #         },
+        #         "kubernetes_software_config": {"component_version": {"SPARK": b"3"}},
+        #     },
+        #     "staging_bucket": "test-staging-bucket",
+        # }
+
+
         CLUSTER_GENERATOR_CONFIG = ClusterGenerator(
             project_id=PROJECT_ID,
             master_machine_type="n2-standard-4",
             master_disk_size=32,
-            worker_machine_type="n2-standard-2",
+            worker_machine_type="n2-standard-4",
             worker_disk_size=32,
             num_workers=2,
             storage_bucket=BUCKET_NAME,
diff --git a/requirements.txt b/requirements.txt
index a371fce..9b19e4c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ apache-airflow-providers-google==10.9.0
 apache-airflow-providers-postgres==5.6.1
 astronomer-cosmos[dbt-bigquery]==1.1.1
 airflow_provider_great_expectations==0.2.6
-google-cloud-secret-manager==2.16.4
+# google-cloud-secret-manager==2.16.4