Skip to content

Commit

Permalink
chore: use variable
Browse files Browse the repository at this point in the history
  • Loading branch information
njuguna-n committed Oct 4, 2024
1 parent a520e30 commit 8bd222b
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions dbt/dbt-run.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ def get_max_timestamp():
def run_dbt_in_batches():
last_processed_timestamp = get_last_processed_timestamp()
batch_size = int(os.getenv("DBT_BATCH_SIZE") or 10000)
dataemon_interval = int(os.getenv("DATAEMON_INTERVAL") or 5)

while True:
update_dbt_deps()
Expand All @@ -202,14 +203,14 @@ def run_dbt_in_batches():
if result.returncode != 0:
print("Error running dbt")
update_batch_status(last_processed_timestamp, "error")
time.sleep(int(os.getenv("DATAEMON_INTERVAL") or 5))
time.sleep(dataemon_interval)
continue

update_batch_status(last_processed_timestamp, "success")
max_timestamp = get_max_timestamp()

if max_timestamp == last_processed_timestamp:
time.sleep(int(os.getenv("DATAEMON_INTERVAL") or 5))
time.sleep(dataemon_interval)
continue

last_processed_timestamp = max_timestamp
Expand Down

0 comments on commit 8bd222b

Please sign in to comment.