From eaed585c55e5f242fdc397e62dbc9e068b033af4 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 29 Apr 2024 16:32:28 +0900 Subject: [PATCH] [SPARK-48002][PYTHON][SS][TESTS] Adds sleep before event testing after query termination ### What changes were proposed in this pull request? This PR is a followup of https://github.com/apache/spark/pull/46237 that makes to wait 5 secs after the query termination to make sure the events arrive. ### Why are the changes needed? To deflake the test. It's flaky (https://github.com/apache/spark/actions/runs/8873809388/job/24360221027) ### Does this PR introduce _any_ user-facing change? No, test-only. ### How was this patch tested? Manually. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #46275 from HyukjinKwon/SPARK-48002-followup. Authored-by: Hyukjin Kwon Signed-off-by: Hyukjin Kwon --- python/pyspark/sql/tests/streaming/test_streaming_listener.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyspark/sql/tests/streaming/test_streaming_listener.py b/python/pyspark/sql/tests/streaming/test_streaming_listener.py index 9e4325e3c6ab4..15f5575d36479 100644 --- a/python/pyspark/sql/tests/streaming/test_streaming_listener.py +++ b/python/pyspark/sql/tests/streaming/test_streaming_listener.py @@ -232,6 +232,8 @@ def onQueryTerminated(self, event): while q.lastProgress is None or q.lastProgress["batchId"] == 0: q.awaitTermination(0.5) + time.sleep(5) + self.assertTrue(error_listener.num_rows > 0) self.assertTrue(error_listener.num_error_rows > 0)