Skip to content

Commit

Permalink
Move local repo cleanup into worker shutdown.
Browse files Browse the repository at this point in the history
Making cleanup Gunicorn's responsibility ensures that the cleanup will
be run even if the worker is suddenly shut down. This prevents the
worker from having to check for old repos at startup, allowing multiple
workers to coexist in the same pod.
  • Loading branch information
kfindeisen committed Jul 26, 2024
1 parent f9018da commit 781e762
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 6 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
# Include these paths.
!python/activator/*.py
!pipelines
!config/gunicorn.conf.py
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ ARG PORT
WORKDIR $APP_HOME
COPY python/activator activator/
COPY pipelines pipelines/
COPY config/gunicorn.conf.py ./
CMD source /opt/lsst/software/stack/loadLSST.bash \
&& setup lsst_distrib \
&& exec gunicorn --workers 1 --threads 1 --timeout $WORKER_TIMEOUT --max-requests $WORKER_RESTART_FREQ \
--graceful-timeout $WORKER_GRACE_PERIOD \
--bind :$PORT 'activator.activator:create_app()'
--bind :$PORT --config gunicorn.conf.py 'activator.activator:create_app()'
5 changes: 5 additions & 0 deletions config/gunicorn.conf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from activator.activator import cleanup_worker


def worker_exit(server, worker):
cleanup_worker()
17 changes: 12 additions & 5 deletions python/activator/activator.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

__all__ = ["check_for_snap", "next_visit_handler"]
__all__ = ["check_for_snap", "next_visit_handler", "cleanup_worker"]

import collections.abc
import functools
Expand Down Expand Up @@ -151,10 +151,6 @@ def create_app():
_get_storage_client()
_get_central_butler()

for old_repo in find_local_repos(local_repos):
_log.warning("Orphaned repo found at %s, attempting to remove.", old_repo)
flush_local_repo(old_repo, _get_central_butler())

app = Flask(__name__)
app.add_url_rule("/next-visit", view_func=next_visit_handler, methods=["POST"])
app.register_error_handler(500, server_error)
Expand All @@ -167,6 +163,17 @@ def create_app():
sys.exit(3)


def cleanup_worker():
try:
local_repo = _get_local_repo()
_log.info("Removing local repo at %s.", local_repo.name)
flush_local_repo(local_repo.name, _get_central_butler())
local_repo.cleanup()
except FileNotFoundError:
# If the repo doesn't exist, there's no problem.
pass


def _graceful_shutdown(signum: int, stack_frame):
"""Signal handler for cases where the service should gracefully shut down.
Expand Down

0 comments on commit 781e762

Please sign in to comment.