From 513c09ad44ce3bd396347c86118843b458d5bb0f Mon Sep 17 00:00:00 2001 From: Vince Salvino Date: Fri, 23 Aug 2024 19:08:35 -0400 Subject: [PATCH 1/3] Check version control status, and common Django errors, before deploying. --- cr/__init__.py | 28 ++++++++++ cr/api.py | 134 +++++++++++++++++++++++++++++++++++++++++------ cr/cli.py | 36 +++++++++++-- cr/rich_utils.py | 29 +++++++++- cr/utils.py | 113 +++++++++++++++++++++++++++++++++------ 5 files changed, 302 insertions(+), 38 deletions(-) diff --git a/cr/__init__.py b/cr/__init__.py index e19b261..67f6294 100644 --- a/cr/__init__.py +++ b/cr/__init__.py @@ -15,6 +15,34 @@ USER_AGENT = f"CodeRed-CLI/{VERSION} ({DOCS_LINK})" +PROD_BRANCHES = [ + "live", + "main", + "master", + "prod", + "production", + "release", +] + + +STAGING_BRANCHES = [ + "dev", + "develop", + "development", + "gold", + "pre-prod", + "pre-production", + "pre-release", + "preprod", + "preproduction", + "prerelease", + "stage", + "staging", + "test", + "testing", +] + + class ConfigurationError(Exception): """ Raised when project does not match expected configuration. diff --git a/cr/api.py b/cr/api.py index 41fe02d..2e03668 100644 --- a/cr/api.py +++ b/cr/api.py @@ -19,12 +19,13 @@ from urllib.request import urlopen import certifi -from rich.console import Console from rich.panel import Panel from rich.progress import Progress from cr import DOCS_LINK from cr import LOGGER +from cr import PROD_BRANCHES +from cr import STAGING_BRANCHES from cr import USER_AGENT from cr import VERSION from cr import AppType @@ -32,12 +33,18 @@ from cr import DatabaseType from cr import Env from cr import UserCancelError +from cr.rich_utils import Console from cr.utils import django_manage_check from cr.utils import django_requirements_check +from cr.utils import django_run_check +from cr.utils import django_run_migratecheck from cr.utils import django_settings_check from cr.utils import django_settings_fix from cr.utils import django_wsgi_check from cr.utils import django_wsgi_find +from cr.utils import git_branch +from cr.utils import git_uncommitted +from cr.utils import git_unpushed from cr.utils import html_index_check from cr.utils import wagtail_settings_fix from cr.utils import wordpress_wpconfig_check @@ -108,7 +115,7 @@ def database(self) -> DatabaseServer: """ return getattr(self, f"{self.env}_dbserver") - def local_check_path(self, p: Path, c: Optional[Console]) -> None: + def local_check(self, p: Path, c: Optional[Console]) -> None: """ Check that provided Path ``p`` appears to contain a valid AppType project. If Console ``c`` is provided, ask the user for input to @@ -207,13 +214,10 @@ def local_check_django(self, p: Path, c: Optional[Console] = None) -> None: # If settings file is misconfigured, offer to fix it. except ConfigurationError: LOGGER.warning("Settings file may be misconfigured. %s", settings) - if ( - c - and "y" - == c.input( - f"Settings file `{settings_rel}` may be misconfigured. " - "Correct it? [prompt.choices](y/N)[/] " - ).lower() + if c and c.prompt_yn( + f"Settings file `{settings_rel}` may be misconfigured. " + "Correct it?", + nouser=False, ): fix_me = True if fix_me: @@ -235,6 +239,105 @@ def local_check_wordpress( except FileNotFoundError as err: _prompt_filenotfound(err, c) + def local_predeploy(self, p: Path, c: Optional[Console]) -> None: + """ + Runs various common checks before a deployment, to help prevent the + user from creating a broken deployment or deviating from the recommended + development process. + """ + + # Check git branch. + b = git_branch() + if b and self.env == Env.STAGING and b not in STAGING_BRANCHES: + if c and not c.prompt_yn( + f"Your are deploying to STAGING from the `{b}` branch!\n" + "It is recommended to deploy from a dedicated staging branch.\n" + "Continue anyways?", + nouser=True, + ): + raise UserCancelError() + if b and self.env == Env.PROD and b not in PROD_BRANCHES: + if c and not c.prompt_yn( + f"Your are deploying to PROD from the `{b}` branch!\n" + "It is recommended to deploy from a dedicated production branch.\n" + "Continue anyways?", + nouser=True, + ): + raise UserCancelError() + + # Check for un-commited changes. + if git_uncommitted(): + if c and not c.prompt_yn( + "You have changes which are NOT COMMITTED to git!\n" + "It is recommended to commit and push your changes before deploying.\n" + "Continue anyways?", + nouser=True, + ): + raise UserCancelError() + + # Check for un-pushed. + if git_unpushed(): + if c and not c.prompt_yn( + "You have changes which are NOT PUSHED to git!\n" + "It is recommended to push your changes before deploying.\n" + "Continue anyways?", + nouser=True, + ): + raise UserCancelError() + + if self.app_type in [ + AppType.CODEREDCMS, + AppType.DJANGO, + AppType.WAGTAIL, + ]: + self.local_predeploy_django(p, c) + + def local_predeploy_django(self, p: Path, c: Optional[Console]) -> None: + """ + Runs Django-specific checks before a deployment, to help prevent the + user from creating a broken deployment. + """ + if c: + c.print("Checking Django project for potential problems...", end="") + try: + ok, output = django_run_check(p) + except FileNotFoundError: + ok = False + output = "Could not find python on this system." + if c and ok: + c.print(" [cr.success]OK[/]") + if c and not ok: + c.print(" [cr.fail]FAIL[/]") + if not c.prompt_yn( + "Django check returned the following errors:\n\n" + f"{output}\n\n" + "TIP: be sure to activate your virtual environment and install requirements.txt.\n" + "Continue anyways?", + nouser=True, + ): + raise UserCancelError() + + # Check migrations. + if c: + c.print("Checking for missing migrations...", end="") + try: + ok, output = django_run_migratecheck(p) + except FileNotFoundError: + ok = False + output = "Could not find python on this system." + if c and ok: + c.print(" [cr.success]OK[/]") + if c and not ok: + c.print(" [cr.fail]FAIL[/]") + if not c.prompt_yn( + f"\n{output}\n\n" + "TIP: did you forget to run `manage.py makemigrations`?\n" + "TIP: be sure to activate your virtual environment and install requirements.txt.\n" + "Continue anyways?", + nouser=True, + ): + raise UserCancelError() + def api_set_django_project(self, name: str) -> None: """ PATCH the webapp on coderedapi and set the local django_project. @@ -512,13 +615,10 @@ def _prompt_filenotfound( found. """ LOGGER.warning("%s file does not exist!", err) - if ( - c - and "y" - != c.input( - f"Missing `{err}` file. " - "Without this your app will not deploy correctly. " - "Continue anyways? [prompt.choices](y/N)[/] ", - ).lower() + if c and not c.prompt_yn( + f"Missing `{err}` file.\n" + "Without this your app will not deploy correctly. " + "Continue anyways?", + nouser=False, ): raise UserCancelError() diff --git a/cr/cli.py b/cr/cli.py index b8d3924..83538d3 100644 --- a/cr/cli.py +++ b/cr/cli.py @@ -241,12 +241,22 @@ def add_args(self, p: argparse.ArgumentParser): "Re-deploys the website version already on CodeRed Cloud." ), ) + p.add_argument( + "--skip-predeploy", + action="store_true", + help=( + "Skip common pre-deployment checks. " + "Only checks for local configuration errors." + ), + ) @classmethod def run(self, args: argparse.Namespace): w = self.get_webapp(args) if not args.no_upload: - w.local_check_path(args.path, CONSOLE) + w.local_check(args.path, CONSOLE) + if not args.skip_predeploy: + w.local_predeploy(args.path, CONSOLE) with Progress( TextColumn("[progress.description]{task.description}"), @@ -357,11 +367,21 @@ def add_args(self, p: argparse.ArgumentParser): p.add_argument(*arg_env.args, **arg_env.kwargs) p.add_argument(*arg_token.args, **arg_token.kwargs) p.add_argument(*arg_path.args, **arg_path.kwargs) + p.add_argument( + "--skip-predeploy", + action="store_true", + help=( + "Skip common pre-deployment checks. " + "Only checks for local configuration errors." + ), + ) @classmethod def run(self, args: argparse.Namespace): w = self.get_webapp(args) - w.local_check_path(args.path, CONSOLE) + w.local_check(args.path, CONSOLE) + if not args.skip_predeploy: + w.local_predeploy(args.path, CONSOLE) class Logs(Command): @@ -521,6 +541,14 @@ def add_args(self, p: argparse.ArgumentParser): "Defaults to `/www` which is the main directory." ), ) + p.add_argument( + "--skip-predeploy", + action="store_true", + help=( + "Skip common pre-deployment checks. " + "Only checks for local configuration errors." + ), + ) @classmethod def run(self, args: argparse.Namespace): @@ -529,7 +557,9 @@ def run(self, args: argparse.Namespace): # If the destination is the usual ``/www`` dir, and ``--path`` is a # directory, confirm with the user. if args.remote == PurePosixPath("/www") and args.path.is_dir(): - w.local_check_path(args.path, CONSOLE) + w.local_check(args.path, CONSOLE) + if not args.skip_predeploy: + w.local_predeploy(args.path, CONSOLE) with Progress( TextColumn("[progress.description]{task.description}"), diff --git a/cr/rich_utils.py b/cr/rich_utils.py index 8d821e0..6c5edb3 100644 --- a/cr/rich_utils.py +++ b/cr/rich_utils.py @@ -11,7 +11,7 @@ from typing import Optional from rich.console import WINDOWS -from rich.console import Console +from rich.console import Console as _Console from rich.console import Group from rich.console import RenderableType from rich.highlighter import RegexHighlighter @@ -45,6 +45,8 @@ "cr.argparse_text": "default", "cr.code": "bright_magenta", "cr.progress_print": "bright_black", + "cr.success": "bright_green", + "cr.fail": "bright_red", "cr.update_border": "bright_black", } ) @@ -64,6 +66,31 @@ class CustomHighlighter(RegexHighlighter): ] +class Console(_Console): + """ + Adds extra functionality for custom prompt behavior. + """ + + def prompt_yn(self, message: str, nouser: bool) -> bool: + """ + Prompts the user with Yes/No. If user enters Yes, return True. + Otherwise, return False. + + If the prompt occurs within a headless terminal (i.e. in a CI/CD + pipeline), always return the value of ``nouser``. + """ + if self.is_interactive: + val = self.input(message + " [prompt.choices](y/N)[/] ") + return val.strip().lower() == "y" + else: + self.print(message + " [prompt.choices](yes/no)[/] ") + if nouser: + self.print(" (Continuing without user input)") + else: + self.print(" (Not continuting without user input)") + return nouser + + CONSOLE = Console(highlighter=CustomHighlighter(), theme=RICH_THEME) CONSOLE_ERR = Console( highlighter=CustomHighlighter(), theme=RICH_THEME, stderr=True diff --git a/cr/utils.py b/cr/utils.py index cdbf334..9033403 100644 --- a/cr/utils.py +++ b/cr/utils.py @@ -1,6 +1,9 @@ """ Subprocess and filesystem utilities for working with the local project. +NOTE: For any git functionality here, it should fail gracefully if git is not +installed, or the project is not in a git repository. + Copyright (c) 2022-2024 CodeRed LLC. """ @@ -47,7 +50,7 @@ def get_command(program: str) -> Path: ``git`` might return ``C:\Program Files\Git\bin\git.exe`` :param str program: - The program to search for. Must be an exectable, not a shell built-in. + The program to search for. Must be an executable, not a shell built-in. :raise FileNotFoundError: if the program cannot be found on the PATH. @@ -156,17 +159,75 @@ def exec_proc( LOGGER.debug("Closing `%s`.", outfile) stdout.close() - return (proc.returncode, com_stdout, com_stderr) + return ( + proc.returncode, + com_stdout.strip(" \r\n"), + com_stderr.strip(" \r\n"), + ) + + +def stdout_to_list(stdout: str) -> List[str]: + """ + Cleans and splits newline delimited stdout to a list. + """ + stdout_list = [] + clean = stdout.strip(" \r\n") + if clean: + stdout_list = clean.split("\n") + return stdout_list def git_branch() -> str: """ Returns current git branch. + If git exits with an error, or is not on the path, return empty string. + """ + try: + code, out, err = exec_proc(["git", "branch", "--show-current"]) + if code != 0: + return "" + LOGGER.debug("Git branch `%s`.", out) + return out + except FileNotFoundError: + return "" + + +def git_uncommitted() -> List[str]: + """ + Checks for un-committed changes. + Returns list of changed file names, or empty list if no changes. """ - _, out, err = exec_proc(["git", "branch", "--show-current"]) - branch = out.strip("\r\n") - LOGGER.debug("Git branch `%s`.", branch) - return branch + try: + code, out, err = exec_proc(["git", "diff", "--name-only", "HEAD"]) + if code != 0: + return [] + diff_list = stdout_to_list(out) + return diff_list + except FileNotFoundError: + return [] + + +def git_unpushed() -> List[str]: + """ + Checks for un-pushed changes. + Returns list of changed file names, or empty list if no changes. + """ + # Get current branch. + b = git_branch() + if not b: + return [] + + # Check for changes against the remote branch. + try: + code, out, err = exec_proc( + ["git", "diff", "--name-only", f"origin/{b}"], + ) + if code != 0: + return [] + diff_list = stdout_to_list(out) + return diff_list + except FileNotFoundError: + return [] def git_ignored(p: Optional[Path] = None) -> List[Path]: @@ -190,7 +251,7 @@ def git_ignored(p: Optional[Path] = None) -> List[Path]: return lp # Split stdout by newline. - ls = out.strip("\r\n").split("\n") + ls = stdout_to_list(out) # Convert each entry to a Path. for s in ls: @@ -200,16 +261,6 @@ def git_ignored(p: Optional[Path] = None) -> List[Path]: return lp -def git_tag() -> str: - """ - Finds the current git tag. - """ - _, out, err = exec_proc(["git", "describe", "--tags"]) - tag = out.strip("\r\n") - LOGGER.debug("Git tag `%s`.", tag) - return tag - - def paths_to_deploy( r: Path, e: List[Path] = [], i: List[Path] = [] ) -> List[Path]: @@ -290,12 +341,40 @@ def check_handle(value: str) -> bool: return is_urlsafe(value) and len(value) <= 32 +def django_run_check(p: Path) -> Tuple[bool, str]: + """ + Runs ``manage.py check``. + + Returns a Tuple of success, and program output. + """ + code, out, err = exec_proc(["python", (p / "manage.py"), "check"]) + return (code == 0, out + err) + + +def django_run_migratecheck(p: Path) -> Tuple[bool, str]: + """ + Runs ``manage.py makemigrations --check``. + + Returns a Tuple of success, and program output. + """ + code, out, err = exec_proc( + ["python", (p / "manage.py"), "makemigrations", "--check"] + ) + return (code == 0, out + err) + + def django_manage_check(p: Path) -> None: + """ + Checks for existence of manage.py file. + """ if not (p / "manage.py").is_file(): raise FileNotFoundError("manage.py") def django_requirements_check(p: Path) -> None: + """ + Checks for existence of requirements.txt file. + """ if not (p / "requirements.txt").is_file(): raise FileNotFoundError("requirements.txt") From ae6ae87be7407c5664dbb2845d8e89706810bb0f Mon Sep 17 00:00:00 2001 From: Vince Salvino Date: Fri, 23 Aug 2024 19:16:31 -0400 Subject: [PATCH 2/3] Fix mypy errors --- cr/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cr/utils.py b/cr/utils.py index 9033403..2deae38 100644 --- a/cr/utils.py +++ b/cr/utils.py @@ -347,7 +347,7 @@ def django_run_check(p: Path) -> Tuple[bool, str]: Returns a Tuple of success, and program output. """ - code, out, err = exec_proc(["python", (p / "manage.py"), "check"]) + code, out, err = exec_proc(["python", str(p / "manage.py"), "check"]) return (code == 0, out + err) @@ -358,7 +358,7 @@ def django_run_migratecheck(p: Path) -> Tuple[bool, str]: Returns a Tuple of success, and program output. """ code, out, err = exec_proc( - ["python", (p / "manage.py"), "makemigrations", "--check"] + ["python", str(p / "manage.py"), "makemigrations", "--check"] ) return (code == 0, out + err) From 6b995e33823f2ea4627e16d141e873dbfb811ce3 Mon Sep 17 00:00:00 2001 From: Vince Salvino Date: Sun, 25 Aug 2024 01:37:41 -0400 Subject: [PATCH 3/3] Fix typos --- cr/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cr/api.py b/cr/api.py index 2e03668..5dd0562 100644 --- a/cr/api.py +++ b/cr/api.py @@ -250,7 +250,7 @@ def local_predeploy(self, p: Path, c: Optional[Console]) -> None: b = git_branch() if b and self.env == Env.STAGING and b not in STAGING_BRANCHES: if c and not c.prompt_yn( - f"Your are deploying to STAGING from the `{b}` branch!\n" + f"You are deploying to STAGING from the `{b}` branch!\n" "It is recommended to deploy from a dedicated staging branch.\n" "Continue anyways?", nouser=True, @@ -258,7 +258,7 @@ def local_predeploy(self, p: Path, c: Optional[Console]) -> None: raise UserCancelError() if b and self.env == Env.PROD and b not in PROD_BRANCHES: if c and not c.prompt_yn( - f"Your are deploying to PROD from the `{b}` branch!\n" + f"You are deploying to PROD from the `{b}` branch!\n" "It is recommended to deploy from a dedicated production branch.\n" "Continue anyways?", nouser=True,