From 40d9a58b0ac7a990ca36aabc0a500c0c04d9c556 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 11:56:57 -0400 Subject: [PATCH 1/9] environment variable to show debug options --- dvc/_debug.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index 1c4001b2d7..e57dc87e06 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -197,34 +197,42 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs): def add_debugging_flags(parser): from argparse import SUPPRESS + import os + DVC_SHOW_DEBUG_OPTIONS = os.environ.get('DVC_SHOW_DEBUG_OPTIONS', '') - parser.add_argument("--cprofile", action="store_true", default=False, help=SUPPRESS) - parser.add_argument("--yappi", action="store_true", default=False, help=SUPPRESS) + def debug_help(msg): + if DVC_SHOW_DEBUG_OPTIONS: + return msg + else: + return SUPPRESS + + parser.add_argument("--cprofile", action="store_true", default=False, help=debug_help('')) + parser.add_argument("--yappi", action="store_true", default=False, help=debug_help('')) parser.add_argument( "--yappi-separate-threads", action="store_true", default=False, - help=SUPPRESS, + help=debug_help(''), ) parser.add_argument( - "--viztracer", action="store_true", default=False, help=SUPPRESS + "--viztracer", action="store_true", default=False, help=debug_help('') ) - parser.add_argument("--viztracer-depth", type=int, help=SUPPRESS) + parser.add_argument("--viztracer-depth", type=int, help=debug_help('')) parser.add_argument( - "--viztracer-async", action="store_true", default=False, help=SUPPRESS + "--viztracer-async", action="store_true", default=False, help=debug_help('') ) - parser.add_argument("--cprofile-dump", help=SUPPRESS) - parser.add_argument("--pdb", action="store_true", default=False, help=SUPPRESS) + parser.add_argument("--cprofile-dump", help=debug_help('')) + parser.add_argument("--pdb", action="store_true", default=False, help=debug_help('')) parser.add_argument( - "--instrument", action="store_true", default=False, help=SUPPRESS + "--instrument", action="store_true", default=False, help=debug_help('') ) parser.add_argument( - "--instrument-open", action="store_true", default=False, help=SUPPRESS + "--instrument-open", action="store_true", default=False, help=debug_help('') ) parser.add_argument( "--show-stack", "--ss", action="store_true", default=False, - help=SUPPRESS, + help=debug_help(''), ) From ca5943d1b732e078e306fac97d6e7ca78c5f13c6 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 12:24:16 -0400 Subject: [PATCH 2/9] lint [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dvc/_debug.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index e57dc87e06..a0cd23db61 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -196,43 +196,50 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs): def add_debugging_flags(parser): - from argparse import SUPPRESS import os - DVC_SHOW_DEBUG_OPTIONS = os.environ.get('DVC_SHOW_DEBUG_OPTIONS', '') + from argparse import SUPPRESS + + dvc_show_debug_options = os.environ.get("DVC_SHOW_DEBUG_OPTIONS", "") def debug_help(msg): - if DVC_SHOW_DEBUG_OPTIONS: + if dvc_show_debug_options: return msg else: return SUPPRESS - parser.add_argument("--cprofile", action="store_true", default=False, help=debug_help('')) - parser.add_argument("--yappi", action="store_true", default=False, help=debug_help('')) + parser.add_argument( + "--cprofile", action="store_true", default=False, help=debug_help("") + ) + parser.add_argument( + "--yappi", action="store_true", default=False, help=debug_help("") + ) parser.add_argument( "--yappi-separate-threads", action="store_true", default=False, - help=debug_help(''), + help=debug_help(""), + ) + parser.add_argument( + "--viztracer", action="store_true", default=False, help=debug_help("") ) + parser.add_argument("--viztracer-depth", type=int, help=debug_help("")) parser.add_argument( - "--viztracer", action="store_true", default=False, help=debug_help('') + "--viztracer-async", action="store_true", default=False, help=debug_help("") ) - parser.add_argument("--viztracer-depth", type=int, help=debug_help('')) + parser.add_argument("--cprofile-dump", help=debug_help("")) parser.add_argument( - "--viztracer-async", action="store_true", default=False, help=debug_help('') + "--pdb", action="store_true", default=False, help=debug_help("") ) - parser.add_argument("--cprofile-dump", help=debug_help('')) - parser.add_argument("--pdb", action="store_true", default=False, help=debug_help('')) parser.add_argument( - "--instrument", action="store_true", default=False, help=debug_help('') + "--instrument", action="store_true", default=False, help=debug_help("") ) parser.add_argument( - "--instrument-open", action="store_true", default=False, help=debug_help('') + "--instrument-open", action="store_true", default=False, help=debug_help("") ) parser.add_argument( "--show-stack", "--ss", action="store_true", default=False, - help=debug_help(''), + help=debug_help(""), ) From e07e55c9f821ee7e9c4ad8eeb5b7666c9d97bf42 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 12:51:05 -0400 Subject: [PATCH 3/9] Add message to debug opts --- dvc/_debug.py | 55 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index a0cd23db61..ecb293630b 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -199,6 +199,8 @@ def add_debugging_flags(parser): import os from argparse import SUPPRESS + # For detailed info see: + # https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC dvc_show_debug_options = os.environ.get("DVC_SHOW_DEBUG_OPTIONS", "") def debug_help(msg): @@ -208,38 +210,69 @@ def debug_help(msg): return SUPPRESS parser.add_argument( - "--cprofile", action="store_true", default=False, help=debug_help("") + "--cprofile", + action="store_true", + default=False, + help=debug_help("Generate cprofile data for tools like snakeviz / tuna"), ) parser.add_argument( - "--yappi", action="store_true", default=False, help=debug_help("") + "--yappi", + action="store_true", + default=False, + help=debug_help( + "Generate a callgrind file for use with tools like kcachegrind / qcachegrind" + ), ) parser.add_argument( "--yappi-separate-threads", action="store_true", default=False, - help=debug_help(""), + help=debug_help("Generate one callgrind file per thread"), + ) + parser.add_argument( + "--viztracer", + action="store_true", + default=False, + help=debug_help("Generate a viztracer file for use with vizviewer"), + ) + parser.add_argument( + "--viztracer-depth", + type=int, + help=debug_help("Set viztracer maximum stack depth"), ) parser.add_argument( - "--viztracer", action="store_true", default=False, help=debug_help("") + "--viztracer-async", + action="store_true", + default=False, + help=debug_help("Treat async tasks as threads"), ) - parser.add_argument("--viztracer-depth", type=int, help=debug_help("")) parser.add_argument( - "--viztracer-async", action="store_true", default=False, help=debug_help("") + "--cprofile-dump", help=debug_help("location to dump cprofile file") ) - parser.add_argument("--cprofile-dump", help=debug_help("")) parser.add_argument( - "--pdb", action="store_true", default=False, help=debug_help("") + "--pdb", + action="store_true", + default=False, + help=debug_help("Drop into the pdb/ipdb debugger on any exception"), ) parser.add_argument( - "--instrument", action="store_true", default=False, help=debug_help("") + "--instrument", + action="store_true", + default=False, + help=debug_help("Use pyinstrument CLI profiler"), ) parser.add_argument( - "--instrument-open", action="store_true", default=False, help=debug_help("") + "--instrument-open", + action="store_true", + default=False, + help=debug_help("Use pyinstrument web profiler"), ) parser.add_argument( "--show-stack", "--ss", action="store_true", default=False, - help=debug_help(""), + help=debug_help( + "Use CTRL+T on macOS or Ctrl+/ on Linux to print the stack frame currently executing. Not available on Windows." + ), ) From 9e8d26ee4e56df8d4e15445cd13cca910bce4a9d Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 12:54:22 -0400 Subject: [PATCH 4/9] spelling --- dvc/_debug.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index ecb293630b..c82bb1603f 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -247,7 +247,7 @@ def debug_help(msg): help=debug_help("Treat async tasks as threads"), ) parser.add_argument( - "--cprofile-dump", help=debug_help("location to dump cprofile file") + "--cprofile-dump", help=debug_help("Location to dump cprofile file") ) parser.add_argument( "--pdb", @@ -273,6 +273,6 @@ def debug_help(msg): action="store_true", default=False, help=debug_help( - "Use CTRL+T on macOS or Ctrl+/ on Linux to print the stack frame currently executing. Not available on Windows." + r"Use Ctrl+T on macOS or Ctrl+\ on Linux to print the stack frame currently executing. Unavailable on Windows." ), ) From 3bba14a878c79ab4c779c79d32ae5a1ff6e71e10 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 13:17:16 -0400 Subject: [PATCH 5/9] Add argument group and move cprofile-dump line length linelen lint --- dvc/_debug.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index c82bb1603f..865539c821 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -206,8 +206,9 @@ def add_debugging_flags(parser): def debug_help(msg): if dvc_show_debug_options: return msg - else: - return SUPPRESS + return SUPPRESS + + parser = parser.add_argument_group("debug options") parser.add_argument( "--cprofile", @@ -215,12 +216,16 @@ def debug_help(msg): default=False, help=debug_help("Generate cprofile data for tools like snakeviz / tuna"), ) + parser.add_argument( + "--cprofile-dump", help=debug_help("Location to dump cprofile file") + ) parser.add_argument( "--yappi", action="store_true", default=False, help=debug_help( - "Generate a callgrind file for use with tools like kcachegrind / qcachegrind" + "Generate a callgrind file for use with tools like " + "kcachegrind / qcachegrind" ), ) parser.add_argument( @@ -246,9 +251,6 @@ def debug_help(msg): default=False, help=debug_help("Treat async tasks as threads"), ) - parser.add_argument( - "--cprofile-dump", help=debug_help("Location to dump cprofile file") - ) parser.add_argument( "--pdb", action="store_true", @@ -273,6 +275,7 @@ def debug_help(msg): action="store_true", default=False, help=debug_help( - r"Use Ctrl+T on macOS or Ctrl+\ on Linux to print the stack frame currently executing. Unavailable on Windows." + r"Use Ctrl+T on macOS or Ctrl+\ on Linux to print the stack " + "frame currently executing. Unavailable on Windows." ), ) From b886fa69dc890f65d527436f6c4301bb1db891c1 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 14:04:02 -0400 Subject: [PATCH 6/9] use DVC_DEBUG instead of DVC_SHOW_DEBUG_OPTIONS --- dvc/_debug.py | 3 ++- dvc/env.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index 865539c821..b8f3e910e4 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -198,10 +198,11 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs): def add_debugging_flags(parser): import os from argparse import SUPPRESS + from dvc.env import DVC_DEBUG # For detailed info see: # https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC - dvc_show_debug_options = os.environ.get("DVC_SHOW_DEBUG_OPTIONS", "") + dvc_show_debug_options = os.environ.get(DVC_DEBUG, "") def debug_help(msg): if dvc_show_debug_options: diff --git a/dvc/env.py b/dvc/env.py index fc2ad3f15e..3a2af5c513 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -1,4 +1,5 @@ DVC_DAEMON = "DVC_DAEMON" +DVC_DEBUG = "DVC_DEBUG" DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH" DVC_EXP_BASELINE_REV = "DVC_EXP_BASELINE_REV" DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE" From 187cb77711958c1ac1c8f4944e97272f96a20d72 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 14:21:40 -0400 Subject: [PATCH 7/9] use -v instead of environ --- dvc/_debug.py | 4 +--- dvc/cli/parser.py | 2 +- dvc/env.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index b8f3e910e4..052da012e9 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -196,13 +196,11 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs): def add_debugging_flags(parser): - import os from argparse import SUPPRESS - from dvc.env import DVC_DEBUG # For detailed info see: # https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC - dvc_show_debug_options = os.environ.get(DVC_DEBUG, "") + dvc_show_debug_options = parser.parse_known_args()[0].verbose def debug_help(msg): if dvc_show_debug_options: diff --git a/dvc/cli/parser.py b/dvc/cli/parser.py index fb6b7fe9d5..d38a13bd51 100644 --- a/dvc/cli/parser.py +++ b/dvc/cli/parser.py @@ -137,7 +137,6 @@ def get_parent_parser(): from dvc._debug import add_debugging_flags parent_parser = argparse.ArgumentParser(add_help=False) - add_debugging_flags(parent_parser) log_level_group = parent_parser.add_mutually_exclusive_group() log_level_group.add_argument( "-q", "--quiet", action="count", default=0, help="Be quiet." @@ -145,6 +144,7 @@ def get_parent_parser(): log_level_group.add_argument( "-v", "--verbose", action="count", default=0, help="Be verbose." ) + add_debugging_flags(parent_parser) return parent_parser diff --git a/dvc/env.py b/dvc/env.py index 3a2af5c513..fc2ad3f15e 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -1,5 +1,4 @@ DVC_DAEMON = "DVC_DAEMON" -DVC_DEBUG = "DVC_DEBUG" DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH" DVC_EXP_BASELINE_REV = "DVC_EXP_BASELINE_REV" DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE" From 23f916277ada730243295ef511b4a0b80020537e Mon Sep 17 00:00:00 2001 From: Jon Crall Date: Mon, 3 Jul 2023 14:55:21 -0400 Subject: [PATCH 8/9] Update dvc/_debug.py Co-authored-by: skshetry <18718008+skshetry@users.noreply.github.com> --- dvc/_debug.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index 052da012e9..92762d2820 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -200,7 +200,8 @@ def add_debugging_flags(parser): # For detailed info see: # https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC - dvc_show_debug_options = parser.parse_known_args()[0].verbose + args, _ = parser.parse_known_args() + verbose = args.verbose def debug_help(msg): if dvc_show_debug_options: From 3369f2cbe1e94b42ffac9839cbfe24a9dba78030 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 3 Jul 2023 14:56:57 -0400 Subject: [PATCH 9/9] fix --- dvc/_debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/_debug.py b/dvc/_debug.py index 92762d2820..c40327e140 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -204,7 +204,7 @@ def add_debugging_flags(parser): verbose = args.verbose def debug_help(msg): - if dvc_show_debug_options: + if verbose: return msg return SUPPRESS