Add a text post processing phase to the translator.

Macros or problems can add a hook by calling
`add_content_post_processor` with a subroutine that will be called after
the translator has processed answers.  For all display modes except TeX,
the subroutine will be passed the problem text parsed into a Mojo::DOM
object and a reference to the header text.  For the TeX display mode a
reference to the problem text is passed.  The subroutine can then modify
the problem DOM or text, and header text.  The resulting Mojo::DOM
object will be converted back to a string and the problem text reference
returned by the translator set to point to that string.

The hidden MathQuill inputs for the latex string is added in this post
processing stage (see the ENDDOCUMENT method in PG.pl).

The scaffold.pl macro is updated to use this post processing.  This
means that answers are no longer evaluated by the macro.  Instead the
scaffold sections are finalized after the answers are processed normally
by the translator.

.github/workflows/unit-tests.yml

@@ -28,6 +28,7 @@ jobs:
             libhtml-parser-perl \
             libjson-perl \
             libjson-xs-perl \
+            libmojolicious-perl \
             libtest2-suite-perl \
             libtie-ixhash-perl \
             libuuid-tiny-perl \

cpanfile

@@ -17,6 +17,7 @@ on runtime => sub {
 	requires 'JSON';
 	requires 'JSON::XS';
 	requires 'Locale::Maketext';
+	requires 'Mojolicious';
 	requires 'Tie::IxHash';
 	requires 'Types::Serialiser';
 	requires 'UUID::Tiny';

docker/pg.Dockerfile

@@ -20,6 +20,7 @@ RUN apt-get update \
 		libhtml-parser-perl \
 		libjson-perl \
 		libjson-xs-perl \
+		libmojolicious-perl \
 		libtest2-suite-perl \
 		libtie-ixhash-perl \
 		libuuid-tiny-perl \

htdocs/js/InputColor/color.js

@@ -39,7 +39,7 @@
 				answerInput.focus();
 			});
 		} else {
-			answerLink.href = '';
+			answerLink.removeAttribute('href');
 		}
 	};
 

lib/PGcore.pm

@@ -97,6 +97,7 @@ sub new {
 		PG_alias                     => undef,
 		PG_problem_grader            => undef,
 		displayMode                  => undef,
+		content_post_processors      => [],
 		envir                        => $envir,
 		WARNING_messages             => [],
 		DEBUG_messages               => [],
@@ -563,6 +564,12 @@ sub get_persistent_data {
 	return $self->{PERSISTENCE_HASH}{$label};
 }
 
+sub add_content_post_processor {
+	my ($self, $handler) = @_;
+	push(@{ $self->{content_post_processors} }, $handler) if ref($handler) eq 'CODE';
+	return;
+}
+
 sub check_answer_hash {
 	my $self = shift;
 	foreach my $key (keys %{ $self->{PG_ANSWERS_HASH} }) {

lib/WeBWorK/PG.pm

@@ -139,7 +139,7 @@ sub new_helper ($invocant, %options) {
 		}
 	}
 
-	$translator->translate();
+	$translator->translate;
 
 	# IMPORTANT: The translator environment should not be trusted after the problem code runs.
 
@@ -172,6 +172,8 @@ sub new_helper ($invocant, %options) {
 		);
 	}
 
+	$translator->post_process_content;
+
 	# HTML_dpng uses an ImageGenerator. We have to render the queued equations.
 	if ($image_generator) {
 		my $sourceFile = "$options{templateDirectory}$options{sourceFilePath}";

lib/WeBWorK/PG/Translator.pm

@@ -15,57 +15,63 @@
 
 package WeBWorK::PG::Translator;
 
-use strict;
-use warnings;
-
-use utf8;
-use v5.12;
-binmode(STDOUT, ":encoding(UTF-8)");
-
-use Opcode;
-use Carp;
-
-use WWSafe;
-use PGUtil qw(pretty_print);
-use WeBWorK::PG::IO qw(fileFromPath);
-
 =head1 NAME
 
 WeBWorK::PG::Translator - Evaluate PG code and evaluate answers safely
 
 =head1 SYNPOSIS
 
-    my $pt = new WeBWorK::PG::Translator;    # create a translator
+    my $pt = WeBWorK::PG::Translator->new;   # create a translator
     $pt->environment(\%envir);               # provide the environment variable for the problem
-    $pt->initialize();                       # initialize the translator
-    $pt-> set_mask();                        # set the operation mask for the translator safe compartment
+    $pt->initialize;                         # initialize the translator
+    $pt->set_mask;                           # set the operation mask for the translator safe compartment
+
     $pt->source_string($source);             # provide the source string for the problem
+                                             # or
+    $pt->source_file($sourceFilePath);       # provide the proble file containing the source
 
     # Load the unprotected macro files.
     # These files are evaluated with the Safe compartment wide open.
     # Other macros are loaded from within the problem using loadMacros.
-    $pt->unrestricted_load("${courseScriptsDirectory}PG.pl");
+    # This should not be done if the safe cache is used which is only the case if $ENV{MOJO_MODE} exists.
+    $pt->unrestricted_load("${pgMacrosDirectory}PG.pl");
 
-    $pt->translate();    # translate the problem (the following pieces of information are created)
+    $pt->translate;    # translate the problem (the following pieces of information are created)
 
-    $PG_PROBLEM_TEXT_ARRAY_REF = $pt->ra_text();      # output text for the body of the HTML file (in array form)
-    $PG_PROBLEM_TEXT_REF = $pt->r_text();             # output text for the body of the HTML file
-    $PG_HEADER_TEXT_REF = $pt->r_header;              # text for the header of the HTML file
+    $PG_PROBLEM_TEXT_ARRAY_REF = $pt->ra_text;        # reference to an array of output text for body of the problem
+    $PG_PROBLEM_TEXT_REF = $pt->r_text;               # reference to output text for the body of problem
+    $PG_HEADER_TEXT_REF = $pt->r_header;              # reference to text for the header in HTML output
     $PG_POST_HEADER_TEXT_REF = $pt->r_post_header
     $PG_ANSWER_HASH_REF = $pt->rh_correct_answers;    # a hash of answer evaluators
     $PG_FLAGS_REF = $pt->rh_flags;                    # misc. status flags.
 
-    $pt->process_answers;    # evaluates all of the answers
+    $pt->process_answers;                                 # evaluates all of the answers
+    my $rh_answer_results = $pt->rh_evaluated_answers;    # provides a hash of the results of evaluating the answers.
+    my $rh_problem_result = $pt->grade_problem(%options); # grades the problem.
 
-    my $rh_answer_results = $pt->rh_evaluated_answers;  # provides a hash of the results of evaluating the answers.
-    my $rh_problem_result = $pt->grade_problem;         # grades the problem using the default problem grading method.
+    $pt->post_process_content;   # Execute macro or problem hooks that further modify the problem content.
 
 =head1 DESCRIPTION
 
 This module defines an object which will translate a problem written in the Problem Generating (PG) language
 
 =cut
 
+use strict;
+use warnings;
+
+use utf8;
+use v5.12;
+binmode(STDOUT, ":encoding(UTF-8)");
+
+use Opcode;
+use Carp;
+use Mojo::DOM;
+
+use WWSafe;
+use PGUtil qw(pretty_print);
+use WeBWorK::PG::IO qw(fileFromPath);
+
 =head2 be_strict
 
 This creates a substitute for C<use strict;> which cannot be used in PG problem
@@ -146,15 +152,18 @@ BEGIN {
 	}
 
 	# Also define in Main:: for PG modules.
-	sub Main::be_strict { return &be_strict; }
+	sub Main::be_strict { return be_strict(); }
 }
 
 =head2 evaluate_modules
 
-    Usage:  $obj->evaluate_modules('WWPlot', 'Fun', 'Circle');
+Adds modules to the list of modules which can be used by the PG problems.
+
+For example,
 
-Adds the modules WWPlot.pm, Fun.pm and Circle.pm in the courseScripts directory to the list of modules
-which can be used by the PG problems.
+    $obj->evaluate_modules('LaTeXImage', 'DragNDrop');
+
+adds modules to the C<LaTeXImage> and C<DragNDrop> modules.
 
 =cut
 
@@ -179,13 +188,13 @@ sub evaluate_modules {
 
 =head2 load_extra_packages
 
-    Usage:  $obj->load_extra_packages('AlgParserWithImplicitExpand',
-                                      'Expr','ExprWithImplicitExpand');
+Loads extra packages for modules that contain more than one package.  Works in
+conjunction with evaluate_modules.  It is assumed that the file containing the
+extra packages (along with the base package name which is the same as the name
+of the file minus the .pm extension) has already been loaded using
+evaluate_modules.
 
-Loads extra packages for modules that contain more than one package.  Works in conjunction with
-evaluate_modules.  It is assumed that the file containing the extra packages (along with the base
-package name which is the same as the name of the file minus the .pm extension) has already been
-loaded using evaluate_modules
+    Usage:  $obj->load_extra_packages('AlgParserWithImplicitExpand', 'ExprWithImplicitExpand');
 
 =cut
 
@@ -209,7 +218,7 @@ sub load_extra_packages {
 
 =head2 new
 
-    Creates the translator object.
+Creates the translator object.
 
 =cut
 
@@ -249,40 +258,19 @@ sub new {
 	return bless $self, $class;
 }
 
-=pod
-
-(b) The following routines defined within the PG module are shared:
-
-    &be_strict
-    &read_whole_problem_file
-    &convertPath
-    &surePathToTmpFile
-    &fileFromPath
-    &directoryFromPath
-    &PG_answer_eval
-    &PG_restricted_eval
-    &send_mail_to
-
-In addition the environment hash C<%envir> is shared.  This variable is unpacked
-when PG.pl is run and provides most of the environment variables for each problem
-template.
+=head2 initialize
 
-=for html
-    <a href = "${Global::webworkDocsURL}techdescription/pglanguage/PGenvironment.html">environment variables</a>
+The following translator methods are shared to the safe compartment:
 
-(c) Sharing macros:
+	&PG_answer_eval
+	&PG_restricted_eval
+	&PG_macro_file_eval
+	&be_strict
 
-The macros shared with the safe compartment are
+Also all methods that are exported by WeBWorK::PG::IO are shared.
 
-    '&read_whole_problem_file'
-    '&convertPath'
-    '&surePathToTmpFile'
-    '&fileFromPath'
-    '&directoryFromPath'
-    '&PG_answer_eval'
-    '&PG_restricted_eval'
-    '&be_strict'
-    '&send_mail_to'
+In addition the environment hash C<%envir> is shared.  This variable is unpacked
+when PG.pl is run.
 
 =cut
 
@@ -526,10 +514,10 @@ sub errors {
 
 =head2 set_mask
 
-(e) Now we close the safe compartment.  Only the certain operations can be used
-within PG problems and the PG macro files.  These include the subroutines
-shared with the safe compartment as defined above and most Perl commands which
-do not involve file access, access to the system or evaluation.
+Limit allowed operations in the safe compartment.  Only the certain operations
+can be used within PG problems and the PG macro files.  These include the
+subroutines shared with the safe compartment as defined above and most Perl
+commands which do not involve file access, access to the system or evaluation.
 
 Specifically the following are allowed:
 
@@ -646,7 +634,7 @@ sub PG_errorMessage {
 
 =head2 Translate
 
-(3) B<Preprocess the problem text>
+B<Preprocess the problem text>
 
 The input text is subjected to some global replacements.
 
@@ -703,20 +691,20 @@ Note that there are several other replacements that are now done that are not
 documented here.  See the C<default_preprocess_code> method for all
 replacements that are done.
 
-(4) B<Evaluate the problem text>
+B<Evaluate the problem text>
 
 Evaluate the text within the safe compartment.  Save the errors. The safe
 compartment is a new one unless the $safeCompartment was set to zero in which
 case the previously defined safe compartment is used. (See item 1.)
 
-(5) B<Process errors>
+B<Process errors>
 
 The error provided by Perl is truncated slightly and returned. In the text
 string which would normally contain the rendered problem.
 
 The original text string is given line numbers and concatenated to the errors.
 
-(6) B<Prepare return values>
+B<Prepare return values>
 
 Sets the following hash keys of the translator object:
     PG_PROBLEM_TEXT_ARRAY_REF:  Reference to an array of strings containing the
@@ -860,7 +848,7 @@ sub translate {
 
 =cut
 
-=head3  access methods
+=head3 access methods
 
     $obj->rh_student_answers
 
@@ -1202,6 +1190,50 @@ sub avg_problem_grader {
 	return (\%problem_result, \%problem_state);
 }
 
+sub post_process_content {
+	my $self = shift;
+
+	my $outer_sig_warn = $SIG{__WARN__};
+	my @warnings;
+	local $SIG{__WARN__} = sub { push(@warnings, $_[0]) };
+
+	my $outer_sig_die = $SIG{__DIE__};
+	local $SIG{__DIE__} = sub {
+		ref $outer_sig_die eq "CODE"
+			? $outer_sig_die->(PG_errorMessage('traceback', $_[0]))
+			: die PG_errorMessage('traceback', $_[0]);
+	};
+
+	if ($self->{rh_pgcore}{displayMode} eq 'TeX') {
+		our $PG_PROBLEM_TEXT_REF = $self->{PG_PROBLEM_TEXT_REF};
+		$self->{safe}->share('$PG_PROBLEM_TEXT_REF');
+		$self->{safe}->reval('for (@{ $main::PG->{content_post_processors} }) { $_->($PG_PROBLEM_TEXT_REF); }', 1);
+		warn "ERRORS from post processing PG text:\n$@\n" if $@;
+
+		$self->{PG_PROBLEM_TEXT_ARRAY_REF} = [ split(/^/, ${ $self->{PG_PROBLEM_TEXT_REF} }) ];
+	} else {
+		$self->{safe}->share_from('main', [qw(%Mojo::Base:: %Mojo::Collection:: %Mojo::DOM::)]);
+		our $problemDOM = Mojo::DOM->new(${ $self->{PG_PROBLEM_TEXT_REF} });
+		$problemDOM->xml(1) if $self->{rh_pgcore}{displayMode} eq 'PTX';
+		our $PG_HEADER_TEXT_REF = $self->{PG_HEADER_TEXT_REF};
+		$self->{safe}->share('$problemDOM', '$PG_HEADER_TEXT_REF');
+		$self->{safe}
+			->reval('for (@{ $main::PG->{content_post_processors} }) { $_->($problemDOM, $PG_HEADER_TEXT_REF); }', 1);
+		warn "ERRORS from post processing PG text:\n$@\n" if $@;
+
+		$self->{PG_PROBLEM_TEXT_REF}       = \($problemDOM->to_string);
+		$self->{PG_PROBLEM_TEXT_ARRAY_REF} = [ split(/^/, ${ $self->{PG_PROBLEM_TEXT_REF} }) ];
+	}
+
+	if (@warnings) {
+		ref $outer_sig_warn eq "CODE"
+			? $outer_sig_warn->(PG_errorMessage('message', @warnings))
+			: warn PG_errorMessage('message', @warnings);
+	}
+
+	return;
+}
+
 =head2 PG_restricted_eval
 
     PG_restricted_eval($string)
@@ -1230,7 +1262,7 @@ sub PG_restricted_eval {
 
 	my $out        = PG_restricted_eval_helper($string);
 	my $err        = $@;
-	my $err_report = $err if $err =~ /\S/;
+	my $err_report = $err =~ /\S/ ? $err : undef;
 	return wantarray ? ($out, $err, $err_report) : $out;
 }