Expose the pbulk logs outside of the sandbox

Add a new PBULK_LOG configuration variable that points to a directory
outside of the sandbox and that is used to record all pbulk build logs.
These logs are necessary to troubleshoot any build failures that happen
during the build, and are also useful under successful conditions to
get summaries about what was built.

Fixes #3.

NEWS

@@ -3,6 +3,10 @@ Changes in version 2.1
 
 **STILL UNDER DEVELOPMENT; NOT RELEASED YET.**
 
+* Issue 3: Added a new `PBULK_LOG` setting to point at a location that
+  holds all build logs, and made the `build` command print out details
+  about such logs on success and failure.
+
 * Issue 4: Fixed the installable bootstrap kit generated during a build
   to not leak pkg_comp-internal paths.
 

admin/make-macos-pkg.sh

@@ -162,6 +162,7 @@ EOF
     edit_config "${f}" PKGSRCDIR /var/pkg_comp/pkgsrc
     edit_config "${f}" DISTDIR /var/pkg_comp/distfiles
     edit_config "${f}" PACKAGES /var/pkg_comp/packages
+    edit_config "${f}" PBULK_LOG /var/pkg_comp/log/pbulk
     edit_config "${f}" PBULK_PACKAGES /var/pkg_comp/pbulk-packages
     edit_config "${f}" LOCALBASE /opt/pkg
     edit_config "${f}" PKG_DBDIR /opt/pkg/libdata/pkgdb

default.conf.in

@@ -19,6 +19,7 @@ SANDBOX_CONFFILE="__PKG_COMP_ETCDIR__/sandbox.conf"
 PKGSRCDIR="/usr/pkgsrc"
 DISTDIR="${PKGSRCDIR}/distfiles"
 PACKAGES="${PKGSRCDIR}/packages"
+PBULK_LOG="${PACKAGES}/log"
 PBULK_PACKAGES="${PACKAGES}/pbulk"
 
 # Target file layout.  These are the directories that the built binary

pkg_comp.8.in

@@ -25,7 +25,7 @@
 .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd February 17, 2017
+.Dd August 30, 2018
 .Dt PKG_COMP 8
 .Os
 .Sh NAME
@@ -253,6 +253,20 @@ This is necessary to keep fully-populated
 databases.
 However, because pbulk reuses existing binary packages and only rebuilds updated
 packages, any packages not given on the command line should "build" quickly.
+.Pp
+In case of build problems, detailed logs for each package that failed to build
+will be located in the directory pointed at by
+.Va PBULK_LOG .
+Of special interest is the
+.Pa SUMMARY
+subdirectory, which contains summary reports of the whole operation in various
+different formats.
+Examples are:
+.Pa report.html
+and
+.Pa report.txt ,
+both of which summarize how many packages were built and include details of all
+that failed.
 .Ss The config command
 The config command dumps the loaded configuration to the standard output.
 The format of the output is not a script, so it cannot be fed back into

pkg_comp.conf.5

@@ -25,7 +25,7 @@
 .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd February 14, 2017
+.Dd August 30, 2018
 .Dt PKG_COMP.CONF 5
 .Os
 .Sh NAME
@@ -156,6 +156,14 @@ packages and bootstrap kit for the target environment.
 .Pp
 Default:
 .Pa /usr/pkgsrc/packages .
+.It Va PBULK_LOG
+Absolute path to the directory in the host that will hold the logs of the
+pbulk build.
+The contents of this directory are wiped at the beginning of each
+build.
+.Pp
+Default:
+.Pa /usr/pkgsrc/packages/log .
 .It Va PBULK_PACKAGES
 Absolute path to the directory in the host that will hold the built binary
 packages and bootstrap kit for the pbulk installation in the sandbox.

pkg_comp.sh

@@ -44,8 +44,8 @@ SHTK_MODULESPATH="${PKG_COMP_SHTK_MODULESDIR}" shtk_import pkgsrc
 # Please remember to update pkg_comp.conf(5) if you change this list.
 PKG_COMP_CONFIG_VARS="AUTO_PACKAGES CVS_ROOT CVS_TAG DISTDIR EXTRA_MKCONF
                       FETCH_VCS GIT_BRANCH GIT_URL LOCALBASE NJOBS PACKAGES
-                      PBULK_PACKAGES PKG_DBDIR PKGSRCDIR SANDBOX_CONFFILE
-                      SYSCONFDIR UPDATE_SOURCES VARBASE"
+                      PBULK_LOG PBULK_PACKAGES PKG_DBDIR PKGSRCDIR
+                      SANDBOX_CONFFILE SYSCONFDIR UPDATE_SOURCES VARBASE"
 
 
 # Paths to installed files.
@@ -72,6 +72,7 @@ pkg_comp_set_defaults() {
     shtk_config_set LOCALBASE "/usr/pkg"
     shtk_config_set NJOBS "$(shtk_hw_ncpus)"
     shtk_config_set PACKAGES "/usr/pkgsrc/packages"
+    shtk_config_set PBULK_LOG "/usr/pkgsrc/packages/log"
     shtk_config_set PBULK_PACKAGES "/usr/pkgsrc/packages/pbulk"
     shtk_config_set PKG_DBDIR "/usr/pkg/libdata/pkgdb"
     shtk_config_set PKGSRCDIR "/usr/pkgsrc"
@@ -119,6 +120,7 @@ run_sandboxctl() {
             cat <<EOF
 DISTDIR="$(shtk_config_has DISTDIR && shtk_config_get DISTDIR)"
 PACKAGES="$(shtk_config_has PACKAGES && shtk_config_get PACKAGES)"
+PBULK_LOG="$(shtk_config_has PBULK_LOG && shtk_config_get PBULK_LOG)"
 PBULK_PACKAGES="$(shtk_config_has PBULK_PACKAGES \
     && shtk_config_get PBULK_PACKAGES)"
 PKGSRCDIR="$(shtk_config_has PKGSRCDIR && shtk_config_get PKGSRCDIR)"
@@ -385,9 +387,9 @@ EOF
     pbulk_set unprivileged_user root
 
     # Configure pbulk's file layout.
-    pbulk_set bulklog /pkg_comp/work/bulklog
+    pbulk_set bulklog /pkg_comp/bulklog
     pbulk_set limited_list /pkg_comp/pbulk/etc/pbulk.list
-    pbulk_set loc /pkg_comp/work/bulklog/meta
+    pbulk_set loc /pkg_comp/bulklog/SUMMARY
 
     # Configure pkgsrc's file layout.
     pbulk_set bootstrapkit "/pkg_comp/packages/pkg/bootstrap.tgz"
@@ -546,14 +548,25 @@ pkg_comp_build() {
         echo "${package}" >>"${list}"
     done
 
+    local bulklog="$(shtk_config_get PBULK_LOG)"
+
     # Removing bulklog/success seems to be necessary to restart a build, as
     # otherwise bulkbuild does not build modified packages.  Is this correct?
-    rm -f "${root}/pkg_comp/work/bulklog/success"
+    rm -f "${root}/pkg_comp/bulklog/success"
+
     shtk_cli_info "Starting pbulk build in the sandbox"
     run_sandboxctl run /pkg_comp/pbulk/bin/bulkbuild || \
-        shtk_cli_error "bulkbuild failed; see ${root}/pkg_comp/work/bulklog/" \
-            "for possible details"
+        shtk_cli_error "bulkbuild failed; see ${bulklog}/ for possible details"
     generate_pkg_summary "${root}"
+
+    if [ -s "${bulklog}/SUMMARY/error" ]; then
+        local errors="$(cat ${bulklog}/SUMMARY/error)"
+        shtk_cli_error "Failed to build ${errors}; see ${bulklog}/ for" \
+            "detailed logs"
+    else
+        shtk_cli_info "All packages built successfully; summary reports are" \
+            "in ${bulklog}/SUMMARY/"
+    fi
 }
 
 

pkg_comp_inttest.sh

@@ -95,6 +95,7 @@ integration_body() {
     cat >pkg_comp.conf <<EOF
 DISTDIR="$(atf_config_get distdir)"
 PACKAGES="$(pwd)/packages/pkg"
+PBULK_LOG="$(pwd)/bulklog"
 PBULK_PACKAGES="$(pwd)/packages/pbulk"
 PKGSRCDIR="$(atf_config_get pkgsrcdir)"
 SANDBOX_CONFFILE="$(pwd)/sandbox.conf"
@@ -131,6 +132,12 @@ setup_fetch_from_local_git() {
         echo "FETCH_VCS=git" >>pkg_comp.conf
         echo "GIT_URL='file://$(atf_config_get pkgsrcdir)'" >>pkg_comp.conf
     fi
+
+    # Configure Git in case our tests want to modify the pkgsrc files.  In those
+    # cases, pkg_comp will need to stash the modifications during "fetch" and
+    # we need these details.
+    git config --global user.email "[email protected]"
+    git config --global user.name "Travis"
 }
 
 
@@ -599,6 +606,40 @@ auto_workflow_reusing_sandbox_intbody() {
 }
 
 
+integration_test_case logs_workflow
+logs_workflow_intbody() {
+    reuse_bootstrap
+    reuse_packages cwrappers digest
+
+    setup_fetch_from_local_git
+
+    atf_check -o ignore -e ignore pkg_comp -c pkg_comp.conf fetch
+
+    check_files pkgsrc/pkgtools/verifypc/Makefile
+    cat >>pkgsrc/pkgtools/verifypc/Makefile <<EOF
+pre-extract: always-fail
+always-fail: .PHONY
+	false
+EOF
+
+    atf_check \
+        -s exit:1 \
+        -o match:'Starting build of .*verifypc' \
+        -o match:'Failed to build.*verifypc' \
+        -e match:'Failed to build.*verifypc.*detailed logs' \
+        pkg_comp -c pkg_comp.conf auto verifypc
+
+    # Check presence of pbulk summary logs.
+    test -f bulklog/report.txt || fail "report.txt not found in bulklog"
+    test -f bulklog/report.html || fail "report.txt not found in bulklog"
+
+    # Check presence of package-specific logs.
+    test -f bulklog/verifypc*/failure || fail "verifypc-specific logs not found"
+
+    save_state
+}
+
+
 atf_init_test_cases() {
     local tests=
     tests="${tests} auto_workflow"
@@ -608,6 +649,7 @@ atf_init_test_cases() {
     tests="${tests} build_workflow"
     tests="${tests} fetch_workflow"
     tests="${tests} functional_pkgsrc_after_bootstrap"
+    tests="${tests} logs_workflow"
 
     local i=0
     for t in ${tests}; do

pkg_comp_test.sh

@@ -144,6 +144,7 @@ GIT_URL = https://github.com/NetBSD/pkgsrc.git
 LOCALBASE = /usr/pkg
 NJOBS = 99
 PACKAGES = /usr/pkgsrc/packages
+PBULK_LOG = /usr/pkgsrc/packages/log
 PBULK_PACKAGES = /usr/pkgsrc/packages/pbulk
 PKG_DBDIR = /usr/pkg/libdata/pkgdb
 PKGSRCDIR = /usr/pkgsrc
@@ -228,6 +229,7 @@ GIT_URL = https://github.com/NetBSD/pkgsrc.git
 LOCALBASE = /usr/pkg
 NJOBS = 80
 PACKAGES = /usr/pkgsrc/packages
+PBULK_LOG = /usr/pkgsrc/packages/log
 PBULK_PACKAGES = /usr/pkgsrc/packages/pbulk
 PKG_DBDIR = /usr/pkg/libdata/pkgdb
 PKGSRCDIR is undefined
@@ -447,6 +449,7 @@ sandboxctl__create_destroy_integration_body() {
     cat >pkg_comp.conf <<EOF
 DISTDIR="$(pwd)/distfiles"
 PACKAGES="$(pwd)/packages/pkg"
+PBULK_LOG="$(pwd)/packages/log"
 PBULK_PACKAGES="$(pwd)/packages/pbulk"
 PKGSRCDIR="$(pwd)/pkgsrc"
 SANDBOX_CONFFILE="$(pwd)/sandbox.conf"

sandbox.conf.post

@@ -54,10 +54,12 @@ post_create_hook() {
     # sandbox... but this behavior is less annoying.
     mkdir -p "${DISTDIR}"
     mkdir -p "${PACKAGES}"
+    mkdir -p "${PBULK_LOG}"
     mkdir -p "${PBULK_PACKAGES}"
     mkdir -p "${PKGSRCDIR}"
 
     mkdir "${SANDBOX_ROOT}/pkg_comp"
+    mkdir "${SANDBOX_ROOT}/pkg_comp/bulklog"
     mkdir "${SANDBOX_ROOT}/pkg_comp/distfiles"
     mkdir "${SANDBOX_ROOT}/pkg_comp/packages"
     mkdir "${SANDBOX_ROOT}/pkg_comp/packages/pbulk"
@@ -74,6 +76,7 @@ post_create_hook() {
 post_mount_hook() {
     _pkg_comp_validate_variables
 
+    sandbox_bindfs -o rw "${PBULK_LOG}" "${SANDBOX_ROOT}/pkg_comp/bulklog"
     sandbox_bindfs -o rw "${DISTDIR}" "${SANDBOX_ROOT}/pkg_comp/distfiles"
     sandbox_bindfs -o rw "${PACKAGES}" "${SANDBOX_ROOT}/pkg_comp/packages/pkg"
     sandbox_bindfs -o rw "${PBULK_PACKAGES}" \