Merge branch 'main' of github.com:performancecopilot/pcp

man/man1/pmie_check.1

@@ -223,7 +223,11 @@ only the data file to be compressed, and also prevents the program from
 attempting to compress it more than once.
 The default
 .I regex
-is "\.(meta|index|Z|gz|bz2|zip|xz|lzma|lzo|lz4)$" \- such files are
+is
+.br
+"\e.(meta|index|Z|gz|bz2|zip|xz|lzma|lzo|lz4|zst)$"
+.br
+\- such files are
 filtered using the
 .B \-v
 option to

man/man1/pmlogcheck.1

@@ -23,16 +23,43 @@
 [\f3\-S\f1 \f2start\f1]
 [\f3\-T\f1 \f2finish\f1]
 [\f3\-Z\f1 \f2timezone\f1]
-\f2archive\f1
+\f2archive\f1 ...
+.de EX
+.in +0.5i
+.ie t .ft CB
+.el .ft B
+.ie t .sp .5v
+.el .sp
+.ta \\w' 'u*8
+.nf
+..
+.de EE
+.fi
+.ie t .sp .5v
+.el .sp
+.ft R
+.in
+..
 .SH DESCRIPTION
 .B pmlogcheck
 prints information about the nature of any invalid data which it detects
-in the files of a PCP archive.
+in the files of PCP archives.
 .PP
-The archive has the base name
+Each archive has the base name
 .I archive
 and must have been previously created using
 .BR pmlogger (1).
+Alternatively,
+.I archive
+can be the name of a physical file that forms part of a PCP archive.
+.PP
+When multiple
+.I archive
+options are present, all of the corresponding PCP archives will be
+checked, however if the
+.I archive
+options are file names, then each associated PCP archive will be checked
+at most once.
 .SH OPTIONS
 The available command line options are:
 .TP 5
@@ -183,6 +210,27 @@ in addition to the time at which the counter wrap occurred.
 If the extent of the data being checked is less than 24 hours, a more
 precise format is used (time is displayed with millisecond precision, but
 without the date).
+.SH EXAMPLES
+The following are all equivalent:
+.EX
+$ pmlogcheck myarchive
+.br
+$ pmlogcheck myarchive.0
+.br
+$ pmlogcheck myarchive.meta
+.br
+$ pmlogcheck myarchive.0 myarchive.meta
+.EE
+.PP
+If the current directory contains multiple PCP archives, then
+the command
+.EX
+$ pmlogcheck *.meta*
+.EE
+will check them all, as there is one
+.B .meta
+file for each PCP archive, but this file may have been compressed
+(hence the trailing \fB*\fP).
 .SH FILES
 .TP 5
 .I $PCP_VAR_DIR/pmns/*
@@ -214,4 +262,5 @@ and
 .BR PMNS (5).
 
 .\" control lines for scripts/man-spell
-.\" +ok+ endtime starttime
+.\" +ok+ endtime starttime myarchive {from example}
+.\" +ok+ sp {from troff .sp}

man/man1/pmlogger_daily.1

@@ -544,7 +544,11 @@ only the data file to be compressed, and also prevents the program from
 attempting to compress it more than once.
 The default
 .I regex
-is "\.(index|Z|gz|bz2|zip|xz|lzma|lzo|lz4)$" \- such files are
+is
+.br
+"\e.(index|Z|gz|bz2|zip|xz|lzma|lzo|lz4|zst)$"
+.br
+\- such files are
 filtered using the
 .B \-v
 option to

qa/1570

@@ -0,0 +1,25 @@
+#!/bin/sh
+# PCP QA Test No. 1570
+# Exercise compressed archive files - zstd version
+#
+# Copyright (c) 2014 Red Hat.
+# Copyright (c) 2010,2024 Ken McDonell.  All Rights Reserved.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+. ./common.compress
+
+which zstd >/dev/null 2>&1 || _notrun No zstd binary installed
+
+status=1	# failure is the default!
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "cd $here; rm -rf $tmp $tmp.*; exit \$status" 0 1 2 3 15
+
+# real QA test starts here
+_prepare_compress "zstd --rm --quiet --no-progress" "unzstd --rm --quiet --no-progress" zst
+_exercise_compression
+
+status=0	# success, we're all done
+exit

qa/1570.out

@@ -0,0 +1,84 @@
+QA output created by 1570
+expect only a few lines of diff output ...
+
+--- zstd --rm --quiet --no-progress first volume ---
+> pmdumplog ...
+> pminfo ...
+> pmprobe in the middle ...
+> pmval & pmval -r ...
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin
+> pmie ...
+
+--- zstd --rm --quiet --no-progress last volume and use existing .9.zst in -a arg ---
+> pmdumplog ...
+> pminfo ...
+> pmprobe in the middle ...
+> pmval & pmval -r ...
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.9.zst
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.9.zst
+> pmie ...
+
+--- zstd --rm --quiet --no-progress middle volume and used existing .1 in -a arg ---
+> pmdumplog ...
+> pminfo ...
+> pmprobe in the middle ...
+> pmval & pmval -r ...
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.1
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.1
+> pmie ...
+
+--- zstd --rm --quiet --no-progress first, middle and last volume and use .meta in -a arg ---
+> pmdumplog ...
+> pminfo ...
+> pmprobe in the middle ...
+> pmval & pmval -r ...
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.meta
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.meta
+> pmie ...
+
+--- zstd --rm --quiet --no-progress first few, middle and last few volumes and use existing .7.zst in -a arg ---
+> pmdumplog ...
+> pminfo ...
+> pmprobe in the middle ...
+> pmval & pmval -r ...
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.7.zst
+< archive:   tmparch/mv-bigbin
+---
+> archive:   mv-bigbin.7.zst
+> pmie ...
+
+--- some error cases ---
+pminfo: Cannot open archive "mv-bigbin.10": No such file or directory
+pmprobe: Cannot open archive "mv-bigbin.10": No such file or directory
+pmval: Cannot open archive "mv-bigbin.10": No such file or directory
+pmie: cannot open archive mv-bigbin.10
+pmNewContext failed: No such file or directory
+pminfo: Cannot open archive "mv-bigbin.10.zst": No such file or directory
+pmprobe: Cannot open archive "mv-bigbin.10.zst": No such file or directory
+pmval: Cannot open archive "mv-bigbin.10.zst": No such file or directory
+pmie: cannot open archive mv-bigbin.10.zst
+pmNewContext failed: No such file or directory
+--- compressed empty data volume ---
+pminfo: Cannot open archive "null": Empty archive file
+--- empty data volume pretending to be compressed ---
+zstd: ./null.0.zst: unexpected end of file 
+pminfo: Cannot open archive "null": Corrupted record in a PCP archive

qa/1571

@@ -0,0 +1,115 @@
+#!/bin/sh
+# PCP QA Test No. 1571
+# pmlogger_rewrite with xz compression
+# ... based on No. 676
+#
+# Copyright (c) 2018,2024 Ken McDonell.  All Rights Reserved.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+if which xz >/dev/null 2>&1
+then
+    PROG="xz -0 --block-size=10MiB"
+    SUFF=xz
+else
+    _notrun "cannot find a xz compression program!"
+fi
+
+_cleanup()
+{
+    cd $here
+    $sudo rm -rf $tmp $tmp.*
+}
+
+_filter()
+{
+    sed \
+	-e "s@$tmp@TMP@g" \
+    # end
+}
+
+status=1	# failure is the default!
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# setup the log farm
+#
+cat <<End-of-File >$tmp.farm
+onevol		archives/snort-disks	20011002
+onevol		archives/moomba.client	20011003
+multivol	archives/ok-mv-bar	20011002
+End-of-File
+
+mkdir $tmp
+if ./mk.logfarm -c $tmp.farm $tmp >$tmp.out 2>&1
+then
+    _filter <$tmp.out
+else
+    cat $tmp.out
+    echo "Arrgh ... mk.logfarm failed!"
+    exit
+fi
+
+# real QA test starts here
+cd $tmp
+
+echo
+echo "--- rewrite with data volume compression for multivol ---"
+cat <<End-of-File >$tmp.rewrite
+metric pmcd.pmlogger.port {
+    pmid -> 2.4095.1023
+}
+End-of-File
+for file in multivol/*.[0-9]
+do
+    $PROG $file
+done
+$PCP_BINADM_DIR/pmlogger_rewrite -VV -c $tmp.rewrite onevol/*.meta multivol/*.meta
+for arch in onevol/*.meta multivol/*.meta
+do
+    arch=`echo $arch | sed -e 's/.meta//'`
+    echo $arch:
+    pminfo -a $arch -m pmcd.pmlogger.port
+done
+rm -rf onevol multivol
+cd $here
+if ./mk.logfarm -c $tmp.farm $tmp >$tmp.out 2>&1
+then
+    :
+else
+    cat $tmp.out
+    echo "Arrgh ... mk.logfarm failed!"
+    exit
+fi
+cd $tmp
+
+echo
+echo "--- rewrite with metadata and volume compression for onevol and multivol ---"
+cat <<End-of-File >$tmp.rewrite
+metric pmcd.pmlogger.port {
+    pmid -> 2.4095.1023
+}
+End-of-File
+for file in multivol/*.[0-9] multivol/*.meta onevol/*.[0-9] onevol/*.meta
+do
+    $PROG $file
+done
+$PCP_BINADM_DIR/pmlogger_rewrite -VV -c $tmp.rewrite onevol/*.meta* multivol/*.meta*
+for arch in onevol/*.meta* multivol/*.meta*
+do
+    arch=`echo $arch | sed -e 's/.meta.*//'`
+    echo $arch:
+    pminfo -a $arch -m pmcd.pmlogger.port
+done
+cd $tmp
+
+# success, all done
+status=0
+exit

qa/1571.out

@@ -0,0 +1,41 @@
+QA output created by 1571
+
+--- rewrite with data volume compression for multivol ---
+suffix stripping: arg: onevol/20011002.meta -> onevol/20011002
+suffix stripping: arg: onevol/20011003.meta -> onevol/20011003
+suffix stripping: arg: multivol/20011002.meta -> multivol/20011002
+changed: onevol/20011002.0
+changed: onevol/20011002.meta
+changed: onevol/20011003.0
+changed: onevol/20011003.meta
+changed and recompressed: multivol/20011002.0.xz
+changed and recompressed: multivol/20011002.1.xz
+changed and recompressed: multivol/20011002.2.xz
+changed and recompressed: multivol/20011002.3.xz
+changed: multivol/20011002.meta
+onevol/20011002:
+pmcd.pmlogger.port PMID: 2.4095.1023
+onevol/20011003:
+pmcd.pmlogger.port PMID: 2.4095.1023
+multivol/20011002:
+pmcd.pmlogger.port PMID: 2.4095.1023
+
+--- rewrite with metadata and volume compression for onevol and multivol ---
+suffix stripping: arg: onevol/20011002.meta.xz -> onevol/20011002
+suffix stripping: arg: onevol/20011003.meta.xz -> onevol/20011003
+suffix stripping: arg: multivol/20011002.meta.xz -> multivol/20011002
+changed and recompressed: onevol/20011002.0.xz
+changed and recompressed: onevol/20011002.meta.xz
+changed and recompressed: onevol/20011003.0.xz
+changed and recompressed: onevol/20011003.meta.xz
+changed and recompressed: multivol/20011002.0.xz
+changed and recompressed: multivol/20011002.1.xz
+changed and recompressed: multivol/20011002.2.xz
+changed and recompressed: multivol/20011002.3.xz
+changed and recompressed: multivol/20011002.meta.xz
+onevol/20011002:
+pmcd.pmlogger.port PMID: 2.4095.1023
+onevol/20011003:
+pmcd.pmlogger.port PMID: 2.4095.1023
+multivol/20011002:
+pmcd.pmlogger.port PMID: 2.4095.1023