Blob Blame History Raw
diff -Naurp pcp-5.0.2.orig/qa/1211.out pcp-5.0.2/qa/1211.out
--- pcp-5.0.2.orig/qa/1211.out	2019-12-06 15:18:26.000000000 +1100
+++ pcp-5.0.2/qa/1211.out	2020-02-03 13:23:15.258762963 +1100
@@ -144,6 +144,9 @@ kernel.uname.nodename
 kernel.uname.release
 kernel.uname.sysname
 kernel.uname.version
+pmcd.pmlogger.archive
+pmcd.pmlogger.host
+pmcd.pmlogger.port
 proc.fd.count
 proc.id.egid
 proc.id.egid_nm
@@ -267,6 +270,7 @@ List all instance names ...
 030016 pmlogger -P -c config.default 20110930.17.20
 1 minute
 15 minute
+2950
 5 minute
 cpu0
 cpu1
@@ -398,10 +402,10 @@ fecd5a4b4c6e1273eaa001287a6dd57b7bbd19f7
 Values fetch for a single-valued query ...
 
 d51624d12da45900bfee2fd73f1e23f3ccabb784
-    [Mon Oct  3 09:10:22.959242000 2011] 172598244
-    [Mon Oct  3 09:10:23.300460000 2011] 172598364
-    [Mon Oct  3 09:10:23.802930000 2011] 172598481
     [Mon Oct  3 09:10:24.305845000 2011] 172598559
+    [Mon Oct  3 09:10:23.802930000 2011] 172598481
+    [Mon Oct  3 09:10:23.300460000 2011] 172598364
+    [Mon Oct  3 09:10:22.959242000 2011] 172598244
 
 Values fetch with a one-second interval ...
 
@@ -420,15 +424,18 @@ d51624d12da45900bfee2fd73f1e23f3ccabb784
 Values fetch for a multi-valued query ...
 
 fecd5a4b4c6e1273eaa001287a6dd57b7bbd19f7
-    [Mon Oct  3 09:10:23.300460000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
-    [Mon Oct  3 09:10:23.300460000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
-    [Mon Oct  3 09:10:23.300460000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
-    [Mon Oct  3 09:10:23.802930000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
-    [Mon Oct  3 09:10:23.802930000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
-    [Mon Oct  3 09:10:23.802930000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
     [Mon Oct  3 09:10:24.305845000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
     [Mon Oct  3 09:10:24.305845000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
     [Mon Oct  3 09:10:24.305845000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
+    [Mon Oct  3 09:10:23.802930000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
+    [Mon Oct  3 09:10:23.802930000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
+    [Mon Oct  3 09:10:23.802930000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
+    [Mon Oct  3 09:10:23.300460000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
+    [Mon Oct  3 09:10:23.300460000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
+    [Mon Oct  3 09:10:23.300460000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
+    [Mon Oct  3 09:10:22.959242000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
+    [Mon Oct  3 09:10:22.959242000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
+    [Mon Oct  3 09:10:22.959242000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
 
 Multi-series lookups from a multi-series query ...
 2db1da4d276d81c42c578c2829e99188ae7cc898
diff -Naurp pcp-5.0.2.orig/qa/1573 pcp-5.0.2/qa/1573
--- pcp-5.0.2.orig/qa/1573	1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.0.2/qa/1573	2020-02-03 13:36:17.288581801 +1100
@@ -0,0 +1,103 @@
+#!/bin/sh
+# PCP QA Test No. 1573
+# Exercise libpcp_web memory leak without a redis-server.
+#
+# Copyright (c) 2020 Red Hat.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+_check_series
+
+_cleanup()
+{
+    cd $here
+    if $need_restore
+    then
+	need_restore=false
+	_service pmlogger stop >/dev/null
+	$sudo rm -rf $PCP_LOG_DIR/pmlogger
+	$sudo mv $PCP_LOG_DIR/pmlogger.$seq $PCP_LOG_DIR/pmlogger
+	_restore_config $PCP_ETC_DIR/pcp/pmlogger
+	_service pcp restart 2>&1 | _filter_pcp_stop | _filter_pcp_start
+	_wait_for_pmcd
+	_wait_for_pmlogger
+	echo === restarting pmproxy
+	_restore_config $PCP_SYSCONF_DIR/pmproxy
+	_service pmproxy restart 2>&1 | _filter_pcp_start
+	_wait_for_pmproxy
+    fi
+    $sudo rm -rf $tmp $tmp.*
+}
+
+status=1	# failure is the default!
+need_restore=false
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# real QA test starts here
+_save_config $PCP_SYSCONF_DIR/pmproxy
+need_restore=true
+
+# only want the primary logger running
+_save_config $PCP_ETC_DIR/pcp/pmlogger
+_restore_pmlogger_control
+
+#$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/*
+echo "[pmproxy]" > $tmp.conf
+echo "pcp.enabled = true" >> $tmp.conf
+echo "http.enabled = true" >> $tmp.conf
+echo "redis.enabled = true" >> $tmp.conf
+echo "[discover]" >> $tmp.conf
+echo "enabled = true" >> $tmp.conf
+echo "[pmseries]" >> $tmp.conf
+echo "enabled = false" >> $tmp.conf
+$sudo cp $tmp.conf $PCP_SYSCONF_DIR/pmproxy/pmproxy.conf
+
+_service pmlogger stop >/dev/null
+
+# move aside existing logs so we can measure base memory footprint
+[ -d $PCP_LOG_DIR/pmlogger.$seq ] && $sudo mv $PCP_LOG_DIR/pmlogger.$seq $PCP_LOG_DIR/pmlogger.$seq.saved
+$sudo mv $PCP_LOG_DIR/pmlogger $PCP_LOG_DIR/pmlogger.$seq
+$sudo mkdir -p $PCP_LOG_DIR/pmlogger
+$sudo chmod 775 $PCP_LOG_DIR/pmlogger
+$sudo chown $PCP_USER:$PCP_USER $PCP_LOG_DIR/pmlogger
+
+_service pmproxy restart 2>&1 | _filter_pcp_stop | _filter_pcp_start
+_wait_for_pmproxy
+
+pmproxy_pid=`_get_pids_by_name -a pmproxy`
+[ -z "$pmproxy_pid" ] && echo === pmproxy not running && status=1 && exit 1
+echo === extract initial rss
+pmproxy_rss1=`pminfo -f proc.memory.rss |
+	$PCP_AWK_PROG '{ if ($2 == "['$pmproxy_pid'") { print $NF} }'`
+
+echo === restarting pmlogger # primary only
+_service pmlogger restart 2>&1 | _filter_pcp_start
+_wait_for_pmlogger
+
+echo === wait for pmproxy to process filesystem events
+pmsleep 4.2
+
+echo === extract updated rss
+pmproxy_rss2=`pminfo -f proc.memory.rss |
+	$PCP_AWK_PROG '{ if ($2 == "['$pmproxy_pid'") { print $NF} }'`
+
+echo === checking rss within tolerance
+_within_tolerance "rss" $pmproxy_rss1 $pmproxy_rss2 10%
+[ $pmproxy_rss2 -gt 10000 ] && echo "Unexpected pmproxy RSS: $pmproxy_rss2, was initially $pmproxy_rss1"
+
+echo "RSS1 for PID $pmproxy_pid is $pmproxy_rss1" >> $here/$seq.full
+echo "RSS2 for PID $pmproxy_pid is $pmproxy_rss2" >> $here/$seq.full
+cat $PCP_LOG_DIR/pmproxy/pmproxy.log >>$seq.full
+echo === see $seq.full for pmproxy rss and logs
+
+# success, all done
+status=0
+exit
diff -Naurp pcp-5.0.2.orig/qa/1573.out pcp-5.0.2/qa/1573.out
--- pcp-5.0.2.orig/qa/1573.out	1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.0.2/qa/1573.out	2020-02-03 13:23:15.259762953 +1100
@@ -0,0 +1,8 @@
+QA output created by 1573
+=== extract initial rss
+=== restarting pmlogger
+=== wait for pmproxy to process filesystem events
+=== extract updated rss
+=== checking rss within tolerance
+=== see 1573.full for pmproxy rss and logs
+=== restarting pmproxy
diff -Naurp pcp-5.0.2.orig/qa/1600 pcp-5.0.2/qa/1600
--- pcp-5.0.2.orig/qa/1600	2019-12-10 17:49:05.000000000 +1100
+++ pcp-5.0.2/qa/1600	2020-02-03 13:23:15.260762942 +1100
@@ -82,7 +82,11 @@ _filter_values()
 _filter_label_values()
 {
     sed \
+	-e "s/^domainname: \"${domainname}\"/domainname: \"DOMAIN\"/g" \
+	-e "s/^machineid: \"${machineid}\"/machineid: \"MACHINE\"/g" \
 	-e "s/^hostname: \"${hostname}\"/hostname: \"HOSTNAME\"/g" \
+	-e "s/^groupid: $groupid/groupid: GID/g" \
+	-e "s/^userid: $userid/userid: UID/g" \
 	-e "s/changed: false, true/changed: false/g" \
 	-e "/metric_label: null/d" \
     #end
diff -Naurp pcp-5.0.2.orig/qa/1600.out pcp-5.0.2/qa/1600.out
--- pcp-5.0.2.orig/qa/1600.out	2019-12-10 10:46:20.000000000 +1100
+++ pcp-5.0.2/qa/1600.out	2020-02-03 13:23:15.260762942 +1100
@@ -27,15 +27,15 @@ TIMESERIES
 == verify metric labels
 
 TIMESERIES
-    inst [100 or "bin-100"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [200 or "bin-200"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [300 or "bin-300"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [400 or "bin-400"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [500 or "bin-500"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [600 or "bin-600"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [700 or "bin-700"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [800 or "bin-800"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
-    inst [900 or "bin-900"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
+    inst [100 or "bin-100"] labels {"agent":"sample","bin":100,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [200 or "bin-200"] labels {"agent":"sample","bin":200,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [300 or "bin-300"] labels {"agent":"sample","bin":300,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [400 or "bin-400"] labels {"agent":"sample","bin":400,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [500 or "bin-500"] labels {"agent":"sample","bin":500,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [600 or "bin-600"] labels {"agent":"sample","bin":600,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [700 or "bin-700"] labels {"agent":"sample","bin":700,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [800 or "bin-800"] labels {"agent":"sample","bin":800,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+    inst [900 or "bin-900"] labels {"agent":"sample","bin":900,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
 == verify metric values
 
 TIMESERIES
@@ -43,15 +43,24 @@ TIMESERIES
     [TIMESTAMP] VALUE
 == verify label names and values
 agent: "mmv", "sample", "pmcd"
+bin: 100, 200, 300, 400, 500, 600, 700, 800, 900
 changed: false
 clan: "mcdonell"
 cluster: "zero"
+domainname: "DOMAIN"
+groupid: GID
 hostname: "HOSTNAME"
 indom_label: 42.001
+latitude: -25.28496
+longitude: 152.87886
+machineid: "MACHINE"
 measure: "speed"
 model: "RGB"
+registry_label: "string"
 role: "testing"
+transient: false, true
 units: "metres per second"
 unitsystem: "SI"
+userid: UID
 == verify archive removal
 == all done
diff -Naurp pcp-5.0.2.orig/qa/1601.out pcp-5.0.2/qa/1601.out
--- pcp-5.0.2.orig/qa/1601.out	2019-11-27 16:01:34.000000000 +1100
+++ pcp-5.0.2/qa/1601.out	2020-02-03 13:23:15.261762932 +1100
@@ -131,7 +131,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
     {
         "series": "605fc77742cd0317597291329561ac4e50c0dd12",
         "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
-        "timestamp": 1317633023300.460,
+        "timestamp": 1317633024305.845,
         "value": "71661"
     },
     {
@@ -147,7 +147,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
         {
             "series": "605fc77742cd0317597291329561ac4e50c0dd12",
             "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
-            "timestamp": 1317633023300.460,
+            "timestamp": 1317633024305.845,
             "value": "71661"
         },
         {
@@ -163,7 +163,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
     {
         "series": "605fc77742cd0317597291329561ac4e50c0dd12",
         "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
-        "timestamp": 1317633023300.460,
+        "timestamp": 1317633024305.845,
         "value": "71661"
     },
     {
@@ -179,7 +179,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
         {
             "series": "605fc77742cd0317597291329561ac4e50c0dd12",
             "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
-            "timestamp": 1317633023300.460,
+            "timestamp": 1317633024305.845,
             "value": "71661"
         },
         {
diff -Naurp pcp-5.0.2.orig/qa/1661 pcp-5.0.2/qa/1661
--- pcp-5.0.2.orig/qa/1661	2019-12-10 17:04:20.000000000 +1100
+++ pcp-5.0.2/qa/1661	2020-02-03 13:23:15.261762932 +1100
@@ -41,8 +41,7 @@ _restore_pmlogger_control
 echo;echo === restarting pmproxy service to ensure sane starting condition 
 _service pmlogger stop 2>&1 | _filter_pcp_stop
 _service pmproxy restart 2>&1 | _filter_pcp_stop | _filter_pcp_start
-# give pmproxy a chance to startup
-pmsleep 2; _wait_for_pmproxy
+_wait_for_pmproxy
 
 pmproxy_pid=`_get_pids_by_name -a pmproxy`
 [ -z "$pmproxy_pid" ] && echo === pmproxy not running && status=1 && exit 1
diff -Naurp pcp-5.0.2.orig/qa/group pcp-5.0.2/qa/group
--- pcp-5.0.2.orig/qa/group	2019-12-11 14:06:06.000000000 +1100
+++ pcp-5.0.2/qa/group	2020-02-03 13:23:15.261762932 +1100
@@ -1688,6 +1688,7 @@ BAD
 1545 pcp2xml python pcp2xxx local
 1546 pmrep python local
 1547 pmrep python local
+1573 pmproxy libpcp_web pmlogger local
 1588 python pmiostat local
 1598 pmda.statsd local
 1599 pmda.statsd local
diff -Naurp pcp-5.0.2.orig/src/include/pcp/libpcp.h pcp-5.0.2/src/include/pcp/libpcp.h
--- pcp-5.0.2.orig/src/include/pcp/libpcp.h	2019-09-24 17:23:36.000000000 +1000
+++ pcp-5.0.2/src/include/pcp/libpcp.h	2020-02-03 13:23:15.261762932 +1100
@@ -7,7 +7,7 @@
  *	remain fixed across releases, and they may not work, or may
  *	provide different semantics at some point in the future.
  *
- * Copyright (c) 2012-2019 Red Hat.
+ * Copyright (c) 2012-2020 Red Hat.
  * Copyright (c) 2008-2009 Aconex.  All Rights Reserved.
  * Copyright (c) 1995-2002 Silicon Graphics, Inc.  All Rights Reserved.
  *
@@ -846,6 +846,13 @@ PCP_CALL extern int __pmLogPutText(__pmA
 PCP_CALL extern int __pmLogWriteLabel(__pmFILE *, const __pmLogLabel *);
 PCP_CALL extern int __pmLogLoadLabel(__pmArchCtl *, const char *);
 PCP_CALL extern int __pmLogLoadMeta(__pmArchCtl *);
+PCP_CALL extern int __pmLogAddDesc(__pmArchCtl *, const pmDesc *);
+PCP_CALL extern int __pmLogAddInDom(__pmArchCtl *, const pmTimespec *, const pmInResult *, int *, int);
+PCP_CALL extern int __pmLogAddPMNSNode(__pmArchCtl *, pmID, const char *);
+PCP_CALL extern int __pmLogAddLabelSets(__pmArchCtl *, const pmTimespec *, unsigned int, unsigned int, int, pmLabelSet *);
+PCP_CALL extern int __pmLogAddText(__pmArchCtl *, unsigned int, unsigned int, const char *);
+PCP_CALL extern int __pmLogAddVolume(__pmArchCtl *, unsigned int);
+
 #define PMLOGREAD_NEXT		0
 #define PMLOGREAD_TO_EOF	1
 PCP_CALL extern int __pmLogRead(__pmArchCtl *, int, __pmFILE *, pmResult **, int);
@@ -862,7 +869,9 @@ PCP_CALL extern int __pmLogLookupText(__
 PCP_CALL extern int __pmLogNameInDom(__pmArchCtl *, pmInDom, pmTimeval *, int, char **);
 PCP_CALL extern const char *__pmLogLocalSocketDefault(int, char *buf, size_t bufSize);
 PCP_CALL extern const char *__pmLogLocalSocketUser(int, char *buf, size_t bufSize);
+PCP_CALL extern int __pmLogCompressedSuffix(const char *);
 PCP_CALL extern char *__pmLogBaseName(char *);
+PCP_CALL extern char *__pmLogBaseNameVol(char *, int *);
 PCP_DATA extern int __pmLogReads;
 
 /* Convert opaque context handle to __pmContext pointer */
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/exports.master pcp-5.0.2/src/libpcp/src/exports.master
--- pcp-5.0.2.orig/src/libpcp/src/exports.master	2019-10-02 14:40:30.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/exports.master	2020-02-03 13:23:15.262762921 +1100
@@ -683,3 +683,15 @@ PCP_3.26 {
   global:
     __pmDupLabelSets;
 } PCP_3.25;
+
+PCP_3.26_1 {
+  global:
+    __pmLogAddDesc;
+    __pmLogAddInDom;
+    __pmLogAddPMNSNode;
+    __pmLogAddLabelSets;
+    __pmLogAddText;
+    __pmLogAddVolume;
+    __pmLogCompressedSuffix;
+    __pmLogBaseNameVol;
+} PCP_3.26;
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/io.c pcp-5.0.2/src/libpcp/src/io.c
--- pcp-5.0.2.orig/src/libpcp/src/io.c	2018-06-09 11:43:34.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/io.c	2020-02-03 13:23:15.262762921 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2018 Red Hat.
+ * Copyright (c) 2017-2018,2020 Red Hat.
  * 
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -47,7 +47,7 @@ extern __pm_fops __pm_xz;
 #endif
 
 static const struct {
-    const char	*suff;
+    const char	*suffix;
     const int	appl;
     __pm_fops   *handler;
 } compress_ctl[] = {
@@ -61,40 +61,43 @@ static const struct {
 };
 static const int ncompress = sizeof(compress_ctl) / sizeof(compress_ctl[0]);
 
+int
+__pmLogCompressedSuffix(const char *suffix)
+{
+    int		i;
+
+    for (i = 0; i < ncompress; i++)
+	if (strcmp(suffix, compress_ctl[i].suffix) == 0)
+	    return 1;
+    return 0;
+}
+
 /*
- * If name contains '.' and the suffix is "index", "meta" or a string of
- * digits, all optionally followed by one of the compression suffixes,
- * strip the suffix.
- *
- * Modifications are performed on the argument string in-place. If modifications
- * are made, a pointer to the start of the modified string is returned.
- * Otherwise, NULL is returned.
+ * Variant of __pmLogBaseName() - see below that also returns log
+ * the volume number if the file name is an archive log volume.
+ * If the vol argument is NULL it will be ignored.
  */
 char *
-__pmLogBaseName(char *name)
+__pmLogBaseNameVol(char *name, int *vol)
 {
-    char *q;
-    int   strip;
-    int   i;
+    char	*q, *q2;
+    int		strip = 0;
 
-    strip = 0;
+    if (vol)
+	*vol = -1;
     if ((q = strrchr(name, '.')) != NULL) {
-	for (i = 0; i < ncompress; i++) {
-	    if (strcmp(q, compress_ctl[i].suff) == 0) {
-		char	*q2;
-		/*
-		 * The name ends with one of the supported compressed file
-		 * suffixes. Strip it before checking for other known suffixes.
-		 */
-		*q = '\0';
-		if ((q2 = strrchr(name, '.')) == NULL) {
-		    /* no . to the left of the suffix */
-		    *q = '.';
-		    goto done;
-		}
-		q = q2;
-		break;
+	if (__pmLogCompressedSuffix(q)) {
+	    /*
+	     * The name ends with one of the supported compressed file
+	     * suffixes. Strip it before checking for other known suffixes.
+	     */
+	    *q = '\0';
+	    if ((q2 = strrchr(name, '.')) == NULL) {
+		/* no . to the left of the suffix */
+		*q = '.';
+		goto done;
 	    }
+	    q = q2;
 	}
 	if (strcmp(q, ".index") == 0) {
 	    strip = 1;
@@ -109,16 +112,10 @@ __pmLogBaseName(char *name)
 	 */
 	if (q[1] != '\0') {
 	    char	*end;
-	    /*
-	     * Below we don't care about the value from strtol(),
-	     * we're interested in updating the pointer "end".
-	     * The messiness is thanks to gcc and glibc ... strtol()
-	     * is marked __attribute__((warn_unused_result)) ...
-	     * to avoid warnings on all platforms, assign to a
-	     * dummy variable that is explicitly marked unused.
-	     */
-	    long	tmpl __attribute__((unused));
+	    long	tmpl;
 	    tmpl = strtol(q+1, &end, 10);
+	    if (vol)
+		*vol = tmpl;
 	    if (*end == '\0') strip = 1;
 	}
     }
@@ -131,6 +128,21 @@ done:
     return NULL; /* not the name of an archive file. */
 }
 
+/*
+ * If name contains '.' and the suffix is "index", "meta" or a string of
+ * digits, all optionally followed by one of the compression suffixes,
+ * strip the suffix.
+ *
+ * Modifications are performed on the argument string in-place. If modifications
+ * are made, a pointer to the start of the modified string is returned.
+ * Otherwise, NULL is returned.
+ */
+char *
+__pmLogBaseName(char *name)
+{
+    return __pmLogBaseNameVol(name, NULL);
+}
+
 static int
 popen_uncompress(const char *cmd, const char *arg, const char *fname, int fd)
 {
@@ -319,7 +331,7 @@ __pmCompressedFileIndex(char *fname, siz
     char	tmpname[MAXPATHLEN];
 
     for (i = 0; i < ncompress; i++) {
-	suffix = compress_ctl[i].suff;
+	suffix = compress_ctl[i].suffix;
 	pmsprintf(tmpname, sizeof(tmpname), "%s%s", fname, suffix);
 	sts = access(tmpname, R_OK);
 	if (sts == 0 || (errno != ENOENT && errno != ENOTDIR)) {
@@ -358,7 +370,7 @@ index_compress(char *fname, size_t flen)
     suffix = strrchr(fname, '.');
     if (suffix != NULL) {
 	for (i = 0; i < ncompress; i++) {
-	    if (strcmp(suffix, compress_ctl[i].suff) == 0)
+	    if (strcmp(suffix, compress_ctl[i].suffix) == 0)
 		return i;
 	}
     }
@@ -731,7 +743,7 @@ compress_suffix_list(void)
 	const char	*q;
 
 	for (i = 0; i < ncompress; i++) {
-	    q = compress_ctl[i].suff;
+	    q = compress_ctl[i].suffix;
 	    if (i > 0)
 		*p++ = ' ';
 	    while (*q) {
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/logmeta.c pcp-5.0.2/src/libpcp/src/logmeta.c
--- pcp-5.0.2.orig/src/libpcp/src/logmeta.c	2018-09-14 10:22:56.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/logmeta.c	2020-02-03 13:23:15.262762921 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2018 Red Hat.
+ * Copyright (c) 2013-2018, 2020 Red Hat.
  * Copyright (c) 1995-2002 Silicon Graphics, Inc.  All Rights Reserved.
  * 
  * This library is free software; you can redistribute it and/or modify it
@@ -490,7 +490,7 @@ check_dup_labels(const __pmArchCtl *acp)
 }
 
 static int
-addtext(__pmArchCtl *acp, unsigned int ident, unsigned int type, char *buffer)
+addtext(__pmArchCtl *acp, unsigned int ident, unsigned int type, const char *buffer)
 {
     __pmLogCtl		*lcp = acp->ac_log;
     __pmHashNode	*hp;
@@ -553,6 +553,92 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":15",
     return sts;
 }
 
+int
+__pmLogAddDesc(__pmArchCtl *acp, const pmDesc *newdp)
+{
+    __pmHashNode	*hp;
+    __pmLogCtl		*lcp = acp->ac_log;
+    pmDesc		*dp, *olddp;
+
+    if ((hp = __pmHashSearch((int)newdp->pmid, &lcp->l_hashpmid)) != NULL) {
+	/* PMID is already in the hash table - check for conflicts. */
+	olddp = (pmDesc *)hp->data;
+	if (newdp->type != olddp->type)
+	    return PM_ERR_LOGCHANGETYPE;
+	if (newdp->sem != olddp->sem)
+	    return PM_ERR_LOGCHANGESEM;
+	if (newdp->indom != olddp->indom)
+	    return PM_ERR_LOGCHANGEINDOM;
+	if (newdp->units.dimSpace != olddp->units.dimSpace ||
+	    newdp->units.dimTime != olddp->units.dimTime ||
+	    newdp->units.dimCount != olddp->units.dimCount ||
+	    newdp->units.scaleSpace != olddp->units.scaleSpace ||
+	    newdp->units.scaleTime != olddp->units.scaleTime ||
+	    newdp->units.scaleCount != olddp->units.scaleCount)
+	    return PM_ERR_LOGCHANGEUNITS;
+
+	/* PMID is already known and checks out - we're done here. */
+	return 0;
+    }
+
+    /* Add a copy of the descriptor into the PMID:desc hash table. */
+PM_FAULT_POINT("libpcp/" __FILE__ ":2", PM_FAULT_ALLOC);
+    if ((dp = (pmDesc *)malloc(sizeof(pmDesc))) == NULL)
+	return -oserror();
+    *dp = *newdp;
+
+    return __pmHashAdd((int)dp->pmid, (void *)dp, &lcp->l_hashpmid);
+}
+
+int
+__pmLogAddPMNSNode(__pmArchCtl *acp, pmID pmid, const char *name)
+{
+    __pmLogCtl		*lcp = acp->ac_log;
+    int			sts;
+
+    /*
+     * If we see a duplicate name with a different PMID, its a
+     * recoverable error.
+     * We wont be able to see all of the data in the log, but
+     * its better to provide access to some rather than none,
+     * esp. when only one or two metric IDs may be corrupted
+     * in this way (which we may not be interested in anyway).
+     */
+    sts = __pmAddPMNSNode(lcp->l_pmns, pmid, name);
+    if (sts == PM_ERR_PMID)
+	sts = 0;
+    return sts;
+}
+
+int
+__pmLogAddInDom(__pmArchCtl *acp, const pmTimespec *when, const pmInResult *in,
+		int *tbuf, int allinbuf)
+{
+    pmTimeval		tv;
+
+    tv.tv_sec = when->tv_sec;
+    tv.tv_usec = when->tv_nsec / 1000;
+    return addindom(acp->ac_log, in->indom, &tv,
+		    in->numinst, in->instlist, in->namelist, tbuf, allinbuf);
+}
+
+int
+__pmLogAddLabelSets(__pmArchCtl *acp, const pmTimespec *when, unsigned int type,
+		unsigned int ident, int nsets, pmLabelSet *labelsets)
+{
+    pmTimeval		tv;
+
+    tv.tv_sec = when->tv_sec;
+    tv.tv_usec = when->tv_nsec / 1000;
+    return addlabel(acp, type, ident, nsets, labelsets, &tv);
+}
+
+int
+__pmLogAddText(__pmArchCtl *acp, unsigned int ident, unsigned int type, const char *buffer)
+{
+    return addtext(acp, ident, type, buffer);
+}
+
 /*
  * Load _all_ of the hashed pmDesc and __pmLogInDom structures from the metadata
  * log file -- used at the initialization (NewContext) of an archive.
@@ -563,11 +649,8 @@ int
 __pmLogLoadMeta(__pmArchCtl *acp)
 {
     __pmLogCtl		*lcp = acp->ac_log;
-    __pmHashNode	*hp;
     int			rlen;
     int			check;
-    pmDesc		*dp;
-    pmDesc		*olddp;
     int			sts = 0;
     __pmLogHdr		h;
     __pmFILE		*f = lcp->l_mdfp;
@@ -615,13 +698,10 @@ __pmLogLoadMeta(__pmArchCtl *acp)
 	}
 	rlen = h.len - (int)sizeof(__pmLogHdr) - (int)sizeof(int);
 	if (h.type == TYPE_DESC) {
+	    pmDesc		desc;
+
 	    numpmid++;
-PM_FAULT_POINT("libpcp/" __FILE__ ":2", PM_FAULT_ALLOC);
-	    if ((dp = (pmDesc *)malloc(sizeof(pmDesc))) == NULL) {
-		sts = -oserror();
-		goto end;
-	    }
-	    if ((n = (int)__pmFread(dp, 1, sizeof(pmDesc), f)) != sizeof(pmDesc)) {
+	    if ((n = (int)__pmFread(&desc, 1, sizeof(pmDesc), f)) != sizeof(pmDesc)) {
 		if (pmDebugOptions.logmeta) {
 		    fprintf(stderr, "__pmLogLoadMeta: pmDesc read -> %d: expected: %d\n",
 			    n, (int)sizeof(pmDesc));
@@ -632,67 +712,25 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
 		}
 		else
 		    sts = PM_ERR_LOGREC;
-		free(dp);
 		goto end;
 	    }
-	    else {
-		/* swab desc */
-		dp->type = ntohl(dp->type);
-		dp->sem = ntohl(dp->sem);
-		dp->indom = __ntohpmInDom(dp->indom);
-		dp->units = __ntohpmUnits(dp->units);
-		dp->pmid = __ntohpmID(dp->pmid);
-	    }
 
-	    /* Add it to the hash pmid hash table. */
-	    if ((hp = __pmHashSearch((int)dp->pmid, &lcp->l_hashpmid)) != NULL) {
-		/*
-		 * This pmid is already in the hash table. Check for conflicts.
-		 */
-		olddp = (pmDesc *)hp->data;
-		if (dp->type != olddp->type) {
-		    sts = PM_ERR_LOGCHANGETYPE;
-		    free(dp);
-		    goto end;
-		}
-		if (dp->sem != olddp->sem) {
-		    sts = PM_ERR_LOGCHANGESEM;
-		    free(dp);
-		    goto end;
-		}
-		if (dp->indom != olddp->indom) {
-		    sts = PM_ERR_LOGCHANGEINDOM;
-		    free(dp);
-		    goto end;
-		}
-		if (dp->units.dimSpace != olddp->units.dimSpace ||
-		    dp->units.dimTime != olddp->units.dimTime ||
-		    dp->units.dimCount != olddp->units.dimCount ||
-		    dp->units.scaleSpace != olddp->units.scaleSpace ||
-		    dp->units.scaleTime != olddp->units.scaleTime ||
-		    dp->units.scaleCount != olddp->units.scaleCount) {
-		    sts = PM_ERR_LOGCHANGEUNITS;
-		    free(dp);
-		    goto end;
-		}
-                /*
-                 * This pmid is already known, and matches.  We can free the newly
-                 * read copy and use the one in the hash table. 
-                 */
-                free(dp);
-                dp = olddp;
-	    }
-	    else if ((sts = __pmHashAdd((int)dp->pmid, (void *)dp, &lcp->l_hashpmid)) < 0) {
-		free(dp);
+	    /* swab desc */
+	    desc.type = ntohl(desc.type);
+	    desc.sem = ntohl(desc.sem);
+	    desc.indom = __ntohpmInDom(desc.indom);
+	    desc.units = __ntohpmUnits(desc.units);
+	    desc.pmid = __ntohpmID(desc.pmid);
+
+	    if ((sts = __pmLogAddDesc(acp, &desc)) < 0)
 		goto end;
-	    }
 
 	    /* read in the names & store in PMNS tree ... */
 	    if ((n = (int)__pmFread(&numnames, 1, sizeof(numnames), f)) != 
 		sizeof(numnames)) {
 		if (pmDebugOptions.logmeta) {
-		    fprintf(stderr, "__pmLogLoadMeta: numnames read -> %d: expected: %d\n",
-			    n, (int)sizeof(numnames));
+		    fprintf(stderr, "%s: numnames read -> %d: expected: %d\n",
+			    "__pmLogLoadMeta", n, (int)sizeof(numnames));
 		}
 		if (__pmFerror(f)) {
 		    __pmClearerr(f);
@@ -711,8 +749,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
 		if ((n = (int)__pmFread(&len, 1, sizeof(len), f)) != 
 		    sizeof(len)) {
 		    if (pmDebugOptions.logmeta) {
-			fprintf(stderr, "__pmLogLoadMeta: len name[%d] read -> %d: expected: %d\n",
-				i, n, (int)sizeof(len));
+			fprintf(stderr, "%s: len name[%d] read -> %d: expected: %d\n",
+				"__pmLogLoadMeta", i, n, (int)sizeof(len));
 		    }
 		    if (__pmFerror(f)) {
 			__pmClearerr(f);
@@ -729,8 +767,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
 
 		if ((n = (int)__pmFread(name, 1, len, f)) != len) {
 		    if (pmDebugOptions.logmeta) {
-			fprintf(stderr, "__pmLogLoadMeta: name[%d] read -> %d: expected: %d\n",
-				i, n, len);
+			fprintf(stderr, "%s: name[%d] read -> %d: expected: %d\n",
+				"__pmLogLoadMeta", i, n, len);
 		    }
 		    if (__pmFerror(f)) {
 			__pmClearerr(f);
@@ -743,36 +781,23 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
 		name[len] = '\0';
 		if (pmDebugOptions.logmeta) {
 		    char	strbuf[20];
-		    fprintf(stderr, "__pmLogLoadMeta: PMID: %s name: %s\n",
-			    pmIDStr_r(dp->pmid, strbuf, sizeof(strbuf)), name);
+		    fprintf(stderr, "%s: PMID: %s name: %s\n",
+			    "__pmLogLoadMeta",
+			    pmIDStr_r(desc.pmid, strbuf, sizeof(strbuf)), name);
 		}
-		/* Add the new PMNS node */
-		if ((sts = __pmAddPMNSNode(lcp->l_pmns, dp->pmid, name)) < 0) {
-		    /*
-		     * If we see a duplicate name with a different PMID, its a
-		     * recoverable error.
-		     * We wont be able to see all of the data in the log, but
-		     * its better to provide access to some rather than none,
-		     * esp. when only one or two metric IDs may be corrupted
-		     * in this way (which we may not be interested in anyway).
-		     */
-		    if (sts != PM_ERR_PMID)
-			goto end;
-		} 
+
+		/* Add the new PMNS node into this context */
+		if ((sts = __pmLogAddPMNSNode(acp, desc.pmid, name)) < 0)
+		    goto end;
 	    }/*for*/
 	}
 	else if (h.type == TYPE_INDOM) {
-	    int			*tbuf;
-	    pmInDom		indom;
-	    pmTimeval		*when;
-	    int			numinst;
-	    int			*instlist;
-	    char		**namelist;
+	    pmTimeval		*tv;
+	    pmTimespec		when;
+	    pmInResult		in;
 	    char		*namebase;
-	    int			*stridx;
-	    int			i;
-	    int			k;
-	    int			allinbuf = 0;
+	    int			*tbuf, *stridx;
+	    int			i, k, allinbuf = 0;
 
 PM_FAULT_POINT("libpcp/" __FILE__ ":3", PM_FAULT_ALLOC);
 	    if ((tbuf = (int *)malloc(rlen)) == NULL) {
@@ -781,8 +806,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":3",
 	    }
 	    if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) {
 		if (pmDebugOptions.logmeta) {
-		    fprintf(stderr, "__pmLogLoadMeta: indom read -> %d: expected: %d\n",
-			    n, rlen);
+		    fprintf(stderr, "%s: indom read -> %d: expected: %d\n",
+			    "__pmLogLoadMeta", n, rlen);
 		}
 		if (__pmFerror(f)) {
 		    __pmClearerr(f);
@@ -795,44 +820,44 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":3",
 	    }
 
 	    k = 0;
-	    when = (pmTimeval *)&tbuf[k];
-	    when->tv_sec = ntohl(when->tv_sec);
-	    when->tv_usec = ntohl(when->tv_usec);
-	    k += sizeof(*when)/sizeof(int);
-	    indom = __ntohpmInDom((unsigned int)tbuf[k++]);
-	    numinst = ntohl(tbuf[k++]);
-	    if (numinst > 0) {
-		instlist = &tbuf[k];
-		k += numinst;
+	    tv = (pmTimeval *)&tbuf[k];
+	    when.tv_sec = ntohl(tv->tv_sec);
+	    when.tv_nsec = ntohl(tv->tv_usec) * 1000;
+	    k += sizeof(*tv)/sizeof(int);
+	    in.indom = __ntohpmInDom((unsigned int)tbuf[k++]);
+	    in.numinst = ntohl(tbuf[k++]);
+	    if (in.numinst > 0) {
+		in.instlist = &tbuf[k];
+		k += in.numinst;
 		stridx = &tbuf[k];
 #if defined(HAVE_32BIT_PTR)
-		namelist = (char **)stridx;
+		in.namelist = (char **)stridx;
 		allinbuf = 1; /* allocation is all in tbuf */
 #else
 		allinbuf = 0; /* allocation for namelist + tbuf */
 		/* need to allocate to hold the pointers */
 PM_FAULT_POINT("libpcp/" __FILE__ ":4", PM_FAULT_ALLOC);
-		namelist = (char **)malloc(numinst*sizeof(char*));
-		if (namelist == NULL) {
+		in.namelist = (char **)malloc(in.numinst * sizeof(char*));
+		if (in.namelist == NULL) {
 		    sts = -oserror();
 		    free(tbuf);
 		    goto end;
 		}
 #endif
-		k += numinst;
+		k += in.numinst;
 		namebase = (char *)&tbuf[k];
-	        for (i = 0; i < numinst; i++) {
-		    instlist[i] = ntohl(instlist[i]);
-	            namelist[i] = &namebase[ntohl(stridx[i])];
+	        for (i = 0; i < in.numinst; i++) {
+		    in.instlist[i] = ntohl(in.instlist[i]);
+	            in.namelist[i] = &namebase[ntohl(stridx[i])];
 		}
-		if ((sts = addindom(lcp, indom, when, numinst, instlist, namelist, tbuf, allinbuf)) < 0)
+		if ((sts = __pmLogAddInDom(acp, &when, &in, tbuf, allinbuf)) < 0)
 		    goto end;
 		/* If this indom was a duplicate, then we need to free tbuf and
 		   namelist, as appropriate. */
 		if (sts == PMLOGPUTINDOM_DUP) {
 		    free(tbuf);
-		    if (namelist != NULL && !allinbuf)
-			free(namelist);
+		    if (in.namelist != NULL && !allinbuf)
+			free(in.namelist);
 		}
 	    }
 	    else {
@@ -860,8 +885,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11",
 	    }
 	    if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) {
 		if (pmDebugOptions.logmeta) {
-		    fprintf(stderr, "__pmLogLoadMeta: label read -> %d: expected: %d\n",
-			    n, rlen);
+		    fprintf(stderr, "%s: label read -> %d: expected: %d\n",
+			    "__pmLogLoadMeta", n, rlen);
 		}
 		if (__pmFerror(f)) {
 		    __pmClearerr(f);
@@ -908,7 +933,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11",
 
 		if (jsonlen < 0 || jsonlen > PM_MAXLABELJSONLEN) {
 		    if (pmDebugOptions.logmeta)
-			fprintf(stderr, "__pmLogLoadMeta: corrupted json in labelset. jsonlen=%d\n", jsonlen);
+			fprintf(stderr, "%s: corrupted json in labelset. jsonlen=%d\n",
+					"__pmLogLoadMeta", jsonlen);
 		    sts = PM_ERR_LOGREC;
 		    free(labelsets);
 		    free(tbuf);
@@ -935,7 +961,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11",
 		    if (nlabels > PM_MAXLABELS || k + nlabels * sizeof(pmLabel) > rlen) {
 			/* corrupt archive metadata detected. GH #475 */
 			if (pmDebugOptions.logmeta)
-			    fprintf(stderr, "__pmLogLoadMeta: corrupted labelset. nlabels=%d\n", nlabels);
+			    fprintf(stderr, "%s: corrupted labelset. nlabels=%d\n",
+					    "__pmLogLoadMeta", nlabels);
 			sts = PM_ERR_LOGREC;
 			free(labelsets);
 			free(tbuf);
@@ -975,8 +1002,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16",
 	    }
 	    if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) {
 		if (pmDebugOptions.logmeta) {
-		    fprintf(stderr, "__pmLogLoadMeta: text read -> %d: expected: %d\n",
-			    n, rlen);
+		    fprintf(stderr, "%s: text read -> %d: expected: %d\n",
+				    "__pmLogLoadMeta", n, rlen);
 		}
 		if (__pmFerror(f)) {
 		    __pmClearerr(f);
@@ -1005,8 +1032,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16",
 		ident = __ntohpmID(*((unsigned int *)&tbuf[k]));
 	    else {
 		if (pmDebugOptions.logmeta) {
-		    fprintf(stderr, "__pmLogLoadMeta: bad text ident -> %x\n",
-			    type);
+		    fprintf(stderr, "%s: bad text ident -> %x\n",
+				    "__pmLogLoadMeta", type);
 		}
 		free(tbuf);
 		continue;
@@ -1024,8 +1051,9 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16",
 	check = ntohl(check);
 	if (n != sizeof(check) || h.len != check) {
 	    if (pmDebugOptions.logmeta) {
-		fprintf(stderr, "__pmLogLoadMeta: trailer read -> %d or len=%d: expected %d @ offset=%d\n",
-		    n, check, h.len, (int)(__pmFtell(f) - sizeof(check)));
+		fprintf(stderr, "%s: trailer read -> %d or len=%d: "
+				"expected %d @ offset=%d\n", "__pmLogLoadMeta",
+			n, check, h.len, (int)(__pmFtell(f) - sizeof(check)));
 	    }
 	    if (__pmFerror(f)) {
 		__pmClearerr(f);
@@ -1046,7 +1074,7 @@ end:
     if (sts == 0) {
 	if (numpmid == 0) {
 	    if (pmDebugOptions.logmeta) {
-		fprintf(stderr, "__pmLogLoadMeta: no metrics found?\n");
+		fprintf(stderr, "%s: no metrics found?\n", "__pmLogLoadMeta");
 	    }
 	    sts = PM_ERR_LOGREC;
 	}
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/logutil.c pcp-5.0.2/src/libpcp/src/logutil.c
--- pcp-5.0.2.orig/src/libpcp/src/logutil.c	2018-07-08 10:58:08.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/logutil.c	2020-02-03 13:23:15.263762911 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2017 Red Hat.
+ * Copyright (c) 2012-2017,2020 Red Hat.
  * Copyright (c) 1995-2002,2004 Silicon Graphics, Inc.  All Rights Reserved.
  * 
  * This library is free software; you can redistribute it and/or modify it
@@ -764,6 +764,22 @@ __pmLogClose(__pmArchCtl *acp)
 }
 
 int
+__pmLogAddVolume(__pmArchCtl *acp, unsigned int vol)
+{
+    __pmLogCtl	*lcp = acp->ac_log;
+
+    if (lcp->l_minvol == -1) {
+	lcp->l_minvol = vol;
+	lcp->l_maxvol = vol;
+    } else if (vol < lcp->l_minvol) {
+	lcp->l_minvol = vol;
+    } else if (vol > lcp->l_maxvol) {
+	lcp->l_maxvol = vol;
+    }
+    return 0;
+}
+
+int
 __pmLogLoadLabel(__pmArchCtl *acp, const char *name)
 {
     __pmLogCtl	*lcp = acp->ac_log;
@@ -876,21 +892,14 @@ __pmLogLoadLabel(__pmArchCtl *acp, const
 		}
 	    }
 	    else {
-		char	*q;
-		int	vol;
-		vol = (int)strtol(tp, &q, 10);
+		char		*q;
+		unsigned int	vol;
+
+		vol = (unsigned int)strtoul(tp, &q, 10);
 		if (*q == '\0') {
 		    exists = 1;
-		    if (lcp->l_minvol == -1) {
-			lcp->l_minvol = vol;
-			lcp->l_maxvol = vol;
-		    }
-		    else {
-			if (vol < lcp->l_minvol)
-			    lcp->l_minvol = vol;
-			if (vol > lcp->l_maxvol)
-			    lcp->l_maxvol = vol;
-		    }
+		    if ((sts = __pmLogAddVolume(acp, vol)) < 0)
+			goto cleanup;
 		}
 	    }
 	}
@@ -2282,7 +2291,7 @@ __pmLogSetTime(__pmContext *ctxp)
 	int		match = 0;
 	int		vol;
 	int		numti = lcp->l_numti;
-	__pmFILE		*f;
+	__pmFILE	*f;
 	__pmLogTI	*tip = lcp->l_ti;
 	double		t_lo;
 	struct stat	sbuf;
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/discover.c pcp-5.0.2/src/libpcp_web/src/discover.c
--- pcp-5.0.2.orig/src/libpcp_web/src/discover.c	2019-12-10 17:04:20.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/discover.c	2020-02-03 13:36:11.958637560 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2019 Red Hat.
+ * Copyright (c) 2018-2020 Red Hat.
  *
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -14,6 +14,8 @@
 #include "discover.h"
 #include "slots.h"
 #include "util.h"
+#include <dirent.h>
+#include <sys/stat.h>
 
 /* Decode various archive metafile records (desc, indom, labels, helptext) */
 static int pmDiscoverDecodeMetaDesc(uint32_t *, int, pmDesc *, int *, char ***);
@@ -24,11 +26,15 @@ static int pmDiscoverDecodeMetaLabelSet(
 /* array of registered callbacks, see pmDiscoverSetup() */
 static int discoverCallBackTableSize;
 static pmDiscoverCallBacks **discoverCallBackTable;
+static char *pmDiscoverFlagsStr(pmDiscover *);
 
 /* internal hash table of discovered paths */
-#define PM_DISCOVER_HASHTAB_SIZE 64
+#define PM_DISCOVER_HASHTAB_SIZE 16
 static pmDiscover *discover_hashtable[PM_DISCOVER_HASHTAB_SIZE];
 
+/* pmlogger_daily log-roll lock count */
+static int lockcnt = 0;
+
 /* FNV string hash algorithm. Return unsigned in range 0 .. limit-1 */
 static unsigned int
 strhash(const char *s, unsigned int limit)
@@ -43,18 +49,38 @@ strhash(const char *s, unsigned int limi
     return h % limit;
 }
 
+/* ctime string - note static buf is returned */
+static char *
+stamp(void)
+{
+    time_t now = time(NULL);
+    char *p, *c = ctime(&now);
+
+    if ((p = strrchr(c, '\n')) != NULL)
+    	*p = '\0';
+    return c;
+}
+
 /*
- * Lookup or Add a discovered file path (directory or PCP archive file)
+ * Lookup or Add a discovered file path (directory or PCP archive file).
+ * Note: the fullpath suffix (.meta, .[0-9]+) should already be stripped.
  * Return path table entry (new or existing).
  */
 static pmDiscover *
-pmDiscoverLookupAdd(const char *path, pmDiscoverModule *module, void *arg)
+pmDiscoverLookupAdd(const char *fullpath, pmDiscoverModule *module, void *arg)
 {
     pmDiscover		*p, *h;
-    unsigned int	k = strhash(path, PM_DISCOVER_HASHTAB_SIZE);
+    unsigned int	k;
+    sds			name;
+
+    name = sdsnew(fullpath);
+    k = strhash(name, PM_DISCOVER_HASHTAB_SIZE);
+
+    if (pmDebugOptions.discovery)
+	fprintf(stderr, "pmDiscoverLookupAdd: name=%s\n", name);
 
     for (p = NULL, h = discover_hashtable[k]; h != NULL; p = h, h = h->next) {
-    	if (strcmp(h->context.name, path) == 0)
+    	if (sdscmp(h->context.name, name) == 0)
 	    break;
     }
 
@@ -65,14 +91,24 @@ pmDiscoverLookupAdd(const char *path, pm
 	h->ctx = -1; /* no PMAPI context initially */
 	h->flags = PM_DISCOVER_FLAGS_NEW;
 	h->context.type = PM_CONTEXT_ARCHIVE;
-	h->context.name = sdsnew(path);
+	h->context.name = name;
 	h->module = module;
 	h->data = arg;
 	if (p == NULL)
 	    discover_hashtable[k] = h;
 	else
 	    p->next = h;
+	if (pmDebugOptions.discovery)
+	    fprintf(stderr, "pmDiscoverLookupAdd: --> new entry %s\n", name);
+
+    }
+    else {
+	/* already in hash table, so free the buffer */
+	if (pmDebugOptions.discovery)
+	    fprintf(stderr, "pmDiscoverLookupAdd: --> existing entry %s\n", name);
+    	sdsfree(name);
     }
+
     return h;
 }
 
@@ -82,12 +118,6 @@ pmDiscoverLookup(const char *path)
     return pmDiscoverLookupAdd(path, NULL, NULL);
 }
 
-static pmDiscover *
-pmDiscoverAdd(const char *path, pmDiscoverModule *module, void *arg)
-{
-    return pmDiscoverLookupAdd(path, module, arg);
-}
-
 static void
 pmDiscoverFree(pmDiscover *p)
 {
@@ -101,39 +131,42 @@ pmDiscoverFree(pmDiscover *p)
 	sdsfree(p->context.source);
     if (p->context.labelset)
 	pmFreeLabelSets(p->context.labelset, 1);
+    if (p->event_handle) {
+	uv_fs_event_stop(p->event_handle);
+	free(p->event_handle);
+	p->event_handle = NULL;
+    }
+
     memset(p, 0, sizeof(*p));
     free(p);
 }
 
 /*
- * Delete tracking of a previously discovered path. Frees resources and
- * destroy PCP context (if any).
+ * Traverse and invoke callback for all paths matching any bit
+ * in the flags bitmap. Callback can be NULL to just get a count.
+ * Return count of matching paths, may be 0.
  */
-static void
-pmDiscoverDelete(sds path)
+static int
+pmDiscoverTraverse(unsigned int flags, void (*callback)(pmDiscover *))
 {
-    pmDiscover		*p, *h;
-    unsigned int	k = strhash(path, PM_DISCOVER_HASHTAB_SIZE);
+    int			count = 0, i;
+    pmDiscover		*p;
 
-    for (p = NULL, h = discover_hashtable[k]; h != NULL; p = h, h = h->next) {
-    	if (sdscmp(h->context.name, path) == 0) {
-	    if (p == NULL)
-	    	discover_hashtable[k] = NULL;
-	    else
-	    	p->next = h->next;
-	    pmDiscoverFree(h);
-	    break;
+    for (i = 0; i < PM_DISCOVER_HASHTAB_SIZE; i++) {
+    	for (p = discover_hashtable[i]; p; p = p->next) {
+	    if (p->flags & flags) {
+		if (callback)
+		    callback(p);
+		count++;
+	    }
 	}
     }
+    return count;
 }
 
-/*
- * Traverse and invoke callback for all paths matching any bit
- * in the flags bitmap. Callback can be NULL to just get a count.
- * Return count of matching paths, may be 0.
- */
+/* as above, but with an extra (void *)arg passed to the cb */
 static int
-pmDiscoverTraverse(unsigned int flags, void (*callback)(pmDiscover *))
+pmDiscoverTraverseArg(unsigned int flags, void (*callback)(pmDiscover *, void *), void *arg)
 {
     int			count = 0, i;
     pmDiscover		*p;
@@ -142,7 +175,7 @@ pmDiscoverTraverse(unsigned int flags, v
     	for (p = discover_hashtable[i]; p; p = p->next) {
 	    if (p->flags & flags) {
 		if (callback)
-		    callback(p);
+		    callback(p, arg);
 		count++;
 	    }
 	}
@@ -150,6 +183,7 @@ pmDiscoverTraverse(unsigned int flags, v
     return count;
 }
 
+
 /*
  * Traverse and purge deleted entries
  * Return count of purged entries.
@@ -173,6 +207,9 @@ pmDiscoverPurgeDeleted(void)
 		    prev->next = next;
 		else
 		    discover_hashtable[i] = next;
+		if (pmDebugOptions.discovery)
+		    fprintf(stderr, "pmDiscoverPurgeDeleted: deleted %s %s\n",
+		    	p->context.name, pmDiscoverFlagsStr(p));
 		pmDiscoverFree(p);
 		count++;
 	    }
@@ -180,14 +217,32 @@ pmDiscoverPurgeDeleted(void)
 	}
     }
 
-    if (pmDebugOptions.discovery)
-	fprintf(stderr, "%s: purged %d entries\n",
-			"pmDiscoverPurgeDeleted", count);
-
     return count;
 }
 
 /*
+ * if string ends with given suffix then return pointer
+ * to start of suffix in string, else NULL
+ */
+static char *
+strsuffix(char *s, const char *suffix)
+{
+    int slen, suflen;
+    char *ret = NULL;
+
+    if (s && suffix) {
+    	slen = strlen(s);
+	suflen = strlen(suffix);
+	if (slen >= suflen) {
+	    ret = s + (slen - suflen);
+	    if (strncmp(ret, suffix, suflen) != 0)
+	    	ret = NULL;
+	}
+    }
+    return ret;
+}
+
+/*
  * Discover dirs and archives - add new entries or refresh existing.
  * Call this for each top-level directory. Discovered paths are not
  * automatically monitored. After discovery, need to traverse and
@@ -196,44 +251,88 @@ pmDiscoverPurgeDeleted(void)
 static int
 pmDiscoverArchives(const char *dir, pmDiscoverModule *module, void *arg)
 {
-    uv_fs_t		sreq, req;
-    uv_dirent_t		dent;
-    uv_stat_t		*s;
+    DIR			*dirp;
+    struct dirent	*dent;
+    struct stat		*s;
+    struct stat		statbuf;
     pmDiscover		*a;
+    char		*suffix;
     char		path[MAXNAMELEN];
-    char		basepath[MAXNAMELEN];
     int			sep = pmPathSeparator();
+    int			vol;
+
+    /*
+     * note: pmDiscoverLookupAdd sets PM_DISCOVER_FLAGS_NEW
+     * if this is a newly discovered archive or directory
+     */
+    a = pmDiscoverLookupAdd(dir, module, arg);
+    a->flags |= PM_DISCOVER_FLAGS_DIRECTORY;
 
-    if (uv_fs_scandir(NULL, &req, dir, 0, NULL) < 0)
+    if ((dirp = opendir(dir)) == NULL) {
+	if (pmDebugOptions.discovery)
+	    fprintf(stderr, "pmDiscoverArchives: opendir %s failed %s: err %d\n", dir, path, errno);
 	return -ESRCH;
+    }
 
-    a = pmDiscoverAdd(dir, module, arg);
-    a->flags |= PM_DISCOVER_FLAGS_DIRECTORY;
+    while ((dent = readdir(dirp)) != NULL) {
+	if (dent->d_name[0] == '.')
+	    continue;
+	pmsprintf(path, sizeof(path), "%s%c%s", dir, sep, dent->d_name);
+
+	if (pmDebugOptions.discovery)
+	    fprintf(stderr, "pmDiscoverArchives: readdir found %s\n", path);
 
-    while (uv_fs_scandir_next(&req, &dent) != UV_EOF) {
-	pmsprintf(path, sizeof(path), "%s%c%s", dir, sep, dent.name);
-	if (uv_fs_stat(NULL, &sreq, path, NULL) < 0)
+	if (stat(path, &statbuf) < 0) {
+	    if (pmDebugOptions.discovery)
+		fprintf(stderr, "pmDiscoverArchives: stat failed %s, err %d\n", path, errno);
 	    continue;
-	s = &sreq.statbuf;
-	strncpy(basepath, path, sizeof(basepath)); /* __pmLogBaseName modifies it's argument */
-	if (S_ISREG(s->st_mode) && __pmLogBaseName(basepath) != NULL) {
-	    /*
-	     * An archive file (index, meta or data vol). If compressed, then
-	     * it is read-only and we don't have to monitor it for growth.
-	     */
-	    a = pmDiscoverAdd(path, module, arg);
-	    a->flags &= ~PM_DISCOVER_FLAGS_DELETED;
+	}
 
-	    if (strstr(path, ".meta"))
-	    	a->flags |= PM_DISCOVER_FLAGS_META;
-	    else if (strstr(path, ".index"))
-	    	a->flags |= PM_DISCOVER_FLAGS_INDEX;
-	    else
-	    	a->flags |= PM_DISCOVER_FLAGS_DATAVOL;
-
-	    /* compare to libpcp io.c for suffix list */
-	    if (strstr(path, ".xz") || strstr(path, ".gz"))
-	    	a->flags |= PM_DISCOVER_FLAGS_COMPRESSED;
+	s = &statbuf;
+	if (S_ISREG(s->st_mode)) {
+	    if ((suffix = strsuffix(path, ".meta")) != NULL) {
+		/*
+		 * An uncompressed PCP archive meta file. Track the meta
+		 * file - the matching logvol filename varies because logvols
+		 * are periodically rolled by pmlogger. Importantly, process all
+		 * available metadata to EOF before processing any logvol data.
+		 */
+		*suffix = '\0'; /* strip suffix from path giving archive name */
+		a = pmDiscoverLookupAdd(path, module, arg);
+
+		/*
+		 * note: pmDiscoverLookupAdd sets PM_DISCOVER_FLAGS_NEW
+		 * if this is a newly discovered archive, otherwise we're
+		 * already tracking this archive.
+		 */
+		a->flags |= PM_DISCOVER_FLAGS_META;
+	    }
+	    else if ((suffix = __pmLogBaseNameVol(path, &vol)) != NULL && vol >= 0) {
+		/*
+		 * An archive logvol. This logvol may have been created since
+		 * the context was first opened. Update the context maxvol
+		 * to be sure pmFetchArchive can switch to it in due course.
+		 */
+		if ((a = pmDiscoverLookup(path)) != NULL) {
+		    a->flags |= PM_DISCOVER_FLAGS_DATAVOL;
+		    /* ensure archive context knows about this volume */
+		    if (pmDebugOptions.discovery)
+			fprintf(stderr, "pmDiscoverArchives: found logvol %s %s vol=%d\n",
+			    a->context.name, pmDiscoverFlagsStr(a), vol);
+		    if (a->ctx >= 0 && vol >= 0) {
+			__pmContext *ctxp = __pmHandleToPtr(a->ctx);
+			__pmArchCtl *acp = ctxp->c_archctl;
+
+		    	__pmLogAddVolume(acp, vol);
+			PM_UNLOCK(ctxp->c_lock);
+		    }
+		    if (pmDebugOptions.discovery)
+			fprintf(stderr, "pmDiscoverArchives: added logvol %s %s vol=%d\n",
+			    a->context.name, pmDiscoverFlagsStr(a), vol);
+		}
+	    } else if (pmDebugOptions.discovery) {
+		fprintf(stderr, "pmDiscoverArchives: ignored regular file %s\n", path);
+	    }
 	}
 	else if (S_ISDIR(s->st_mode)) {
 	    /*
@@ -241,29 +340,117 @@ pmDiscoverArchives(const char *dir, pmDi
 	     */
 	    pmDiscoverArchives(path, module, arg);
 	}
-	uv_fs_req_cleanup(&sreq);
     }
-    uv_fs_req_cleanup(&req);
+    if (dirp)
+	closedir(dirp);
 
     /* success */
     return 0;
 }
 
+/*
+ * Return 1 if monitored path has been deleted.
+ * For archives, we only check the meta file because
+ * a logvol can be deleted (e.g. via compression when
+ * the logvol is rolled to a new volume) without
+ * actually deleting the archive.
+ */
+static int
+is_deleted(pmDiscover *p, struct stat *sbuf)
+{
+    int			ret = 0;
+
+    if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+	if (stat(p->context.name, sbuf) < 0)
+	    ret = 1; /* directory has been deleted */
+    }
+
+    if (p->flags & (PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL)) {
+    	sds meta = sdsnew(p->context.name);
+	meta = sdscat(meta, ".meta");
+	if (stat(meta, sbuf) < 0) {
+	    /*
+	     * Archive metadata file has been deleted (or compressed)
+	     * hence consider the archive to be deleted because there
+	     * is no more data to logtail.
+	     */
+	    ret = 1;
+	}
+	sdsfree(meta);
+    }
+
+    if (pmDebugOptions.discovery) {
+	fprintf(stderr, "is_deleted: checking %s (%s) -> %s\n",
+		p->context.name, pmDiscoverFlagsStr(p), ret ? "DELETED" : "no");
+    }
+
+    return ret;
+}
+
+static void
+logdir_is_locked_callBack(pmDiscover *p, void *arg)
+{
+    int			*cntp = (int *)arg;
+    char		sep = pmPathSeparator();
+    char		path[MAXNAMELEN];
+
+    pmsprintf(path, sizeof(path), "%s%c%s", p->context.name, sep, "lock");
+    if (access(path, F_OK) == 0)
+    	(*cntp)++;
+}
+
+static void
+check_deleted(pmDiscover *p)
+{
+    struct stat sbuf;
+    if (!(p->flags & PM_DISCOVER_FLAGS_DELETED) && is_deleted(p, &sbuf))
+    	p->flags |= PM_DISCOVER_FLAGS_DELETED;
+}
+
 static void
 fs_change_callBack(uv_fs_event_t *handle, const char *filename, int events, int status)
 {
     char		buffer[MAXNAMELEN];
     size_t		bytes = sizeof(buffer) - 1;
     pmDiscover		*p;
-    uv_fs_t		sreq;
+    char		*s;
     sds			path;
-    int			path_changed = 0;
+    int			count = 0;
+    struct stat		statbuf;
+
+    /*
+     * check if logs are currently being rolled by pmlogger_daily et al
+     * in any of the directories we are tracking. For mutex, the log control
+     * scripts use a 'lock' file in each directory as it is processed.
+     */
+    pmDiscoverTraverseArg(PM_DISCOVER_FLAGS_DIRECTORY,
+    	logdir_is_locked_callBack, (void *)&count);
+
+    if (lockcnt == 0 && count > 0) {
+	/* log-rolling has started */
+    	fprintf(stderr, "%s discovery callback ignored: log-rolling is now in progress\n", stamp());
+	lockcnt = count;
+	return;
+    }
+
+    if (lockcnt > 0 && count > 0) {
+	/* log-rolling is still in progress */
+	lockcnt = count;
+	return;
+    }
+
+    if (lockcnt > 0 && count == 0) {
+    	/* log-rolling is finished: check what got deleted, and then purge */
+    	fprintf(stderr, "%s discovery callback: finished log-rolling\n", stamp());
+	pmDiscoverTraverse(PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL, check_deleted);
+    }
+    lockcnt = count;
 
     uv_fs_event_getpath(handle, buffer, &bytes);
     path = sdsnewlen(buffer, bytes);
 
     if (pmDebugOptions.discovery) {
-	fprintf(stderr, "%s: event on %s -", "fs_change_callBack", path);
+	fprintf(stderr, "fs_change_callBack: event on %s -", path);
 	if (events & UV_RENAME)
 	    fprintf(stderr, " renamed");
 	if (events & UV_CHANGE)
@@ -271,38 +458,40 @@ fs_change_callBack(uv_fs_event_t *handle
 	fputc('\n', stderr);
     }
 
+    	
     /*
-     * Lookup the path, stat and update it's flags accordingly. If the
-     * path has been deleted, stop it's event monitor and free the req buffer.
-     * Then call the pmDiscovery callback.
+     * Strip ".meta" suffix (if any) and lookup the path. stat and update it's
+     * flags accordingly. If the path has been deleted, stop it's event monitor
+     * and free the req buffer, else call the pmDiscovery callback.
      */
-    if ((p = pmDiscoverLookup(path)) == NULL) {
+    if ((s = strsuffix(path, ".meta")) != NULL)
+	*s = '\0';
+
+    p = pmDiscoverLookup(path);
+    if (p && pmDebugOptions.discovery) {
+	fprintf(stderr, "fs_change_callBack: ---> found entry %s (%s)\n",
+		p->context.name, pmDiscoverFlagsStr(p));
+    }
+
+    if (p == NULL) {
 	if (pmDebugOptions.discovery)
-	    fprintf(stderr, "%s: filename %s lookup failed\n",
-		    "fs_change_callBack", filename);
+	    fprintf(stderr, "fs_change_callBack: %s lookup failed\n", filename);
     }
-    else if (uv_fs_stat(NULL, &sreq, p->context.name, NULL) < 0) {
-    	p->flags |= PM_DISCOVER_FLAGS_DELETED;
-	if (p->event_handle) {
-	    uv_fs_event_stop(p->event_handle);
-	    free(p->event_handle);
-	    p->event_handle = NULL;
-	}
+    else if (is_deleted(p, &statbuf)) {
 	/* path has been deleted. statbuf is invalid */
+    	p->flags |= PM_DISCOVER_FLAGS_DELETED;
 	memset(&p->statbuf, 0, sizeof(p->statbuf));
-	path_changed = 1;
-    }
-    else {
-	/* avoid spurious events. only call the callBack if it really changed */
-	if (p->statbuf.st_mtim.tv_sec != sreq.statbuf.st_mtim.tv_sec ||
-	    p->statbuf.st_mtim.tv_nsec != sreq.statbuf.st_mtim.tv_nsec)
-	    path_changed = 1;
-	p->statbuf = sreq.statbuf; /* struct copy */
-	uv_fs_req_cleanup(&sreq);
+	if (pmDebugOptions.discovery)
+	    fprintf(stderr, "fs_change_callBack: %s (%s) has been deleted",
+	    	p->context.name, pmDiscoverFlagsStr(p));
     }
 
-    if (p && p->changed && path_changed && !(p->flags & PM_DISCOVER_FLAGS_DELETED))
-	p->changed(p);
+    /*
+     * Something in the directory changed - new or deleted archive, or
+     * a tracked archive meta data file or logvolume grew
+     */
+    if (p)
+	p->changed(p); /* returns immediately if PM_DISCOVER_FLAGS_DELETED */
 
     sdsfree(path);
 }
@@ -316,9 +505,14 @@ pmDiscoverMonitor(sds path, void (*callb
 {
     discoverModuleData	*data;
     pmDiscover		*p;
+    sds			eventfilename;
 
-    if ((p = pmDiscoverLookup(path)) == NULL)
+    if ((p = pmDiscoverLookup(path)) == NULL) {
+	if (pmDebugOptions.discovery) {
+	    fprintf(stderr, "pmDiscoverMonitor: lookup failed for %s\n", path);
+	}
 	return -ESRCH;
+    }
     data = getDiscoverModuleData(p->module);
 
     /* save the discovery callback to be invoked */
@@ -330,9 +524,29 @@ pmDiscoverMonitor(sds path, void (*callb
 	 * Start monitoring, using given uv loop. Up to the caller to create
 	 * a PCP PMAPI context and to fetch/logtail in the changed callback.
 	 */
+	eventfilename = sdsnew(p->context.name);
 	uv_fs_event_init(data->events, p->event_handle);
-	uv_fs_event_start(p->event_handle, fs_change_callBack, p->context.name,
+
+	if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+	    uv_fs_event_start(p->event_handle, fs_change_callBack, eventfilename,
+			    UV_FS_EVENT_WATCH_ENTRY);
+	}
+	else {
+	    /*
+	     * Monitor an archive file. This tracks the archive meta file
+	     * but the change callback processes both meta and logvol on
+	     * every callback (meta before logvol).
+	     */
+	    eventfilename = sdscat(eventfilename, ".meta");
+	    uv_fs_event_start(p->event_handle, fs_change_callBack, eventfilename,
 			UV_FS_EVENT_WATCH_ENTRY);
+	}
+
+	if (pmDebugOptions.discovery) {
+	    fprintf(stderr, "pmDiscoverMonitor: added event for %s (%s)\n",
+	    	eventfilename, pmDiscoverFlagsStr(p));
+	}
+	sdsfree(eventfilename);
     }
 
     return 0;
@@ -411,41 +625,23 @@ static void changed_callback(pmDiscover
 static void
 created_callback(pmDiscover *p)
 {
+    if (p->flags & (PM_DISCOVER_FLAGS_COMPRESSED|PM_DISCOVER_FLAGS_INDEX))
+    	return; /* compressed archives don't grow and we ignore archive index files */
+
     if (pmDebugOptions.discovery)
 	fprintf(stderr, "CREATED %s, %s\n", p->context.name, pmDiscoverFlagsStr(p));
 
-    p->flags &= ~PM_DISCOVER_FLAGS_NEW;
-
-    if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED)
-    	return; /* compressed archives don't grow */
-
     if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
 	if (pmDebugOptions.discovery)
 	    fprintf(stderr, "MONITOR directory %s\n", p->context.name);
 	pmDiscoverMonitor(p->context.name, changed_callback);
     }
-
-    if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) {
+    else if (p->flags & (PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL)) {
 	if (pmDebugOptions.discovery)
-	    fprintf(stderr, "MONITOR logvol %s\n", p->context.name);
+	    fprintf(stderr, "MONITOR archive %s\n", p->context.name);
 	pmDiscoverMonitor(p->context.name, changed_callback);
     }
-
-    if (p->flags & PM_DISCOVER_FLAGS_META) {
-	if (pmDebugOptions.discovery)
-	    fprintf(stderr, "MONITOR metadata %s\n", p->context.name);
-	pmDiscoverMonitor(p->context.name, changed_callback);
-    }
-}
-
-static void
-deleted_callback(pmDiscover *p)
-{
-    if (pmDebugOptions.discovery)
-	fprintf(stderr, "DELETED %s (%s)\n", p->context.name,
-			pmDiscoverFlagsStr(p));
-    pmDiscoverDelete(p->context.name);
-    /* p is now no longer valid */
+    p->flags &= ~PM_DISCOVER_FLAGS_NEW;
 }
 
 static void
@@ -509,37 +705,84 @@ static void
 pmDiscoverInvokeMetricCallBacks(pmDiscover *p, pmTimespec *ts, pmDesc *desc,
 		int numnames, char **names)
 {
+    discoverModuleData	*data = getDiscoverModuleData(p->module);
     pmDiscoverCallBacks	*callbacks;
     pmDiscoverEvent	event;
     char		buf[32];
-    int			i;
+    int			i, sts;
 
     if (pmDebugOptions.discovery) {
 	fprintf(stderr, "%s[%s]: %s name%s", "pmDiscoverInvokeMetricCallBacks",
 			timespec_str(ts, buf, sizeof(buf)),
 			p->context.source, numnames > 0 ? " " : "(none)\n");
 	for (i = 0; i < numnames; i++)
-	    printf("\"%s\"%s", names[i], i < numnames - 1 ? ", " : "\n");
+	    fprintf(stderr, "[%u/%u] \"%s\"%s", i+1, numnames, names[i],
+			    i < numnames - 1 ? ", " : "\n");
 	pmPrintDesc(stderr, desc);
 	if (pmDebugOptions.labels)
 	    fprintf(stderr, "context labels %s\n", p->context.labelset->json);
     }
 
+    if (data->pmids) {
+	if (dictFind(data->pmids, &desc->pmid) != NULL)
+	    goto out;	/* metric contains an already excluded PMID */
+	for (i = 0; i < numnames; i++) {
+	    if (regexec(&data->exclude_names, names[i], 0, NULL, 0) == 0)
+		break;
+	}
+	if (i != numnames) {
+	    if (pmDebugOptions.discovery)
+		fprintf(stderr, "%s: excluding metric %s\n",
+				"pmDiscoverInvokeMetricCallBacks", names[i]);
+	    /* add this pmid to the exclusion list and return early */
+	    dictAdd(data->pmids, &desc->pmid, NULL);
+	    goto out;
+	}
+    }
+    if (data->indoms) {
+	if (dictFind(data->indoms, &desc->indom) != NULL)
+	    goto out;	/* metric contains an already excluded InDom */
+    }
+
+    if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+	__pmContext	*ctxp = __pmHandleToPtr(p->ctx);
+	__pmArchCtl	*acp = ctxp->c_archctl;
+	char		idstr[32];
+
+	if ((sts = __pmLogAddDesc(acp, desc)) < 0)
+	    fprintf(stderr, "%s: failed to add metric descriptor for %s\n",
+			    "pmDiscoverInvokeMetricCallBacks",
+			    pmIDStr_r(desc->pmid, idstr, sizeof(idstr)));
+	for (i = 0; i < numnames; i++) {
+	    if ((sts = __pmLogAddPMNSNode(acp, desc->pmid, names[i])) < 0)
+		fprintf(stderr, "%s: failed to add metric name %s for %s\n",
+				"pmDiscoverInvokeMetricCallBacks", names[i],
+				pmIDStr_r(desc->pmid, idstr, sizeof(idstr)));
+	}
+	PM_UNLOCK(ctxp->c_lock);
+    }
+
     discover_event_init(p, ts, &event);
     for (i = 0; i < discoverCallBackTableSize; i++) {
 	if ((callbacks = discoverCallBackTable[i]) &&
 	    callbacks->on_metric != NULL)
 	    callbacks->on_metric(&event, desc, numnames, names, p->data);
     }
+
+out:
+    for (i = 0; i < numnames; i++)
+	free(names[i]);
+    free(names);
 }
 
 static void
 pmDiscoverInvokeInDomCallBacks(pmDiscover *p, pmTimespec *ts, pmInResult *in)
 {
+    discoverModuleData	*data = getDiscoverModuleData(p->module);
     pmDiscoverCallBacks	*callbacks;
     pmDiscoverEvent	event;
     char		buf[32], inbuf[32];
-    int			i;
+    int			i, sts = PMLOGPUTINDOM_DUP; /* free after callbacks */
 
     if (pmDebugOptions.discovery) {
 	fprintf(stderr, "%s[%s]: %s numinst %d indom %s\n",
@@ -551,22 +794,48 @@ pmDiscoverInvokeInDomCallBacks(pmDiscove
 	    fprintf(stderr, "context labels %s\n", p->context.labelset->json);
     }
 
+    if (data->indoms) {
+	if (dictFind(data->indoms, &in->indom) != NULL)
+	    goto out;	/* excluded InDom */
+    }
+
+    if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+	__pmContext	*ctxp = __pmHandleToPtr(p->ctx);
+	__pmArchCtl	*acp = ctxp->c_archctl;
+	char		errmsg[PM_MAXERRMSGLEN];
+
+	if ((sts = __pmLogAddInDom(acp, ts, in, NULL, 0)) < 0)
+	    fprintf(stderr, "%s: failed to add indom for %s: %s\n",
+			"pmDiscoverInvokeInDomCallBacks", pmIDStr(in->indom),
+			pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+	PM_UNLOCK(ctxp->c_lock);
+    }
+
     discover_event_init(p, ts, &event);
     for (i = 0; i < discoverCallBackTableSize; i++) {
 	if ((callbacks = discoverCallBackTable[i]) &&
 	    callbacks->on_indom != NULL)
 	    callbacks->on_indom(&event, in, p->data);
     }
+
+out:
+    if (sts == PMLOGPUTINDOM_DUP) {
+	for (i = 0; i < in->numinst; i++)
+	    free(in->namelist[i]);
+	free(in->namelist);
+	free(in->instlist);
+    }
 }
 
 static void
 pmDiscoverInvokeLabelsCallBacks(pmDiscover *p, pmTimespec *ts,
 		int ident, int type, pmLabelSet *sets, int nsets)
 {
+    discoverModuleData	*data = getDiscoverModuleData(p->module);
     pmDiscoverCallBacks	*callbacks;
     pmDiscoverEvent	event;
     char		buf[32], idbuf[64];
-    int			i;
+    int			i, sts = -EAGAIN; /* free labelsets after callbacks */
 
     if (pmDebugOptions.discovery) {
 	__pmLabelIdentString(ident, type, idbuf, sizeof(idbuf));
@@ -579,22 +848,48 @@ pmDiscoverInvokeLabelsCallBacks(pmDiscov
 	    fprintf(stderr, "context labels %s\n", p->context.labelset->json);
     }
 
+    if ((type & PM_LABEL_ITEM) && data->pmids) {
+	if (dictFind(data->pmids, &ident) != NULL)
+	    goto out;	/* text from an already excluded InDom */
+    }
+    if ((type & (PM_LABEL_INDOM|PM_LABEL_INSTANCES)) && data->indoms) {
+	if (dictFind(data->indoms, &ident) != NULL)
+	    goto out;	/* text from an already excluded InDom */
+    }
+
+    if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+	__pmContext	*ctxp = __pmHandleToPtr(p->ctx);
+	__pmArchCtl	*acp = ctxp->c_archctl;
+	char		errmsg[PM_MAXERRMSGLEN];
+
+	if ((sts = __pmLogAddLabelSets(acp, ts, type, ident, nsets, sets)) < 0)
+	    fprintf(stderr, "%s: failed to add log labelset: %s\n",
+			"pmDiscoverInvokeLabelsCallBacks",
+			pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+	PM_UNLOCK(ctxp->c_lock);
+    }
+
     discover_event_init(p, ts, &event);
     for (i = 0; i < discoverCallBackTableSize; i++) {
 	if ((callbacks = discoverCallBackTable[i]) &&
 	    callbacks->on_labels != NULL)
 	    callbacks->on_labels(&event, ident, type, sets, nsets, p->data);
     }
+
+out:
+    if (sts < 0)
+	pmFreeLabelSets(sets, nsets);
 }
 
 static void
 pmDiscoverInvokeTextCallBacks(pmDiscover *p, pmTimespec *ts,
 		int ident, int type, char *text)
 {
+    discoverModuleData	*data = getDiscoverModuleData(p->module);
     pmDiscoverCallBacks	*callbacks;
     pmDiscoverEvent	event;
     char		buf[32];
-    int			i;
+    int			i, sts;
 
     if (pmDebugOptions.discovery) {
 	fprintf(stderr, "%s[%s]: %s ", "pmDiscoverInvokeTextCallBacks",
@@ -612,12 +907,36 @@ pmDiscoverInvokeTextCallBacks(pmDiscover
 	    fprintf(stderr, "context labels %s\n", p->context.labelset->json);
     }
 
+    if ((type & PM_TEXT_PMID) && data->pmids) {
+	if (dictFind(data->pmids, &ident) != NULL)
+	    goto out;	/* text from an already excluded InDom */
+    }
+    if ((type & PM_TEXT_INDOM) && data->indoms) {
+	if (dictFind(data->indoms, &ident) != NULL)
+	    goto out;	/* text from an already excluded InDom */
+    }
+
+    if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+	__pmContext	*ctxp = __pmHandleToPtr(p->ctx);
+	__pmArchCtl	*acp = ctxp->c_archctl;
+	char		errmsg[PM_MAXERRMSGLEN];
+
+	if ((sts = __pmLogAddText(acp, ident, type, text)) < 0)
+	    fprintf(stderr, "%s: failed to add %u text for %u: %s\n",
+	               "pmDiscoverInvokeTextCallBacks", type, ident,
+			pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+	PM_UNLOCK(ctxp->c_lock);
+    }
+
     discover_event_init(p, ts, &event);
     for (i = 0; i < discoverCallBackTableSize; i++) {
 	if ((callbacks = discoverCallBackTable[i]) &&
 	    callbacks->on_text != NULL)
 	    callbacks->on_text(&event, ident, type, text, p->data);
     }
+
+out:
+    free(text);
 }
 
 static void
@@ -645,8 +964,8 @@ pmDiscoverNewSource(pmDiscover *p, int c
     p->context.labelset = labelset;
 
     /* use timestamp from file creation as starting time */
-    timestamp.tv_sec = p->statbuf.st_birthtim.tv_sec;
-    timestamp.tv_nsec = p->statbuf.st_birthtim.tv_nsec;
+    timestamp.tv_sec = p->statbuf.st_ctim.tv_sec;
+    timestamp.tv_nsec = p->statbuf.st_ctim.tv_nsec;
 
     /* inform utilities that a source has been discovered */
     pmDiscoverInvokeSourceCallBacks(p, &timestamp);
@@ -664,7 +983,7 @@ process_metadata(pmDiscover *p)
     pmDesc		desc;
     off_t		off;
     char		*buffer;
-    int			e, i, nb, len, nsets;
+    int			e, nb, len, nsets;
     int			type, id; /* pmID or pmInDom */
     int			nnames;
     char		**names;
@@ -674,6 +993,8 @@ process_metadata(pmDiscover *p)
     __pmLogHdr		hdr;
     sds			msg, source;
     static uint32_t	*buf = NULL;
+    int			deleted;
+    struct stat		sbuf;
     static int		buflen = 0;
 
     /*
@@ -683,14 +1004,17 @@ process_metadata(pmDiscover *p)
      */
     p->flags |= PM_DISCOVER_FLAGS_META_IN_PROGRESS;
     if (pmDebugOptions.discovery)
-	fprintf(stderr, "%s: in progress, flags=%s\n",
-			"process_metadata", pmDiscoverFlagsStr(p));
+	fprintf(stderr, "process_metadata: %s in progress %s\n",
+		p->context.name, pmDiscoverFlagsStr(p));
     for (;;) {
 	off = lseek(p->fd, 0, SEEK_CUR);
 	nb = read(p->fd, &hdr, sizeof(__pmLogHdr));
 
-	if (nb <= 0) {
-	    /* we're at EOF or an error. But may still be part way through a record */
+	deleted = is_deleted(p, &sbuf);
+	if (nb <= 0 || deleted) {
+	    /* we're at EOF or an error, or deleted. But may still be part way through a record */
+	    if (deleted)
+	    	p->flags |= PM_DISCOVER_FLAGS_DELETED;
 	    break;
 	}
 
@@ -750,10 +1074,6 @@ process_metadata(pmDiscover *p)
 	    ts.tv_sec = p->statbuf.st_mtim.tv_sec;
 	    ts.tv_nsec = p->statbuf.st_mtim.tv_nsec;
 	    pmDiscoverInvokeMetricCallBacks(p, &ts, &desc, nnames, names);
-	    for (i = 0; i < nnames; i++)
-		free(names[i]);
-	    if (names)
-		free(names);
 	    break;
 
 	case TYPE_INDOM:
@@ -765,12 +1085,6 @@ process_metadata(pmDiscover *p)
 		break;
 	    }
 	    pmDiscoverInvokeInDomCallBacks(p, &ts, &inresult);
-	    if (inresult.numinst > 0) {
-		for (i = 0; i < inresult.numinst; i++)
-		    free(inresult.namelist[i]);
-		free(inresult.namelist);
-		free(inresult.instlist);
-	    }
 	    break;
 
 	case TYPE_LABEL:
@@ -795,13 +1109,13 @@ process_metadata(pmDiscover *p)
 		} else {
 		    sdsfree(p->context.source);
 		    p->context.source = source;
-		    p->context.labelset = labelset;
+		    if (p->context.labelset)
+			pmFreeLabelSets(p->context.labelset, 1);
+		    p->context.labelset = __pmDupLabelSets(labelset, 1);
 		    pmDiscoverInvokeSourceCallBacks(p, &ts);
 		}
 	    }
 	    pmDiscoverInvokeLabelsCallBacks(p, &ts, id, type, labelset, nsets);
-	    if (labelset != p->context.labelset)
-		pmFreeLabelSets(labelset, nsets);
 	    break;
 
 	case TYPE_TEXT:
@@ -819,8 +1133,6 @@ process_metadata(pmDiscover *p)
 	    ts.tv_sec = p->statbuf.st_mtim.tv_sec;
 	    ts.tv_nsec = p->statbuf.st_mtim.tv_nsec;
 	    pmDiscoverInvokeTextCallBacks(p, &ts, id, type, buffer);
-	    if (buffer)
-		free(buffer);
 	    break;
 
 	default:
@@ -833,38 +1145,89 @@ process_metadata(pmDiscover *p)
     }
 
     if (partial == 0)
-	/* flag that all available metadata has been now been read */
+	/* flag that all available metadata has now been read */
 	p->flags &= ~PM_DISCOVER_FLAGS_META_IN_PROGRESS;
 
     if (pmDebugOptions.discovery)
-	fprintf(stderr, "%s : completed, partial=%d flags=%s\n",
-			"process_metadata", partial, pmDiscoverFlagsStr(p));
+	fprintf(stderr, "%s: completed, partial=%d %s %s\n",
+			"process_metadata", partial, p->context.name, pmDiscoverFlagsStr(p));
 }
 
 /*
- * fetch metric values to EOF and call all registered callbacks
+ * Fetch metric values to EOF and call all registered callbacks.
+ * Always process metadata thru to EOF before any logvol data.
  */
 static void
-process_logvol_callback(pmDiscover *p)
+process_logvol(pmDiscover *p)
 {
+    int			sts;
     pmResult		*r;
     pmTimespec		ts;
+    int			oldcurvol;
+    __pmContext		*ctxp;
+    __pmArchCtl		*acp;
+
+    for (;;) {
+	pmUseContext(p->ctx);
+	ctxp = __pmHandleToPtr(p->ctx);
+	acp = ctxp->c_archctl;
+	oldcurvol = acp->ac_curvol;
+	PM_UNLOCK(ctxp->c_lock);
+
+	if ((sts = pmFetchArchive(&r)) < 0) {
+	    /* err handling to skip to the next vol */
+	    ctxp = __pmHandleToPtr(p->ctx);
+	    acp = ctxp->c_archctl;
+	    if (oldcurvol < acp->ac_curvol) {
+	    	__pmLogChangeVol(acp, acp->ac_curvol);
+		acp->ac_offset = 0; /* __pmLogFetch will fix it up */
+	    }
+	    PM_UNLOCK(ctxp->c_lock);
+
+	    if (sts == PM_ERR_EOL) {
+		if (pmDebugOptions.discovery)
+		    fprintf(stderr, "process_logvol: %s end of archive reached\n",
+		    	p->context.name);
+
+		/* succesfully processed to current end of log */
+		break;
+	    } else {
+		/* 
+		 * This log vol was probably deleted (likely compressed)
+		 * under our feet. Try and skip to the next volume.
+		 * We hold the context lock during error recovery here.
+		 */
+		if (pmDebugOptions.discovery)
+		    fprintf(stderr, "process_logvol: %s fetch failed:%s\n",
+			p->context.name, pmErrStr(sts));
+	    }
 
-    pmUseContext(p->ctx);
-    while (pmFetchArchive(&r) == 0) {
+	    /* we are done - return and wait for another callback */
+	    break;
+	}
+
+	/*
+	 * Fetch succeeded - call the values callback and continue
+	 */
 	if (pmDebugOptions.discovery) {
 	    char		tbuf[64], bufs[64];
 
-	    fprintf(stderr, "FETCHED @%s [%s] %d metrics\n",
-		    timeval_str(&r->timestamp, tbuf, sizeof(tbuf)),
+	    fprintf(stderr, "process_logvol: %s FETCHED @%s [%s] %d metrics\n",
+		    p->context.name, timeval_str(&r->timestamp, tbuf, sizeof(tbuf)),
 		    timeval_stream_str(&r->timestamp, bufs, sizeof(bufs)),
 		    r->numpmid);
 	}
+
+	/*
+	 * TODO: persistently save current timestamp, so after being restarted,
+	 * pmproxy can resume where it left off for each archive.
+	 */
 	ts.tv_sec = r->timestamp.tv_sec;
 	ts.tv_nsec = r->timestamp.tv_usec * 1000;
 	pmDiscoverInvokeValuesCallBack(p, &ts, r);
 	pmFreeResult(r);
     }
+
     /* datavol is now up-to-date and at EOF */
     p->flags &= ~PM_DISCOVER_FLAGS_DATAVOL_READY;
 }
@@ -874,12 +1237,13 @@ pmDiscoverInvokeCallBacks(pmDiscover *p)
 {
     int			sts;
     sds			msg;
+    sds			metaname;
 
     if (p->ctx < 0) {
 	/*
 	 * once off initialization on the first event
 	 */
-	if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) {
+	if (p->flags & (PM_DISCOVER_FLAGS_DATAVOL | PM_DISCOVER_FLAGS_META)) {
 	    struct timeval	tvp;
 
 	    /* create the PMAPI context (once off) */
@@ -898,28 +1262,25 @@ pmDiscoverInvokeCallBacks(pmDiscover *p)
 		p->ctx = -1;
 		return;
 	    }
+	    /* seek to end of archive for logvol data - see TODO in process_logvol() */
 	    pmSetMode(PM_MODE_FORW, &tvp, 1);
-	    /* note: we do not scan pre-existing logvol data. */
-	}
-	else if (p->flags & PM_DISCOVER_FLAGS_META) {
-	    if ((sts = pmNewContext(p->context.type, p->context.name)) < 0) {
-		infofmt(msg, "pmNewContext failed for %s: %s\n",
-				p->context.name, pmErrStr(sts));
-		moduleinfo(p->module, PMLOG_ERROR, msg, p->data);
-		return;
-	    }
-	    pmDiscoverNewSource(p, sts);
 
-	    /* for archive meta files, p->fd is the direct file descriptor */
-	    if ((p->fd = open(p->context.name, O_RDONLY)) < 0) {
-		infofmt(msg, "open failed for %s: %s\n", p->context.name,
-				osstrerror());
+	    /*
+	     * For archive meta files, p->fd is the direct file descriptor
+	     * and we pre-scan existing metadata. Note: we do NOT scan
+	     * pre-existing logvol data (see pmSetMode above)
+	     */
+	    metaname = sdsnew(p->context.name);
+	    metaname = sdscat(metaname, ".meta");
+	    if ((p->fd = open(metaname, O_RDONLY)) < 0) {
+		infofmt(msg, "open failed for %s: %s\n", metaname, osstrerror());
 		moduleinfo(p->module, PMLOG_ERROR, msg, p->data);
+		sdsfree(metaname);
 		return;
 	    }
-
-	    /* process all existing metadata */
+	    /* pre-process all existing metadata */
 	    process_metadata(p);
+	    sdsfree(metaname);
 	}
     }
 
@@ -943,15 +1304,61 @@ pmDiscoverInvokeCallBacks(pmDiscover *p)
     }
 
     if (p->flags & PM_DISCOVER_FLAGS_META) {
-	/* process metadata */
+	/* process new metadata, if any */
 	process_metadata(p);
     }
 
-    /* process any unprocessed datavol callbacks */
-    pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL_READY, process_logvol_callback);
+    if ((p->flags & PM_DISCOVER_FLAGS_META_IN_PROGRESS) == 0) {
+	/* no metdata read in progress, so process new datavol data, if any */
+	process_logvol(p);
+    }
+}
+
+static void
+print_callback(pmDiscover *p)
+{
+    if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+	fprintf(stderr, "    DIRECTORY %s %s\n",
+	    p->context.name, pmDiscoverFlagsStr(p));
+    }
+    else {
+	__pmContext *ctxp;
+	__pmArchCtl *acp;
 
-    /* finally, purge deleted entries, if any */
-    pmDiscoverPurgeDeleted();
+	if (p->ctx >= 0 && (ctxp = __pmHandleToPtr(p->ctx)) != NULL) {
+	    acp = ctxp->c_archctl;
+	    fprintf(stderr, "    ARCHIVE %s fd=%d ctx=%d maxvol=%d ac_curvol=%d ac_offset=%ld %s\n",
+		p->context.name, p->fd, p->ctx, acp->ac_log->l_maxvol, acp->ac_curvol,
+		acp->ac_offset, pmDiscoverFlagsStr(p));
+	    PM_UNLOCK(ctxp->c_lock);
+	} else {
+	    /* no context yet - probably PM_DISCOVER_FLAGS_NEW */
+	    fprintf(stderr, "    ARCHIVE %s fd=%d ctx=%d %s\n",
+		p->context.name, p->fd, p->ctx, pmDiscoverFlagsStr(p));
+	}
+    }
+}
+
+/*
+ * p is a tracked archive and arg is a directory path.
+ * If p is in the directory, call it's callbacks to
+ * process metadata and logvol data. This allows better
+ * scalability because we only process archives in the
+ * directories that have changed.
+ */
+static void
+directory_changed_cb(pmDiscover *p, void *arg)
+{
+    char *dirpath = (char *)arg;
+    int dlen = strlen(dirpath);
+
+    if (strncmp(p->context.name, dirpath, dlen) == 0) {
+    	/* this archive is in this directory - process it's metadata and logvols */
+	if (pmDebugOptions.discovery)
+	    fprintf(stderr, "directory_changed_cb: archive %s is in dir %s\n",
+		p->context.name, dirpath);
+	pmDiscoverInvokeCallBacks(p);
+    }
 }
 
 static void
@@ -962,27 +1369,46 @@ changed_callback(pmDiscover *p)
 			pmDiscoverFlagsStr(p));
 
     if (p->flags & PM_DISCOVER_FLAGS_DELETED) {
-	/* path or directory has been deleted - remove from hash table */
-	deleted_callback(p);
-    }
-    else if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
 	/*
-	 * A changed directory path means a new archive or subdirectory
-	 * has been created - traverse and update the hash table.
+	 * Path has been deleted. Do nothing for now. Will be purged
+	 * in due course by pmDiscoverPurgeDeleted.
 	 */
-	pmDiscoverArchives(p->context.name, p->module, p->data);
-	pmDiscoverTraverse(PM_DISCOVER_FLAGS_NEW, created_callback);
+	return;
+	
     }
-    else if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) {
+
+    if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) {
     	/* we do not monitor compressed files - do nothing */
-	; /**/
+	return;
     }
-    else if (p->flags & (PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META)) {
-    	/*
-	 * We only monitor uncompressed logvol and metadata paths. Fetch new data
-	 * (metadata or logvol) and call the registered callbacks.
+
+    if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+	/*
+	 * A changed directory path means a new archive or subdirectory may have
+	 * been created or deleted - traverse and update the hash table.
 	 */
-	pmDiscoverInvokeCallBacks(p);
+	if (pmDebugOptions.discovery) {
+	    fprintf(stderr, "%s DIRECTORY CHANGED %s (%s)\n",
+	    	stamp(), p->context.name, pmDiscoverFlagsStr(p));
+	}
+	pmDiscoverArchives(p->context.name, p->module, p->data);
+	pmDiscoverTraverse(PM_DISCOVER_FLAGS_NEW, created_callback);
+
+	/*
+	 * Walk directory and invoke callbacks for tracked archives in this
+	 * directory that have changed
+	 */
+	pmDiscoverTraverseArg(PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META,
+	    directory_changed_cb, (void *)p->context.name);
+
+	/* finally, purge deleted entries (globally), if any */
+	pmDiscoverPurgeDeleted();
+    }
+
+    if (pmDebugOptions.discovery) {
+	fprintf(stderr, "%s -- tracking status\n", stamp());
+	pmDiscoverTraverse(PM_DISCOVER_FLAGS_ALL, print_callback);
+	fprintf(stderr, "--\n");
     }
 }
 
@@ -995,18 +1421,9 @@ dir_callback(pmDiscover *p)
 static void
 archive_callback(pmDiscover *p)
 {
-    if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED)
-    	return; /* compressed archives don't grow */
-
-    if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) {
-	if (pmDebugOptions.discovery)
-	    fprintf(stderr, "DISCOVERED ARCHIVE LOGVOL %s\n", p->context.name);
-	pmDiscoverMonitor(p->context.name, changed_callback);
-    }
-
     if (p->flags & PM_DISCOVER_FLAGS_META) {
 	if (pmDebugOptions.discovery)
-	    fprintf(stderr, "DISCOVERED ARCHIVE METADATA %s\n", p->context.name);
+	    fprintf(stderr, "DISCOVERED ARCHIVE %s\n", p->context.name);
 	pmDiscoverMonitor(p->context.name, changed_callback);
     }
 }
@@ -1048,9 +1465,9 @@ pmDiscoverRegister(const char *dir, pmDi
     }
 
     if (pmDebugOptions.discovery) {
-	fprintf(stderr, "Now managing %d directories and %d archive files\n",
+	fprintf(stderr, "Now tracking %d directories and %d archives\n",
 	    pmDiscoverTraverse(PM_DISCOVER_FLAGS_DIRECTORY, NULL),
-	    pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL, NULL));
+	    pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META, NULL));
     }
 
     /* monitor the directories */
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/discover.h pcp-5.0.2/src/libpcp_web/src/discover.h
--- pcp-5.0.2.orig/src/libpcp_web/src/discover.h	2019-12-10 17:04:20.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/discover.h	2020-02-03 13:36:09.904659047 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2019 Red Hat.
+ * Copyright (c) 2018-2020 Red Hat.
  *
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -18,7 +18,9 @@
 #include "libpcp.h"
 #include "mmv_stats.h"
 #include "slots.h"
-
+#ifdef HAVE_REGEX_H
+#include <regex.h>
+#endif
 #ifdef HAVE_LIBUV
 #include <uv.h>
 #else
@@ -84,8 +86,8 @@ typedef struct pmDiscover {
     int				fd;		/* meta file descriptor */
 #ifdef HAVE_LIBUV
     uv_fs_event_t		*event_handle;	/* uv fs_notify event handle */ 
-    uv_stat_t			statbuf;	/* stat buffer from event CB */
 #endif
+    struct stat			statbuf;	/* stat buffer */
     void			*baton;		/* private internal lib data */
     void			*data;		/* opaque user data pointer */
 } pmDiscover;
@@ -115,6 +117,10 @@ typedef struct discoverModuleData {
     struct dict			*config;	/* configuration dict */
     uv_loop_t			*events;	/* event library loop */
     redisSlots			*slots;		/* server slots data */
+    regex_t			exclude_names;	/* metric names to exclude */
+    struct dict			*pmids;		/* dict of excluded PMIDs */
+    unsigned int		exclude_indoms;	/* exclude instance domains */
+    struct dict			*indoms;	/* dict of excluded InDoms */
     void			*data;		/* user-supplied pointer */
 } discoverModuleData;
 
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/exports pcp-5.0.2/src/libpcp_web/src/exports
--- pcp-5.0.2.orig/src/libpcp_web/src/exports	2019-11-26 16:29:58.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/exports	2020-02-03 13:23:15.264762900 +1100
@@ -178,3 +178,8 @@ PCP_WEB_1.11 {
   global:
     pmSeriesLabelValues;
 } PCP_WEB_1.10;
+
+PCP_WEB_1.12 {
+  global:
+    SDS_NOINIT;
+} PCP_WEB_1.11;
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/load.c pcp-5.0.2/src/libpcp_web/src/load.c
--- pcp-5.0.2.orig/src/libpcp_web/src/load.c	2019-12-11 14:01:53.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/load.c	2020-02-03 13:36:03.947721365 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
  *
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -112,22 +112,41 @@ load_prepare_metric(const char *name, vo
  * Iterate over an instance domain and extract names and labels
  * for each instance.
  */
-static unsigned int
-get_instance_metadata(seriesLoadBaton *baton, pmInDom indom)
+static void
+get_instance_metadata(seriesLoadBaton *baton, pmInDom indom, int force_refresh)
 {
     context_t		*cp = &baton->pmapi.context;
-    unsigned int	count = 0;
     domain_t		*dp;
     indom_t		*ip;
 
     if (indom != PM_INDOM_NULL) {
 	if ((dp = pmwebapi_add_domain(cp, pmInDom_domain(indom))))
 	    pmwebapi_add_domain_labels(cp, dp);
-	if ((ip = pmwebapi_add_indom(cp, dp, indom)) &&
-	    (count = pmwebapi_add_indom_instances(cp, ip)) > 0)
-	    pmwebapi_add_instances_labels(cp, ip);
+	if ((ip = pmwebapi_add_indom(cp, dp, indom)) != NULL) {
+	    if (force_refresh)
+		ip->updated = 1;
+	    if (ip->updated) {
+		pmwebapi_add_indom_instances(cp, ip);
+		pmwebapi_add_instances_labels(cp, ip);
+	    }
+	}
     }
-    return count;
+}
+
+static void
+get_metric_metadata(seriesLoadBaton *baton, metric_t *metric)
+{
+    context_t		*context = &baton->pmapi.context;
+
+    if (metric->cluster) {
+	if (metric->cluster->domain)
+	    pmwebapi_add_domain_labels(context, metric->cluster->domain);
+	pmwebapi_add_cluster_labels(context, metric->cluster);
+    }
+    if (metric->indom)
+	pmwebapi_add_instances_labels(context, metric->indom);
+    pmwebapi_add_item_labels(context, metric);
+    pmwebapi_metric_hash(metric);
 }
 
 static metric_t *
@@ -140,18 +159,25 @@ new_metric(seriesLoadBaton *baton, pmVal
     char		**nameall = NULL;
     int			count, sts, i;
 
-    if ((sts = pmLookupDesc(vsp->pmid, &desc)) < 0) {
+    if ((sts = pmUseContext(context->context)) < 0) {
+	fprintf(stderr, "%s: failed to use context for PMID %s: %s\n",
+		"new_metric",
+		pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)),
+		pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+    } else if ((sts = pmLookupDesc(vsp->pmid, &desc)) < 0) {
 	if (sts == PM_ERR_IPC)
 	    context->setup = 0;
 	if (pmDebugOptions.series)
-	    fprintf(stderr, "failed to lookup metric %s descriptor: %s",
+	    fprintf(stderr, "%s: failed to lookup metric %s descriptor: %s\n",
+		"new_metric",
 		pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)),
 		pmErrStr_r(sts, errmsg, sizeof(errmsg)));
     } else if ((sts = count = pmNameAll(vsp->pmid, &nameall)) < 0) {
 	if (sts == PM_ERR_IPC)
 	    context->setup = 0;
 	if (pmDebugOptions.series)
-	    fprintf(stderr, "failed to lookup metric %s names: %s",
+	    fprintf(stderr, "%s: failed to lookup metric %s names: %s\n",
+		"new_metric",
 		pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)),
 		pmErrStr_r(sts, errmsg, sizeof(errmsg)));
     }
@@ -160,18 +186,10 @@ new_metric(seriesLoadBaton *baton, pmVal
 
     if ((metric = pmwebapi_new_metric(context, NULL, &desc, count, nameall)) == NULL)
 	return NULL;
-    if (metric->cluster) {
-	if (metric->cluster->domain)
-	    pmwebapi_add_domain_labels(context, metric->cluster->domain);
-	pmwebapi_add_cluster_labels(context, metric->cluster);
-    }
-    if (metric->indom)
-	pmwebapi_add_instances_labels(context, metric->indom);
-    pmwebapi_add_item_labels(context, metric);
-    pmwebapi_metric_hash(metric);
+    get_metric_metadata(baton, metric);
 
     if (pmDebugOptions.series) {
-	fprintf(stderr, "new_metric [%s] names:",
+	fprintf(stderr, "%s [%s] names:\n", "new_metric",
 		pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)));
 	for (i = 0; i < count; i++) {
 	    pmwebapi_hash_str(metric->names[i].hash, idbuf, sizeof(idbuf));
@@ -409,7 +427,7 @@ pmwebapi_add_valueset(metric_t *metric,
 }
 
 static void
-series_cache_update(seriesLoadBaton *baton)
+series_cache_update(seriesLoadBaton *baton, struct dict *exclude)
 {
     seriesGetContext	*context = &baton->pmapi;
     context_t		*cp = &context->context;
@@ -418,7 +436,7 @@ series_cache_update(seriesLoadBaton *bat
     metric_t		*metric = NULL;
     char		ts[64];
     sds			timestamp;
-    int			i, write_meta, write_data;
+    int			i, write_meta, write_inst, write_data;
 
     timestamp = sdsnew(timeval_stream_str(&result->timestamp, ts, sizeof(ts)));
     write_data = (!(baton->flags & PM_SERIES_FLAG_METADATA));
@@ -441,6 +459,12 @@ series_cache_update(seriesLoadBaton *bat
 	    dictFetchValue(baton->wanted, &vsp->pmid) == NULL)
 	    continue;
 
+	/* check if metric to be skipped (optional metric exclusion) */
+	if (exclude && (dictFind(exclude, &vsp->pmid)) != NULL)
+	    continue;
+
+	write_meta = write_inst = 0;
+
 	/* check if pmid already in hash list */
 	if ((metric = dictFetchValue(cp->pmids, &vsp->pmid)) == NULL) {
 	    /* create a new metric, and add it to load context */
@@ -448,21 +472,22 @@ series_cache_update(seriesLoadBaton *bat
 		continue;
 	    write_meta = 1;
 	} else {	/* pmid already observed */
-	    write_meta = 0;
+	    if ((write_meta = metric->cached) == 0)
+		get_metric_metadata(baton, metric);
 	}
 
 	/* iterate through result instances and ensure metric_t is complete */
 	if (metric->error == 0 && vsp->numval < 0)
 	    write_meta = 1;
 	if (pmwebapi_add_valueset(metric, vsp) != 0)
-	    write_meta = 1;
+	    write_meta = write_inst = 1;
 
 	/* record the error code in the cache */
 	metric->error = (vsp->numval < 0) ? vsp->numval : 0;
 
 	/* make PMAPI calls to cache metadata */
-	if (write_meta && get_instance_metadata(baton, metric->desc.indom) != 0)
-	    continue;
+	if (write_meta)
+	    get_instance_metadata(baton, metric->desc.indom, write_inst);
 
 	/* initiate writes to backend caching servers (Redis) */
 	server_cache_metric(baton, metric, timestamp, write_meta, write_data);
@@ -549,7 +574,7 @@ server_cache_window(void *arg)
 	    (finish->tv_sec == result->timestamp.tv_sec &&
 	     finish->tv_usec >= result->timestamp.tv_usec)) {
 	    context->done = server_cache_update_done;
-	    series_cache_update(baton);
+	    series_cache_update(baton, NULL);
 	}
 	else {
 	    if (pmDebugOptions.series)
@@ -1023,7 +1048,7 @@ pmSeriesDiscoverSource(pmDiscoverEvent *
     sds			msg;
     int			i;
 
-    if (data == NULL || data->slots == NULL)
+    if (data == NULL || data->slots == NULL || data->slots->setup == 0)
 	return;
 
     baton = (seriesLoadBaton *)calloc(1, sizeof(seriesLoadBaton));
@@ -1032,22 +1057,31 @@ pmSeriesDiscoverSource(pmDiscoverEvent *
 	moduleinfo(module, PMLOG_ERROR, msg, arg);
 	return;
     }
+    if ((set = pmwebapi_labelsetdup(p->context.labelset)) == NULL) {
+	infofmt(msg, "%s: out of memory for labels", "pmSeriesDiscoverSource");
+	moduleinfo(module, PMLOG_ERROR, msg, arg);
+	free(baton);
+	return;
+    }
+
     initSeriesLoadBaton(baton, module, 0 /*flags*/,
 			module->on_info, series_discover_done,
 			data->slots, arg);
     initSeriesGetContext(&baton->pmapi, baton);
     p->baton = baton;
 
+    cp = &baton->pmapi.context;
+
     if (pmDebugOptions.discovery)
-	fprintf(stderr, "%s: new source %s context=%d\n",
-			"pmSeriesDiscoverSource", p->context.name, p->ctx);
+	fprintf(stderr, "%s: new source %s context=%p ctxid=%d\n",
+			"pmSeriesDiscoverSource", p->context.name, cp, p->ctx);
 
-    cp = &baton->pmapi.context;
     cp->context = p->ctx;
     cp->type = p->context.type;
     cp->name.sds = sdsdup(p->context.name);
-    cp->host = p->context.hostname;
-    cp->labelset = set = p->context.labelset;
+    cp->host = sdsdup(p->context.hostname);
+    cp->labelset = set;
+
     pmwebapi_source_hash(cp->name.hash, set->json, set->jsonlen);
     pmwebapi_setup_context(cp);
     set_source_origin(cp);
@@ -1095,21 +1129,22 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
     sds			msg;
     int			i, id;
 
+    if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+	return;
+
     switch (type) {
     case PM_LABEL_CONTEXT:
 	if (pmDebugOptions.discovery)
 	    fprintf(stderr, "%s: context\n", "pmSeriesDiscoverLabels");
 
 	if ((labels = pmwebapi_labelsetdup(sets)) != NULL) {
-#if 0 /* PCP GH#800 do not free this labelset - it's owned by the discover code */
 	    if (cp->labelset)
 		pmFreeLabelSets(cp->labelset, 1);
-#endif
 	    cp->labelset = labels;
 	    pmwebapi_locate_context(cp);
 	    cp->updated = 1;
 	} else {
-	    infofmt(msg, "failed to duplicate label set");
+	    infofmt(msg, "failed to duplicate %s label set", "context");
 	    moduleinfo(event->module, PMLOG_ERROR, msg, arg);
 	}
 	break;
@@ -1125,8 +1160,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
 		pmFreeLabelSets(domain->labelset, 1);
 	    domain->labelset = labels;
 	    domain->updated = 1;
-	} else {
-	    infofmt(msg, "failed to duplicate label set");
+	} else if (domain) {
+	    infofmt(msg, "failed to duplicate %s label set", "domain");
 	    moduleinfo(event->module, PMLOG_ERROR, msg, arg);
 	}
 	break;
@@ -1142,8 +1177,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
 		pmFreeLabelSets(cluster->labelset, 1);
 	    cluster->labelset = labels;
 	    cluster->updated = 1;
-	} else {
-	    infofmt(msg, "failed to duplicate label set");
+	} else if (cluster) {
+	    infofmt(msg, "failed to duplicate %s label set", "cluster");
 	    moduleinfo(event->module, PMLOG_ERROR, msg, arg);
 	}
 	break;
@@ -1159,8 +1194,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
 		pmFreeLabelSets(metric->labelset, 1);
 	    metric->labelset = labels;
 	    metric->updated = 1;
-	} else {
-	    infofmt(msg, "failed to duplicate label set");
+	} else if (metric) {
+	    infofmt(msg, "failed to duplicate %s label set", "item");
 	    moduleinfo(event->module, PMLOG_ERROR, msg, arg);
 	}
 	break;
@@ -1177,8 +1212,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
 		pmFreeLabelSets(indom->labelset, 1);
 		    indom->labelset = labels;
 	    indom->updated = 1;
-	} else {
-	    infofmt(msg, "failed to duplicate label set");
+	} else if (indom) {
+	    infofmt(msg, "failed to duplicate %s label set", "indom");
 	    moduleinfo(event->module, PMLOG_ERROR, msg, arg);
 	}
 	break;
@@ -1196,7 +1231,7 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
 	    if ((instance = dictFetchValue(indom->insts, &id)) == NULL)
 		continue;
 	    if ((labels = pmwebapi_labelsetdup(&sets[i])) == NULL) {
-		infofmt(msg, "failed to dup %s instance labels: %s",
+		infofmt(msg, "failed to dup indom %s instance label set: %s",
 			pmInDomStr_r(indom->indom, idbuf, sizeof(idbuf)),
 			pmErrStr_r(-ENOMEM, errmsg, sizeof(errmsg)));
 		moduleinfo(event->module, PMLOG_ERROR, msg, arg);
@@ -1229,10 +1264,13 @@ pmSeriesDiscoverMetric(pmDiscoverEvent *
 
     if (pmDebugOptions.discovery) {
 	for (i = 0; i < numnames; i++)
-	    fprintf(stderr, "pmSeriesDiscoverMetric: [%d/%d] %s - %s\n",
+	    fprintf(stderr, "%s: [%d/%d] %s - %s\n", "pmSeriesDiscoverMetric",
 			i + 1, numnames, pmIDStr(desc->pmid), names[i]);
     }
 
+    if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+	return;
+
     if ((metric = pmwebapi_add_metric(&baton->pmapi.context,
 				NULL, desc, numnames, names)) == NULL) {
 	infofmt(msg, "%s: failed metric discovery", "pmSeriesDiscoverMetric");
@@ -1244,18 +1282,23 @@ pmSeriesDiscoverMetric(pmDiscoverEvent *
 void
 pmSeriesDiscoverValues(pmDiscoverEvent *event, pmResult *result, void *arg)
 {
+    pmDiscoverModule	*module = event->module;
     pmDiscover		*p = (pmDiscover *)event->data;
     seriesLoadBaton	*baton = p->baton;
     seriesGetContext	*context = &baton->pmapi;
+    discoverModuleData	*data = getDiscoverModuleData(module);
 
     if (pmDebugOptions.discovery)
 	fprintf(stderr, "%s: result numpmids=%d\n", "pmSeriesDiscoverValues", result->numpmid);
 
+    if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+	return;
+
     seriesBatonReference(context, "pmSeriesDiscoverValues");
     baton->arg = arg;
     context->result = result;
 
-    series_cache_update(baton);
+    series_cache_update(baton, data->pmids);
 }
 
 void
@@ -1271,7 +1314,10 @@ pmSeriesDiscoverInDom(pmDiscoverEvent *e
     int			i;
 
     if (pmDebugOptions.discovery)
-	fprintf(stderr, "pmSeriesDiscoverInDom: %s\n", pmInDomStr(id));
+	fprintf(stderr, "%s: %s\n", "pmSeriesDiscoverInDom", pmInDomStr(id));
+
+    if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+	return;
 
     if ((domain = pmwebapi_add_domain(context, pmInDom_domain(id))) == NULL) {
 	infofmt(msg, "%s: failed indom discovery (domain %u)",
@@ -1303,11 +1349,10 @@ pmSeriesDiscoverText(pmDiscoverEvent *ev
     pmDiscover		*p = (pmDiscover *)event->data;
     seriesLoadBaton	*baton = p->baton;
 
-    (void)baton;
-    (void)ident;
-    (void)type;
-    (void)text;
-    (void)arg;
+    if (pmDebugOptions.discovery)
+	fprintf(stderr, "%s: ident=%u type=%u arg=%p\n",
+			"pmSeriesDiscoverText", ident, type, arg);
 
-    /* for Redis, help text will need special handling (RediSearch) */
+    if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+	return;
 }
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/query.c pcp-5.0.2/src/libpcp_web/src/query.c
--- pcp-5.0.2.orig/src/libpcp_web/src/query.c	2019-12-05 17:29:43.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/query.c	2020-02-03 13:23:15.265762890 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
  *
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -1243,24 +1243,43 @@ series_prepare_time_reply(
     series_query_end_phase(baton);
 }
 
+unsigned int
+series_value_count_only(timing_t *tp)
+{
+    if (tp->window.range || tp->window.delta ||
+	tp->window.start || tp->window.end)
+	return 0;
+    return tp->count;
+}
+
 static void
 series_prepare_time(seriesQueryBaton *baton, series_set_t *result)
 {
     timing_t		*tp = &baton->u.query.timing;
     unsigned char	*series = result->series;
     seriesGetSID	*sid;
-    char		buffer[64];
+    char		buffer[64], revbuf[64];
     sds			start, end, key, cmd;
-    unsigned int	i;
+    unsigned int	i, revlen = 0, reverse = 0;
+
+    /* if only 'count' is requested, work back from most recent value */
+    if ((reverse = series_value_count_only(tp)) != 0) {
+	revlen = pmsprintf(revbuf, sizeof(revbuf), "%u", reverse);
+	start = sdsnew("+");
+    } else {
+	start = sdsnew(timeval_stream_str(&tp->start, buffer, sizeof(buffer)));
+    }
 
-    start = sdsnew(timeval_stream_str(&tp->start, buffer, sizeof(buffer)));
     if (pmDebugOptions.series)
 	fprintf(stderr, "START: %s\n", start);
 
-    if (tp->end.tv_sec)
+    if (reverse)
+	end = sdsnew("-");
+    else if (tp->end.tv_sec)
 	end = sdsnew(timeval_stream_str(&tp->end, buffer, sizeof(buffer)));
     else
 	end = sdsnew("+");	/* "+" means "no end" - to the most recent */
+
     if (pmDebugOptions.series)
 	fprintf(stderr, "END: %s\n", end);
 
@@ -1277,12 +1296,21 @@ series_prepare_time(seriesQueryBaton *ba
 
 	key = sdscatfmt(sdsempty(), "pcp:values:series:%S", sid->name);
 
-	/* XRANGE key t1 t2 */
-	cmd = redis_command(4);
-	cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN);
+	/* X[REV]RANGE key t1 t2 [count N] */
+	if (reverse) {
+	    cmd = redis_command(6);
+	    cmd = redis_param_str(cmd, XREVRANGE, XREVRANGE_LEN);
+	} else {
+	    cmd = redis_command(4);
+	    cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN);
+	}
 	cmd = redis_param_sds(cmd, key);
 	cmd = redis_param_sds(cmd, start);
 	cmd = redis_param_sds(cmd, end);
+	if (reverse) {
+	    cmd = redis_param_str(cmd, "COUNT", sizeof("COUNT")-1);
+	    cmd = redis_param_str(cmd, revbuf, revlen);
+	}
 	redisSlotsRequest(baton->slots, XRANGE, key, cmd,
 				series_prepare_time_reply, sid);
     }
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/schema.c pcp-5.0.2/src/libpcp_web/src/schema.c
--- pcp-5.0.2.orig/src/libpcp_web/src/schema.c	2019-11-18 19:35:11.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/schema.c	2020-02-03 13:36:03.948721355 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
  *
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -819,7 +819,7 @@ redis_series_metric(redisSlots *slots, m
      */
 
     /* ensure all metric name strings are mapped */
-    for (i = 0; i < metric->numnames; i++) {
+    for (i = 0; metric->cached == 0 && i < metric->numnames; i++) {
 	assert(metric->names[i].sds != NULL);
 	seriesBatonReference(baton, "redis_series_metric");
 	redisGetMap(slots,
@@ -830,7 +830,8 @@ redis_series_metric(redisSlots *slots, m
 
     /* ensure all metric or instance label strings are mapped */
     if (metric->desc.indom == PM_INDOM_NULL || metric->u.vlist == NULL) {
-	series_metric_label_mapping(metric, baton);
+	if (metric->cached == 0)
+	    series_metric_label_mapping(metric, baton);
     } else {
 	for (i = 0; i < metric->u.vlist->listcount; i++) {
 	    value = &metric->u.vlist->value[i];
@@ -847,7 +848,8 @@ redis_series_metric(redisSlots *slots, m
 			series_name_mapping_callback,
 			baton->info, baton->userdata, baton);
 
-	    series_instance_label_mapping(metric, instance, baton);
+	    if (instance->cached == 0)
+		series_instance_label_mapping(metric, instance, baton);
 	}
     }
 
@@ -941,6 +943,9 @@ redis_series_metadata(context_t *context
     sds				cmd, key;
     int				i;
 
+    if (metric->cached)
+	goto check_instances;
+
     indom = pmwebapi_indom_str(metric, ibuf, sizeof(ibuf));
     pmid = pmwebapi_pmid_str(metric, pbuf, sizeof(pbuf));
     sem = pmwebapi_semantics_str(metric, sbuf, sizeof(sbuf));
@@ -1000,16 +1005,24 @@ redis_series_metadata(context_t *context
 	cmd = redis_param_sha(cmd, metric->names[i].hash);
     redisSlotsRequest(slots, SADD, key, cmd, redis_series_source_callback, arg);
 
+check_instances:
     if (metric->desc.indom == PM_INDOM_NULL || metric->u.vlist == NULL) {
-	redis_series_labelset(slots, metric, NULL, baton);
+	if (metric->cached == 0) {
+	    redis_series_labelset(slots, metric, NULL, baton);
+	    metric->cached = 1;
+	}
     } else {
 	for (i = 0; i < metric->u.vlist->listcount; i++) {
 	    value = &metric->u.vlist->value[i];
 	    if ((instance = dictFetchValue(metric->indom->insts, &value->inst)) == NULL)
 		continue;
-	    redis_series_instance(slots, metric, instance, baton);
-	    redis_series_labelset(slots, metric, instance, baton);
+	    if (instance->cached == 0 || metric->cached == 0) {
+		redis_series_instance(slots, metric, instance, baton);
+		redis_series_labelset(slots, metric, instance, baton);
+	    }
+	    instance->cached = 1;
 	}
+	metric->cached = 1;
     }
 }
 
@@ -1210,7 +1223,6 @@ redis_series_stream(redisSlots *slots, s
 
     redisSlotsRequest(slots, XADD, key, cmd, redis_series_stream_callback, baton);
 
-
     key = sdscatfmt(sdsempty(), "pcp:values:series:%s", hash);
     cmd = redis_command(3);	/* EXPIRE key timer */
     cmd = redis_param_str(cmd, EXPIRE, EXPIRE_LEN);
@@ -1228,9 +1240,6 @@ redis_series_streamed(sds stamp, metric_
     char			hashbuf[42];
     int				i;
 
-    if (metric->updated == 0)
-	return;
-
     for (i = 0; i < metric->numnames; i++) {
 	pmwebapi_hash_str(metric->names[i].hash, hashbuf, sizeof(hashbuf));
 	redis_series_stream(slots, stamp, metric, hashbuf, arg);
@@ -1545,7 +1554,10 @@ redis_load_slots_callback(
     redisSlots		*slots = baton->slots;
 
     seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_load_slots_callback");
+
+    slots->setup = 1;	/* we've received initial response from Redis */
     slots->refresh = 0;	/* we're processing CLUSTER SLOTS command now */
+
     /* no cluster redirection checking is needed for this callback */
     sdsfree(cmd);
 
@@ -1832,12 +1844,47 @@ pmDiscoverSetup(pmDiscoverModule *module
     const char		fallback[] = "/var/log/pcp";
     const char		*paths[] = { "pmlogger", "pmmgr" };
     const char		*logdir = pmGetOptionalConfig("PCP_LOG_DIR");
+    struct dict		*config;
+    unsigned int	domain, serial;
+    pmInDom		indom;
     char		path[MAXPATHLEN];
     char		sep = pmPathSeparator();
-    int			i, sts, count = 0;
+    sds			option, *ids;
+    int			i, sts, nids, count = 0;
 
     if (data == NULL)
 	return -ENOMEM;
+    config = data->config;
+
+    /* double-check that we are supposed to be in here */
+    if ((option = pmIniFileLookup(config, "discover", "enabled"))) {
+	if (strcasecmp(option, "false") == 0)
+	    return 0;
+    }
+
+    /* prepare for optional metric and indom exclusion */
+    if ((option = pmIniFileLookup(config, "discover", "exclude.metrics"))) {
+	if ((data->pmids = dictCreate(&intKeyDictCallBacks, NULL)) == NULL)
+	    return -ENOMEM;
+	/* parse regular expression string for matching on metric names */
+	regcomp(&data->exclude_names, option, REG_EXTENDED|REG_NOSUB);
+    }
+    if ((option = pmIniFileLookup(config, "discover", "exclude.indoms"))) {
+	if ((data->indoms = dictCreate(&intKeyDictCallBacks, NULL)) == NULL)
+	    return -ENOMEM;
+	/* parse comma-separated indoms in 'option', convert to pmInDom */
+	if ((ids = sdssplitlen(option, sdslen(option), ",", 1, &nids))) {
+	    data->exclude_indoms = nids;
+	    for (i = 0; i < nids; i++) {
+		if (sscanf(ids[i], "%u.%u", &domain, &serial) == 2) {
+		    indom = pmInDom_build(domain, serial);
+		    dictAdd(data->indoms, &indom, NULL);
+		}
+		sdsfree(ids[i]);
+	    }
+	    free(ids);
+	}
+    }
 
     /* create global EVAL hashes and string map caches */
     redisGlobalsInit(data->config);
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/schema.h pcp-5.0.2/src/libpcp_web/src/schema.h
--- pcp-5.0.2.orig/src/libpcp_web/src/schema.h	2019-10-11 17:16:29.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/schema.h	2020-02-03 13:23:15.266762879 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2018 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
  * 
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -51,6 +51,10 @@
 #define HSET_LEN	(sizeof(HSET)-1)
 #define HVALS		"HVALS"
 #define HVALS_LEN	(sizeof(HVALS)-1)
+#define INFO		"INFO"
+#define INFO_LEN	(sizeof(INFO)-1)
+#define PING		"PING"
+#define PING_LEN	(sizeof(PING)-1)
 #define PUBLISH		"PUBLISH"
 #define PUBLISH_LEN	(sizeof(PUBLISH)-1)
 #define SADD		"SADD"
@@ -63,6 +67,8 @@
 #define XADD_LEN	(sizeof(XADD)-1)
 #define XRANGE		"XRANGE"
 #define XRANGE_LEN	(sizeof(XRANGE)-1)
+#define XREVRANGE	"XREVRANGE"
+#define XREVRANGE_LEN	(sizeof(XREVRANGE)-1)
 
 /* create a Redis protocol command (e.g. XADD, SMEMBER) */
 static inline sds
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/slots.c pcp-5.0.2/src/libpcp_web/src/slots.c
--- pcp-5.0.2.orig/src/libpcp_web/src/slots.c	2019-10-11 17:16:29.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/slots.c	2020-02-03 13:23:15.266762879 +1100
@@ -356,6 +356,21 @@ redisSlotsRequest(redisSlots *slots, con
 
     if (UNLIKELY(pmDebugOptions.desperate))
 	fputs(cmd, stderr);
+    if (UNLIKELY(!key && !slots->setup)) {
+	/*
+	 * First request must be CLUSTER, PING, or similar - must
+	 * not allow regular requests until these have completed.
+	 * This is because the low layers accumulate async requests
+	 * until connection establishment, which might not happen.
+	 * Over time this becomes a memory leak - if we do not ever
+	 * establish an initial connection).
+	 */
+	if (strcmp(topic, CLUSTER) != 0 &&
+	    strcmp(topic, PING) != 0 && strcmp(topic, INFO) != 0) {
+	    sdsfree(cmd);
+	    return -ENOTCONN;
+	}
+    }
 
     sts = redisAsyncFormattedCommand(context, callback, cmd, arg);
     if (key)
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/slots.h pcp-5.0.2/src/libpcp_web/src/slots.h
--- pcp-5.0.2.orig/src/libpcp_web/src/slots.h	2019-04-08 09:11:00.000000000 +1000
+++ pcp-5.0.2/src/libpcp_web/src/slots.h	2020-02-03 13:23:15.266762879 +1100
@@ -44,10 +44,11 @@ typedef struct redisSlotRange {
 typedef struct redisSlots {
     unsigned int	counter;
     unsigned int	nslots;
+    unsigned int	setup;		/* slots info all successfully setup */
+    unsigned int	refresh;	/* do slot refresh whenever possible */
     redisSlotRange	*slots;		/* all instances; e.g. CLUSTER SLOTS */
     redisMap		*keymap;	/* map command names to key position */
     dict		*contexts;	/* async contexts access by hostspec */
-    unsigned int	refresh;	/* do slot refresh whenever possible */
     void		*events;
 } redisSlots;
 
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/util.c pcp-5.0.2/src/libpcp_web/src/util.c
--- pcp-5.0.2.orig/src/libpcp_web/src/util.c	2019-12-10 17:39:49.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/util.c	2020-02-03 13:23:15.266762879 +1100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
  *
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -535,6 +535,8 @@ pmwebapi_metric_hash(metric_t *metric)
 	sdsclear(identifier);
     }
     sdsfree(identifier);
+
+    metric->cached = 0;
 }
 
 void
@@ -574,6 +576,8 @@ pmwebapi_instance_hash(indom_t *ip, inst
     SHA1Update(&shactx, (unsigned char *)identifier, sdslen(identifier));
     SHA1Final(instance->name.hash, &shactx);
     sdsfree(identifier);
+
+    instance->cached = 0;
 }
 
 sds
@@ -1046,7 +1050,6 @@ pmwebapi_add_instance(struct indom *indo
 	    instance->name.sds = sdscatlen(instance->name.sds, name, length);
 	    pmwebapi_string_hash(instance->name.id, name, length);
 	    pmwebapi_instance_hash(indom, instance);
-	    instance->cached = 0;
 	}
 	return instance;
     }
@@ -1202,12 +1205,14 @@ struct metric *
 pmwebapi_add_metric(context_t *cp, const sds base, pmDesc *desc, int numnames, char **names)
 {
     struct metric	*metric;
-    sds			name = sdsempty();
+    sds			name;
     int			i;
 
     /* search for a match on any of the given names */
     if (base && (metric = dictFetchValue(cp->metrics, base)) != NULL)
 	return metric;
+
+    name = sdsempty();
     for (i = 0; i < numnames; i++) {
 	sdsclear(name);
 	name = sdscat(name, names[i]);
@@ -1217,6 +1222,7 @@ pmwebapi_add_metric(context_t *cp, const
 	}
     }
     sdsfree(name);
+
     return pmwebapi_new_metric(cp, base, desc, numnames, names);
 }
 
@@ -1230,21 +1236,24 @@ pmwebapi_new_pmid(context_t *cp, const s
     int			sts, numnames;
 
     if ((sts = pmUseContext(cp->context)) < 0) {
-	fprintf(stderr, "failed to use context for PMID %s: %s",
+	fprintf(stderr, "%s: failed to use context for PMID %s: %s\n",
+		"pmwebapi_new_pmid",
 		pmIDStr_r(pmid, buffer, sizeof(buffer)),
 		pmErrStr_r(sts, errmsg, sizeof(errmsg)));
     } else if ((sts = pmLookupDesc(pmid, &desc)) < 0) {
 	if (sts == PM_ERR_IPC)
 	    cp->setup = 0;
 	if (pmDebugOptions.series)
-	    fprintf(stderr, "failed to lookup metric %s descriptor: %s",
+	    fprintf(stderr, "%s: failed to lookup metric %s descriptor: %s\n",
+		    "pmwebapi_new_pmid",
 		    pmIDStr_r(pmid, buffer, sizeof(buffer)),
 		    pmErrStr_r(sts, errmsg, sizeof(errmsg)));
     } else if ((numnames = sts = pmNameAll(pmid, &names)) < 0) {
 	if (sts == PM_ERR_IPC)
 	    cp->setup = 0;
 	if (pmDebugOptions.series)
-	    fprintf(stderr, "failed to lookup metric %s names: %s",
+	    fprintf(stderr, "%s: failed to lookup metric %s names: %s\n",
+		    "pmwebapi_new_pmid",
 		    pmIDStr_r(pmid, buffer, sizeof(buffer)),
 		    pmErrStr_r(sts, errmsg, sizeof(errmsg)));
     } else {
diff -Naurp pcp-5.0.2.orig/src/pmproxy/pmproxy.conf pcp-5.0.2/src/pmproxy/pmproxy.conf
--- pcp-5.0.2.orig/src/pmproxy/pmproxy.conf	2019-08-09 15:50:17.000000000 +1000
+++ pcp-5.0.2/src/pmproxy/pmproxy.conf	2020-02-03 13:36:03.948721355 +1100
@@ -43,6 +43,11 @@ secure.enabled = true
 # propogate archives from pmlogger(1) into Redis querying
 enabled = true
 
+# metrics name regex to skip during discovery (eg due to high volume)
+exclude.metrics = proc.*
+
+# comma-separated list of instance domains to skip during discovery
+exclude.indoms = 3.9,79.7
 
 #####################################################################
 ## settings for fast, scalable time series quering via Redis
diff -Naurp pcp-5.0.2.orig/src/pmproxy/src/redis.c pcp-5.0.2/src/pmproxy/src/redis.c
--- pcp-5.0.2.orig/src/pmproxy/src/redis.c	2019-12-02 16:39:33.000000000 +1100
+++ pcp-5.0.2/src/pmproxy/src/redis.c	2020-02-03 13:36:13.585620539 +1100
@@ -145,11 +145,11 @@ setup_redis_module(struct proxy *proxy)
 	proxy->slots = redisSlotsConnect(proxy->config,
 			flags, proxylog, on_redis_connected,
 			proxy, proxy->events, proxy);
-	if (archive_discovery)
+	if (archive_discovery && series_queries)
 	    pmDiscoverSetSlots(&redis_discover.module, proxy->slots);
     }
 
-    if (archive_discovery) {
+    if (archive_discovery && series_queries) {
 	pmDiscoverSetEventLoop(&redis_discover.module, proxy->events);
 	pmDiscoverSetConfiguration(&redis_discover.module, proxy->config);
 	pmDiscoverSetMetricRegistry(&redis_discover.module, metric_registry);