Blame SOURCES/redhat-bugzilla-1947989.patch

5b7fd1
commit 2bad6aef10339f000f7cb578108db5ee80bd640c
5b7fd1
Author: Mark Goodwin <mgoodwin@redhat.com>
5b7fd1
Date:   Wed Jun 9 17:04:33 2021 +1000
5b7fd1
5b7fd1
    pmproxy: add mutex for client req lists, fix https/tls support, QA
5b7fd1
    
5b7fd1
    Add a new mutext to struct proxy and use it to protect parallel
5b7fd1
    multithreaded updates to the proxy->first client list.
5b7fd1
    
5b7fd1
    Also use the same mutext to protect updates to the pending_writes
5b7fd1
    client list and avoid the doubly linked list corruption that was
5b7fd1
    causing parallel https/tls requests to get stuck spinning in
5b7fd1
    flush_secure_module(), as reported in BZ#1947989.
5b7fd1
    
5b7fd1
    qa/1457 is extensively updated to test parallel http, https/tls
5b7fd1
    (and combinations of http and https/tls) RESTAPI calls. Previously
5b7fd1
    it only tested a single https/tls call.
5b7fd1
    
5b7fd1
    With these changes, parallel https/tls RESTAPI requests from the
5b7fd1
    grafana-pcp datasource to pmproxy now work correctly whereas previously
5b7fd1
    pmproxy would hang/spin.
5b7fd1
    
5b7fd1
    Resolves: RHBZ#1947989 - pmproxy hangs and consume 100% cpu if the
5b7fd1
    redis datasource is configured with TLS.
5b7fd1
    
5b7fd1
    Related: https://github.com/performancecopilot/pcp/issues/1311
5b7fd1
5b7fd1
diff --git a/qa/1457 b/qa/1457
5b7fd1
index 94969f6e0..8bf395944 100755
5b7fd1
--- a/qa/1457
5b7fd1
+++ b/qa/1457
5b7fd1
@@ -2,7 +2,7 @@
5b7fd1
 # PCP QA Test No. 1457
5b7fd1
 # Exercise HTTPS access to the PMWEBAPI(3).
5b7fd1
 #
5b7fd1
-# Copyright (c) 2019 Red Hat.
5b7fd1
+# Copyright (c) 2019,2021 Red Hat.
5b7fd1
 #
5b7fd1
 
5b7fd1
 seq=`basename $0`
5b7fd1
@@ -138,14 +138,59 @@ else
5b7fd1
 fi
5b7fd1
 
5b7fd1
 date >>$seq.full
5b7fd1
-echo "=== checking TLS operation ===" | tee -a $seq.full
5b7fd1
-# (-k) allows us to use self-signed (insecure) certificates, so for testing only
5b7fd1
-# (-v) provides very detailed TLS connection information, for debugging only
5b7fd1
-curl -k --get 2>$tmp.err \
5b7fd1
-	"https://localhost:$port/pmapi/metric?name=sample.long.ten" \
5b7fd1
-	| _filter_json
5b7fd1
-cat $tmp.err >>$seq.full
5b7fd1
+echo "=== checking serial http operation ===" | tee -a $seq.full
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+    curl -Gs "http://localhost:$port/pmapi/metric?name=sample.long.ten" 2>$tmp.err$i >$tmp.out$i
5b7fd1
+done
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+echo === out$i === | tee -a $seq.full
5b7fd1
+_filter_json < $tmp.out$i
5b7fd1
+done
5b7fd1
+
5b7fd1
+date >>$seq.full
5b7fd1
+echo "=== checking parallel http operation ===" | tee -a $seq.full
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+    curl -Gs "http://localhost:$port/pmapi/metric?name=sample.long.ten" 2>$tmp.err$i >$tmp.out$i & 2>/dev/null eval pid$i=$!
5b7fd1
+done
5b7fd1
+wait $pid1 $pid2 $pid3 $pid4
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+echo === out$i === | tee -a $seq.full
5b7fd1
+_filter_json < $tmp.out$i
5b7fd1
+done
5b7fd1
+
5b7fd1
+date >>$seq.full
5b7fd1
+echo "=== checking serial https/TLS operation ===" | tee -a $seq.full
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+    curl -k -Gs "https://localhost:$port/pmapi/metric?name=sample.long.ten" 2>$tmp.err$i >$tmp.out$i
5b7fd1
+done
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+echo === out$i === | tee -a $seq.full
5b7fd1
+_filter_json < $tmp.out$i
5b7fd1
+done
5b7fd1
+
5b7fd1
 date >>$seq.full
5b7fd1
+echo "=== checking parallel https/TLS operation ===" | tee -a $seq.full
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+    curl -k -Gs "https://localhost:$port/pmapi/metric?name=sample.long.ten" 2>$tmp.err$i >$tmp.out$i & 2>/dev/null eval pid$i=$!
5b7fd1
+done
5b7fd1
+wait $pid1 $pid2 $pid3 $pid4
5b7fd1
+for i in 1 2 3 4; do
5b7fd1
+echo === out$i === | tee -a $seq.full
5b7fd1
+_filter_json < $tmp.out$i
5b7fd1
+done
5b7fd1
+
5b7fd1
+date >>$seq.full
5b7fd1
+echo "=== checking parallel mixed http and https/TLS operations ===" | tee -a $seq.full
5b7fd1
+for i in 1 3 5 7; do
5b7fd1
+    j=`expr $i + 1`
5b7fd1
+    curl -k -Gs "http://localhost:$port/pmapi/metric?name=sample.long.ten" 2>$tmp.err$i >$tmp.out$i & 2>/dev/null eval pid$i=$!
5b7fd1
+    curl -k -Gs "https://localhost:$port/pmapi/metric?name=sample.long.ten" 2>$tmp.err$j >$tmp.out$j & 2>/dev/null eval pid$j=$!
5b7fd1
+done
5b7fd1
+wait $pid1 $pid2 $pid3 $pid4 $pid5 $pid6 $pid7 $pid8
5b7fd1
+for i in 1 2 3 4 5 6 7 8; do
5b7fd1
+echo === out$i === | tee -a $seq.full
5b7fd1
+_filter_json < $tmp.out$i
5b7fd1
+done
5b7fd1
 
5b7fd1
 echo "=== check pmproxy is running ==="
5b7fd1
 pminfo -v -h localhost@localhost:$port hinv.ncpu
5b7fd1
@@ -156,7 +201,7 @@ else
5b7fd1
 fi
5b7fd1
 
5b7fd1
 # valgrind takes awhile to shutdown too
5b7fd1
-pmsignal $pid
5b7fd1
+pmsignal $pid >/dev/null 2>&1
5b7fd1
 pmsleep 3.5
5b7fd1
 echo "=== valgrind stdout ===" | tee -a $seq.full
5b7fd1
 cat $tmp.valout | _filter_valgrind
5b7fd1
@@ -164,6 +209,9 @@ cat $tmp.valout | _filter_valgrind
5b7fd1
 echo "=== valgrind stderr ===" | tee -a $seq.full
5b7fd1
 cat $tmp.valerr | _filter_pmproxy_log | _filter_port
5b7fd1
 
5b7fd1
+# final kill if it's spinning
5b7fd1
+$sudo kill -9 $pid >/dev/null 2>&1
5b7fd1
+
5b7fd1
 # success, all done
5b7fd1
 status=0
5b7fd1
 exit
5b7fd1
diff --git a/qa/1457.out b/qa/1457.out
5b7fd1
index a7b64cdc5..422176db2 100644
5b7fd1
--- a/qa/1457.out
5b7fd1
+++ b/qa/1457.out
5b7fd1
@@ -1,5 +1,539 @@
5b7fd1
 QA output created by 1457
5b7fd1
-=== checking TLS operation ===
5b7fd1
+=== checking serial http operation ===
5b7fd1
+=== out1 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out2 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out3 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out4 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== checking parallel http operation ===
5b7fd1
+=== out1 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out2 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out3 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out4 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== checking serial https/TLS operation ===
5b7fd1
+=== out1 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out2 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out3 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out4 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== checking parallel https/TLS operation ===
5b7fd1
+=== out1 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out2 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out3 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out4 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== checking parallel mixed http and https/TLS operations ===
5b7fd1
+=== out1 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out2 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out3 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out4 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out5 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out6 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out7 ===
5b7fd1
+{
5b7fd1
+    "context": "CONTEXT"
5b7fd1
+    "metrics": [
5b7fd1
+        {
5b7fd1
+            "name": "sample.long.ten",
5b7fd1
+            "series": "SERIES"
5b7fd1
+            "pmid": "29.0.11",
5b7fd1
+            "type": "32",
5b7fd1
+            "sem": "instant",
5b7fd1
+            "units": "none",
5b7fd1
+            "labels": {
5b7fd1
+                "agent": "sample",
5b7fd1
+                "cluster": "zero",
5b7fd1
+                "domainname": "DOMAINNAME"
5b7fd1
+                "hostname": "HOSTNAME"
5b7fd1
+                "role": "testing"
5b7fd1
+            },
5b7fd1
+            "text-oneline": "10 as a 32-bit integer",
5b7fd1
+            "text-help": "10 as a 32-bit integer"
5b7fd1
+        }
5b7fd1
+    ]
5b7fd1
+}
5b7fd1
+=== out8 ===
5b7fd1
 {
5b7fd1
     "context": "CONTEXT"
5b7fd1
     "metrics": [
5b7fd1
diff --git a/qa/group b/qa/group
5b7fd1
index 462dffaaa..77cac788d 100644
5b7fd1
--- a/qa/group
5b7fd1
+++ b/qa/group
5b7fd1
@@ -1818,7 +1818,7 @@ x11
5b7fd1
 1436 pmda.postgresql local
5b7fd1
 1437 pmda.kvm local
5b7fd1
 1455 pmlogrewrite labels pmdumplog local
5b7fd1
-1457 pmproxy local
5b7fd1
+1457 pmproxy libpcp_web threads secure local
5b7fd1
 1480 pmda.lmsensors local
5b7fd1
 1489 python pmrep pmimport local
5b7fd1
 1490 python local labels
5b7fd1
diff --git a/src/pmproxy/src/secure.c b/src/pmproxy/src/secure.c
5b7fd1
index 77265894c..072e2a085 100644
5b7fd1
--- a/src/pmproxy/src/secure.c
5b7fd1
+++ b/src/pmproxy/src/secure.c
5b7fd1
@@ -16,13 +16,25 @@
5b7fd1
 #include <openssl/opensslv.h>
5b7fd1
 #include <openssl/ssl.h>
5b7fd1
 
5b7fd1
+/* called with proxy->mutex locked */
5b7fd1
 static void
5b7fd1
 remove_connection_from_queue(struct client *client)
5b7fd1
 {
5b7fd1
+    struct proxy *proxy = client->proxy;
5b7fd1
+
5b7fd1
     if (client->secure.pending.writes_buffer != NULL)
5b7fd1
 	free(client->secure.pending.writes_buffer);
5b7fd1
-    if (client->secure.pending.prev != NULL)
5b7fd1
-	*client->secure.pending.prev = client->secure.pending.next;
5b7fd1
+    if (client->secure.pending.prev == NULL) {
5b7fd1
+	/* next (if any) becomes first in pending_writes list */
5b7fd1
+    	proxy->pending_writes = client->secure.pending.next;
5b7fd1
+	if (proxy->pending_writes)
5b7fd1
+	    proxy->pending_writes->secure.pending.prev = NULL;
5b7fd1
+    }
5b7fd1
+    else {
5b7fd1
+	/* link next and prev */
5b7fd1
+	client->secure.pending.prev->secure.pending.next = client->secure.pending.next;
5b7fd1
+	client->secure.pending.next->secure.pending.prev = client->secure.pending.prev;
5b7fd1
+    }
5b7fd1
     memset(&client->secure.pending, 0, sizeof(client->secure.pending));
5b7fd1
 }
5b7fd1
 
5b7fd1
@@ -32,7 +44,9 @@ on_secure_client_close(struct client *client)
5b7fd1
     if (pmDebugOptions.auth || pmDebugOptions.http)
5b7fd1
 	fprintf(stderr, "%s: client %p\n", "on_secure_client_close", client);
5b7fd1
 
5b7fd1
+    uv_mutex_lock(&client->proxy->mutex);
5b7fd1
     remove_connection_from_queue(client);
5b7fd1
+    uv_mutex_unlock(&client->proxy->mutex);
5b7fd1
     /* client->read and client->write freed by SSL_free */
5b7fd1
     SSL_free(client->secure.ssl);
5b7fd1
 }
5b7fd1
@@ -40,6 +54,8 @@ on_secure_client_close(struct client *client)
5b7fd1
 static void
5b7fd1
 maybe_flush_ssl(struct proxy *proxy, struct client *client)
5b7fd1
 {
5b7fd1
+    struct client *c;
5b7fd1
+
5b7fd1
     if (client->secure.pending.queued)
5b7fd1
 	return;
5b7fd1
 
5b7fd1
@@ -47,13 +63,19 @@ maybe_flush_ssl(struct proxy *proxy, struct client *client)
5b7fd1
 	client->secure.pending.writes_count > 0)
5b7fd1
 	return;
5b7fd1
 
5b7fd1
-    client->secure.pending.next = proxy->pending_writes;
5b7fd1
-    if (client->secure.pending.next != NULL)
5b7fd1
-	client->secure.pending.next->secure.pending.prev = &client->secure.pending.next;
5b7fd1
-    client->secure.pending.prev = &proxy->pending_writes;
5b7fd1
+    uv_mutex_lock(&proxy->mutex);
5b7fd1
+    if (proxy->pending_writes == NULL) {
5b7fd1
+    	proxy->pending_writes = client;
5b7fd1
+	client->secure.pending.prev = client->secure.pending.next = NULL;
5b7fd1
+    }
5b7fd1
+    else {
5b7fd1
+    	for (c=proxy->pending_writes; c->secure.pending.next; c = c->secure.pending.next)
5b7fd1
+	    ; /**/
5b7fd1
+	c->secure.pending.next = client;
5b7fd1
+	client->secure.pending.prev = c;
5b7fd1
+    }
5b7fd1
     client->secure.pending.queued = 1;
5b7fd1
-
5b7fd1
-    proxy->pending_writes = client;
5b7fd1
+    uv_mutex_unlock(&proxy->mutex);
5b7fd1
 }
5b7fd1
 
5b7fd1
 static void
5b7fd1
@@ -135,10 +157,12 @@ flush_ssl_buffer(struct client *client)
5b7fd1
 void
5b7fd1
 flush_secure_module(struct proxy *proxy)
5b7fd1
 {
5b7fd1
-    struct client	*client, **head = &proxy->pending_writes;
5b7fd1
+    struct client	*client, **head;
5b7fd1
     size_t		i, used;
5b7fd1
     int			sts;
5b7fd1
 
5b7fd1
+    uv_mutex_lock(&proxy->mutex);
5b7fd1
+    head = &proxy->pending_writes;
5b7fd1
     while ((client = *head) != NULL) {
5b7fd1
 	flush_ssl_buffer(client);
5b7fd1
 
5b7fd1
@@ -188,6 +212,7 @@ flush_secure_module(struct proxy *proxy)
5b7fd1
 		    sizeof(uv_buf_t) * client->secure.pending.writes_count);
5b7fd1
 	}
5b7fd1
     }
5b7fd1
+    uv_mutex_unlock(&proxy->mutex);
5b7fd1
 }
5b7fd1
 
5b7fd1
 void
5b7fd1
diff --git a/src/pmproxy/src/server.c b/src/pmproxy/src/server.c
5b7fd1
index 612d3613b..b5c5d84de 100644
5b7fd1
--- a/src/pmproxy/src/server.c
5b7fd1
+++ b/src/pmproxy/src/server.c
5b7fd1
@@ -149,6 +149,7 @@ server_init(int portcount, const char *localpath)
5b7fd1
 			pmGetProgname());
5b7fd1
 	return NULL;
5b7fd1
     }
5b7fd1
+    uv_mutex_init(&proxy->mutex);
5b7fd1
 
5b7fd1
     count = portcount + (*localpath ? 1 : 0);
5b7fd1
     if (count) {
5b7fd1
@@ -251,6 +252,7 @@ void
5b7fd1
 client_put(struct client *client)
5b7fd1
 {
5b7fd1
     unsigned int	refcount;
5b7fd1
+    struct proxy	*proxy = client->proxy;
5b7fd1
 
5b7fd1
     uv_mutex_lock(&client->mutex);
5b7fd1
     assert(client->refcount);
5b7fd1
@@ -259,9 +261,11 @@ client_put(struct client *client)
5b7fd1
 
5b7fd1
     if (refcount == 0) {
5b7fd1
 	/* remove client from the doubly-linked list */
5b7fd1
+	uv_mutex_lock(&proxy->mutex);
5b7fd1
 	if (client->next != NULL)
5b7fd1
 	    client->next->prev = client->prev;
5b7fd1
 	*client->prev = client->next;
5b7fd1
+	uv_mutex_unlock(&proxy->mutex);
5b7fd1
 
5b7fd1
 	if (client->protocol & STREAM_PCP)
5b7fd1
 	    on_pcp_client_close(client);
5b7fd1
@@ -514,10 +518,12 @@ on_client_connection(uv_stream_t *stream, int status)
5b7fd1
     client->proxy = proxy;
5b7fd1
 
5b7fd1
     /* insert client into doubly-linked list at the head */
5b7fd1
+    uv_mutex_lock(&proxy->mutex);
5b7fd1
     if ((client->next = proxy->first) != NULL)
5b7fd1
 	proxy->first->prev = &client->next;
5b7fd1
     proxy->first = client;
5b7fd1
     client->prev = &proxy->first;
5b7fd1
+    uv_mutex_unlock(&proxy->mutex);
5b7fd1
 
5b7fd1
     status = uv_read_start((uv_stream_t *)&client->stream.u.tcp,
5b7fd1
 			    on_buffer_alloc, on_client_read);
5b7fd1
diff --git a/src/pmproxy/src/server.h b/src/pmproxy/src/server.h
5b7fd1
index f0b7a5f68..f93daeff4 100644
5b7fd1
--- a/src/pmproxy/src/server.h
5b7fd1
+++ b/src/pmproxy/src/server.h
5b7fd1
@@ -118,7 +118,7 @@ typedef struct secure_client {
5b7fd1
     BIO			*write;
5b7fd1
     struct {
5b7fd1
 	struct client	*next;
5b7fd1
-	struct client	**prev;
5b7fd1
+	struct client	*prev;
5b7fd1
 	unsigned int	queued;
5b7fd1
 	size_t		writes_count;
5b7fd1
 	uv_buf_t	*writes_buffer;
5b7fd1
@@ -166,6 +166,7 @@ typedef struct proxy {
5b7fd1
     struct dict		*config;	/* configuration dictionary */
5b7fd1
     uv_loop_t		*events;	/* global, async event loop */
5b7fd1
     uv_callback_t	write_callbacks;
5b7fd1
+    uv_mutex_t		mutex;		/* protects client lists and pending writes */
5b7fd1
 } proxy;
5b7fd1
 
5b7fd1
 extern void proxylog(pmLogLevel, sds, void *);