diff --git a/.gitignore b/.gitignore index 9244c22..6d17819 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -SOURCES/grafana-pcp-1.0.5.tar.gz -SOURCES/grafana-pcp-deps-1.0.5.tar.xz +SOURCES/grafana-pcp-2.0.2.tar.gz +SOURCES/grafana-pcp-deps-2.0.2.tar.xz diff --git a/.grafana-pcp.metadata b/.grafana-pcp.metadata index 4056d29..aca3298 100644 --- a/.grafana-pcp.metadata +++ b/.grafana-pcp.metadata @@ -1,2 +1,2 @@ -cad0edd0cf8126b104a3caa5daca2a286a07ddce SOURCES/grafana-pcp-1.0.5.tar.gz -ab4710bc6471ed6af38bc4180cd05d14333866a3 SOURCES/grafana-pcp-deps-1.0.5.tar.xz +72d5789fd9277bc9816f6a0590bb5f044ad1175c SOURCES/grafana-pcp-2.0.2.tar.gz +226f68fa48b86eb4dfdcca73a79ab292adaa6503 SOURCES/grafana-pcp-deps-2.0.2.tar.xz diff --git a/SOURCES/000-redis-support-wildcards-in-metric-names.patch b/SOURCES/000-redis-support-wildcards-in-metric-names.patch deleted file mode 100644 index f088b76..0000000 --- a/SOURCES/000-redis-support-wildcards-in-metric-names.patch +++ /dev/null @@ -1,92 +0,0 @@ -diff --git a/src/datasources/redis/datasource.ts b/src/datasources/redis/datasource.ts -index 6076585..aea11fc 100644 ---- a/src/datasources/redis/datasource.ts -+++ b/src/datasources/redis/datasource.ts -@@ -96,6 +96,7 @@ export class PCPRedisDatasource { - } - - async handleTarget(instancesValuesGroupedBySeries: Record, -+ metricNames: Record, - descriptions: any, labels: any, target: QueryTarget): Promise { - const metrics: Metric[] = []; - -@@ -125,7 +126,7 @@ export class PCPRedisDatasource { - } - - metrics.push({ -- name: target.expr, // TODO: metric, not expression -+ name: metricNames[series], - instances: metricInstances - }); - } -@@ -179,10 +180,11 @@ export class PCPRedisDatasource { - - const instances = await this.pmSeriesSrv.getValues(seriesList, { start, finish, samples }); - const descriptions = await this.pmSeriesSrv.getDescriptions(seriesList); -+ const metricNames = await this.pmSeriesSrv.getMetricNames(seriesList); - const instanceValuesGroupedBySeries = _.groupBy(instances, "series"); - const labels = this.pmSeriesSrv.getMetricAndInstanceLabels(seriesList); - const targetResults = await Promise.all(targets.map(target => this.handleTarget( -- _.pick(instanceValuesGroupedBySeries, seriesByExpr[target.expr]), descriptions, labels, target -+ _.pick(instanceValuesGroupedBySeries, seriesByExpr[target.expr]), metricNames, descriptions, labels, target - ))); - const panelData = this.transformations.transform(query, targetResults, PCPRedisDatasource.defaultLegendFormatter); - return { -diff --git a/src/datasources/redis/pmseries_srv.ts b/src/datasources/redis/pmseries_srv.ts -index e3c59c6..f3686a5 100644 ---- a/src/datasources/redis/pmseries_srv.ts -+++ b/src/datasources/redis/pmseries_srv.ts -@@ -8,6 +8,11 @@ export interface LabelsResponse { - labels: Labels; - } - -+export interface MetricNamesResponse { -+ series: string; -+ name: string; -+} -+ - class PmSeriesApi { - - constructor(private datasourceRequest: DatasourceRequestFn, private url: string) { -@@ -67,6 +72,15 @@ class PmSeriesApi { - return _.isArray(metrics) ? metrics : []; // TODO: on error (no metrics found), pmproxy returns an object (should be an empty array) - } - -+ async metricsSeries(series: string[]): Promise { -+ const response = await this.datasourceRequest({ -+ url: `${this.url}/series/metrics`, -+ params: { series: series.join(',') } -+ }); -+ const metricNames = response.data; -+ return _.isArray(metricNames) ? metricNames : []; // TODO: on error (no metrics found), pmproxy returns an object (should be an empty array) -+ } -+ - async labels(series: string[]): Promise { - const response = await this.datasourceRequest({ - url: `${this.url}/series/labels`, -@@ -84,6 +98,7 @@ export class PmSeriesSrv { - private instanceCache: Record> = {}; // instanceCache[series][instance] = instance; - private labelCache: Record = {}; // labelCache[series_or_instance] = labels; - private metricNamesCache: Record = {}; // metricNamesCache[prefix] = name[]; -+ private metricNameOfSeriesCache: Record = {}; - - constructor(datasourceRequest: DatasourceRequestFn, url: string) { - this.pmSeriesApi = new PmSeriesApi(datasourceRequest, url); -@@ -108,6 +123,17 @@ export class PmSeriesSrv { - return _.pick(this.descriptionCache, series); - } - -+ async getMetricNames(series: string[]): Promise> { -+ const requiredSeries = _.difference(series, Object.keys(this.metricNameOfSeriesCache)); -+ if (requiredSeries.length > 0) { -+ const metricNames = await this.pmSeriesApi.metricsSeries(requiredSeries); -+ for (const metricName of metricNames) { -+ this.metricNameOfSeriesCache[metricName.series] = metricName.name; -+ } -+ } -+ return _.pick(this.metricNameOfSeriesCache, series); -+ } -+ - async getInstances(series: string[], ignoreCache = false): Promise>> { - const requiredSeries = ignoreCache ? series : _.difference(series, Object.keys(this.instanceCache)); - if (requiredSeries.length > 0) { diff --git a/SOURCES/001-fix-empty-container-dashboard.patch b/SOURCES/001-fix-empty-container-dashboard.patch new file mode 100644 index 0000000..5c57bfd --- /dev/null +++ b/SOURCES/001-fix-empty-container-dashboard.patch @@ -0,0 +1,30 @@ +diff --git a/src/dashboards/pcp-vector-container-overview.json b/src/dashboards/pcp-vector-container-overview.json +index a1a75c2..a674d74 100644 +--- a/src/dashboards/pcp-vector-container-overview.json ++++ b/src/dashboards/pcp-vector-container-overview.json +@@ -127,7 +127,7 @@ + "format": "percentunit", + "label": null, + "logBase": 1, +- "max": "1", ++ "max": null, + "min": "0", + "show": true + }, +diff --git a/src/datasources/lib/datasource_base.ts b/src/datasources/lib/datasource_base.ts +index db0167b..4bca9c0 100644 +--- a/src/datasources/lib/datasource_base.ts ++++ b/src/datasources/lib/datasource_base.ts +@@ -150,7 +150,11 @@ export abstract class PmapiDatasourceBase { + container: container, + endpoint: this.getOrCreateEndpoint(url, container) + }; +- }); ++ }) ++ // getConnectionParams only processes target.container if it is not blank ++ // the only case when container is "" is because it was $container, but the ++ // $container dashboard variable is empty (viewing "all" containers, but none exists) ++ .filter(target => target.container !== ""); + } + + async applyTransformations(pmapiSrv: PmapiSrv, results: TargetResult) { diff --git a/SOURCES/001-redis-fix-legend-and-label-support.patch b/SOURCES/001-redis-fix-legend-and-label-support.patch deleted file mode 100644 index 61f749d..0000000 --- a/SOURCES/001-redis-fix-legend-and-label-support.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/src/datasources/redis/datasource.ts b/src/datasources/redis/datasource.ts -index aea11fc..5980ffa 100644 ---- a/src/datasources/redis/datasource.ts -+++ b/src/datasources/redis/datasource.ts -@@ -138,7 +138,7 @@ export class PCPRedisDatasource { - } - - static defaultLegendFormatter(metric: string, instance: MetricInstance | undefined, labels: Record) { -- let label = instance && instance.id !== null ? instance.name : metric; -+ let label = instance && instance.id !== "" ? instance.name : metric; - if (!_.isEmpty(labels)) { - const pairs: string[] = []; - for (const label of ["hostname", "source"]) { -@@ -182,7 +182,7 @@ export class PCPRedisDatasource { - const descriptions = await this.pmSeriesSrv.getDescriptions(seriesList); - const metricNames = await this.pmSeriesSrv.getMetricNames(seriesList); - const instanceValuesGroupedBySeries = _.groupBy(instances, "series"); -- const labels = this.pmSeriesSrv.getMetricAndInstanceLabels(seriesList); -+ const labels = await this.pmSeriesSrv.getMetricAndInstanceLabels(seriesList); - const targetResults = await Promise.all(targets.map(target => this.handleTarget( - _.pick(instanceValuesGroupedBySeries, seriesByExpr[target.expr]), metricNames, descriptions, labels, target - ))); diff --git a/SOURCES/002-add-process-name-and-pid-in-flamegraph-dashboard.patch b/SOURCES/002-add-process-name-and-pid-in-flamegraph-dashboard.patch new file mode 100644 index 0000000..bb45ef0 --- /dev/null +++ b/SOURCES/002-add-process-name-and-pid-in-flamegraph-dashboard.patch @@ -0,0 +1,22 @@ +diff --git a/src/dashboards/pcp-bpftrace-flame-graphs.json b/src/dashboards/pcp-bpftrace-flame-graphs.json +index 8f3d0ba..a05dbb9 100644 +--- a/src/dashboards/pcp-bpftrace-flame-graphs.json ++++ b/src/dashboards/pcp-bpftrace-flame-graphs.json +@@ -150,7 +150,7 @@ + }, + "targets": [ + { +- "expr": "/*\n * sample kernel stacks every 99 Hz, and clear map every 5 seconds\n *\n * 30-Oct-2019 Andreas Gerstmayr Created this.\n */\n// include: @stacks\n// custom-output-block\n\nprofile:hz:99 { @stacks[kstack] = count(); }\n\ninterval:s:1 {\n print(@stacks);\n @cnt++;\n if (@cnt >= 5) {\n clear(@stacks);\n @cnt = 0;\n }\n}", ++ "expr": "/*\n * sample kernel stacks every 99 Hz, and clear map every 5 seconds\n *\n * 30-Oct-2019 Andreas Gerstmayr Created this.\n * 7-Aug-2020 Andreas Gerstmayr Added process name and PID.\n */\n// include: @stacks\n// custom-output-block\n\nprofile:hz:99 { @stacks[comm,pid,kstack] = count(); }\n\ninterval:s:1 {\n print(@stacks);\n @cnt++;\n if (@cnt >= 5) {\n clear(@stacks);\n @cnt = 0;\n }\n}", + "format": "flamegraph", + "minPcpVersion": "5.0.2", + "refId": "A" +@@ -177,7 +177,7 @@ + }, + "targets": [ + { +- "expr": "/*\n * sample user stacks every 99 Hz, and clear map every 5 seconds\n *\n * 30-Oct-2019 Andreas Gerstmayr Created this.\n */\n// include: @stacks\n// custom-output-block\n\nprofile:hz:99 { @stacks[ustack] = count(); }\n\ninterval:s:1 {\n print(@stacks);\n @cnt++;\n if (@cnt >= 5) {\n clear(@stacks);\n @cnt = 0;\n }\n}", ++ "expr": "/*\n * sample user stacks every 99 Hz, and clear map every 5 seconds\n *\n * 30-Oct-2019 Andreas Gerstmayr Created this.\n * 7-Aug-2020 Andreas Gerstmayr Added process name and PID.\n */\n// include: @stacks\n// custom-output-block\n\nprofile:hz:99 { @stacks[comm,pid,ustack] = count(); }\n\ninterval:s:1 {\n print(@stacks);\n @cnt++;\n if (@cnt >= 5) {\n clear(@stacks);\n @cnt = 0;\n }\n}", + "format": "flamegraph", + "minPcpVersion": "5.0.2", + "refId": "A" diff --git a/SOURCES/002-redis-pass-correct-timespec.patch b/SOURCES/002-redis-pass-correct-timespec.patch deleted file mode 100644 index c9a2464..0000000 --- a/SOURCES/002-redis-pass-correct-timespec.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/src/datasources/redis/datasource.ts b/src/datasources/redis/datasource.ts -index 5980ffa..1d108a9 100644 ---- a/src/datasources/redis/datasource.ts -+++ b/src/datasources/redis/datasource.ts -@@ -170,15 +170,14 @@ export class PCPRedisDatasource { - } - } - -- const sampleIntervalSec = 60; // guessed sample interval -+ const interval = query.intervalMs / 1000; // seconds - // request a bigger time frame to fill the chart (otherwise left and right border of chart is empty) - // because of the rate conversation of counters first datapoint is "lost" -> expand timeframe at the beginning -- const start = Math.round(query.range.from.valueOf() / 1000) - 2 * sampleIntervalSec; -- const finish = Math.round(query.range.to.valueOf() / 1000) + sampleIntervalSec; -- const samples = Math.round((query.range.to.valueOf() - query.range.from.valueOf()) / query.intervalMs); -- // const interval = query.interval; -+ const additionalTimeRange = Math.max(interval, 60); // 60s is the default sample interval of pmlogger -+ const start = Math.floor(query.range.from.valueOf() / 1000 - 2 * additionalTimeRange); // seconds -+ const finish = Math.ceil(query.range.to.valueOf() / 1000 + additionalTimeRange); // seconds - -- const instances = await this.pmSeriesSrv.getValues(seriesList, { start, finish, samples }); -+ const instances = await this.pmSeriesSrv.getValues(seriesList, { start, finish, interval }); - const descriptions = await this.pmSeriesSrv.getDescriptions(seriesList); - const metricNames = await this.pmSeriesSrv.getMetricNames(seriesList); - const instanceValuesGroupedBySeries = _.groupBy(instances, "series"); --- -2.21.1 - diff --git a/SOURCES/create_dependency_bundle.sh b/SOURCES/create_dependency_bundle.sh index 041fbb6..fbcd787 100755 --- a/SOURCES/create_dependency_bundle.sh +++ b/SOURCES/create_dependency_bundle.sh @@ -1,4 +1,4 @@ -#!/bin/sh -eu +#!/bin/bash -eu SRC=$(readlink -f "${1:?Usage: $0 source destination}") DEST=$(readlink -f "${2:?Usage: $0 source destination}") @@ -13,10 +13,10 @@ else PATCHES="" fi -pushd $(mktemp -d) +pushd "$(mktemp -d)" echo Extracting sources... -tar xfz $SRC +tar xfz "$SRC" cd grafana-pcp-* echo Applying patches... @@ -32,6 +32,6 @@ echo Removing files with licensing issues... rm -rf node_modules/node-notifier echo Compressing... -XZ_OPT=-9 tar cJf $DEST node_modules +XZ_OPT=-9 tar cJf "$DEST" node_modules popd diff --git a/SOURCES/list_bundled_nodejs_packages.py b/SOURCES/list_bundled_nodejs_packages.py new file mode 100755 index 0000000..a42e223 --- /dev/null +++ b/SOURCES/list_bundled_nodejs_packages.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +import sys +import os.path +import tarfile +from io import TextIOWrapper +import json +import re +from packaging import version + + +def read_declared_pkgs(f): + package_json = json.load(f) + return list(package_json['devDependencies'].keys()) + list(package_json['dependencies'].keys()) + + +def read_installed_pkgs(f): + lockfile = f.read() + return re.findall(r'^"?' # can start with a " + r'(.+?)@.+(?:,.*)?:\n' # characters up to @ + r' version "(.+)"', # and the version + lockfile, re.MULTILINE) + + +def list_provides(declared_pkgs, installed_pkgs): + for declared_pkg in declared_pkgs: + # there can be multiple versions installed of one package (transitive dependencies) + # but rpm doesn't support Provides: with a single package and multiple versions + # so let's declare the oldest version here + versions = [version.parse(pkg_version) + for pkg_name, pkg_version in installed_pkgs if pkg_name == declared_pkg] + oldest_version = sorted(versions)[0] + yield f"Provides: bundled(nodejs-{declared_pkg}) = {oldest_version}" + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} grafana-pcp-X.Y.Z.tar.gz", file=sys.stdout) + sys.exit(1) + + source_archive_path = sys.argv[1] + root_dir = os.path.basename(source_archive_path)[:-len('.tar.gz')] + with tarfile.open(source_archive_path) as tar: + package_json = TextIOWrapper(tar.extractfile(f'{root_dir}/package.json')) + declared_pkgs = read_declared_pkgs(package_json) + + yarn_lock = TextIOWrapper(tar.extractfile(f'{root_dir}/yarn.lock')) + installed_pkgs = read_installed_pkgs(yarn_lock) + + provides = list_provides(declared_pkgs, installed_pkgs) + for provide in sorted(provides): + print(provide) diff --git a/SPECS/grafana-pcp.spec b/SPECS/grafana-pcp.spec index e931fbe..282b97f 100644 --- a/SPECS/grafana-pcp.spec +++ b/SPECS/grafana-pcp.spec @@ -1,5 +1,5 @@ Name: grafana-pcp -Version: 1.0.5 +Version: 2.0.2 Release: 3%{?dist} Summary: Performance Co-Pilot Grafana Plugin @@ -15,66 +15,66 @@ URL: %{github} Source0: %{github}/archive/v%{version}/%{name}-%{version}.tar.gz Source1: grafana-pcp-deps-%{version}.tar.xz Source2: create_dependency_bundle.sh +Source3: list_bundled_nodejs_packages.py -Patch0: 000-redis-support-wildcards-in-metric-names.patch -Patch1: 001-redis-fix-legend-and-label-support.patch -Patch2: 002-redis-pass-correct-timespec.patch +Patch1: 001-fix-empty-container-dashboard.patch +Patch2: 002-add-process-name-and-pid-in-flamegraph-dashboard.patch BuildRequires: nodejs -Requires: grafana >= 6.2.2, grafana < 6.4.0 +Requires: grafana >= 6.6.0 Suggests: pcp >= 5.0.0 Suggests: redis >= 5.0.0 Suggests: bpftrace >= 0.9.2 # Obsolete old webapps -Obsoletes: pcp-webjs -Obsoletes: pcp-webapp-blinkenlights -Obsoletes: pcp-webapp-grafana -Obsoletes: pcp-webapp-graphite -Obsoletes: pcp-webapp-vector +Obsoletes: pcp-webjs <= 4.3.4 +Obsoletes: pcp-webapp-blinkenlights <= 4.3.4 +Obsoletes: pcp-webapp-grafana <= 4.3.4 +Obsoletes: pcp-webapp-graphite <= 4.3.4 +Obsoletes: pcp-webapp-vector <= 4.3.4 # Bundled npm packages -Provides: bundled(nodejs-@babel/cli) = 7.5.5 -Provides: bundled(nodejs-@babel/core) = 7.5.5 -Provides: bundled(nodejs-@babel/preset-env) = 7.5.5 -Provides: bundled(nodejs-@babel/preset-react) = 7.0.0 -Provides: bundled(nodejs-@babel/preset-typescript) = 7.3.3 -Provides: bundled(nodejs-@grafana/data) = 6.4.0 -Provides: bundled(nodejs-@grafana/ui) = 6.4.0 +Provides: bundled(nodejs-@babel/cli) = 7.8.4 +Provides: bundled(nodejs-@babel/core) = 7.8.4 +Provides: bundled(nodejs-@babel/preset-env) = 7.8.4 +Provides: bundled(nodejs-@babel/preset-react) = 7.8.3 +Provides: bundled(nodejs-@babel/preset-typescript) = 7.8.3 +Provides: bundled(nodejs-@grafana/data) = 6.6.0 +Provides: bundled(nodejs-@grafana/ui) = 6.6.0 Provides: bundled(nodejs-@types/benchmark) = 1.0.31 Provides: bundled(nodejs-@types/d3) = 5.7.2 Provides: bundled(nodejs-@types/grafana) = 4.6.3 -Provides: bundled(nodejs-@types/jest) = 24.0.17 -Provides: bundled(nodejs-@types/lodash) = 4.14.136 -Provides: bundled(nodejs-babel-jest) = 24.8.0 +Provides: bundled(nodejs-@types/jest) = 24.9.1 +Provides: bundled(nodejs-@types/lodash) = 4.14.149 +Provides: bundled(nodejs-babel-jest) = 24.9.0 Provides: bundled(nodejs-babel-loader) = 8.0.6 Provides: bundled(nodejs-babel-plugin-angularjs-annotate) = 0.10.0 Provides: bundled(nodejs-benchmark) = 2.1.4 Provides: bundled(nodejs-clean-webpack-plugin) = 0.1.19 Provides: bundled(nodejs-copy-webpack-plugin) = 5.1.1 -Provides: bundled(nodejs-core-js) = 3.1.4 +Provides: bundled(nodejs-core-js) = 1.2.7 Provides: bundled(nodejs-css-loader) = 1.0.1 -Provides: bundled(nodejs-d3-flame-graph) = 2.1.8 +Provides: bundled(nodejs-d3-flame-graph) = 2.1.9 Provides: bundled(nodejs-d3-selection) = 1.4.0 Provides: bundled(nodejs-expr-eval) = 1.2.3 -Provides: bundled(nodejs-jest) = 24.8.0 -Provides: bundled(nodejs-jest-date-mock) = 1.0.7 +Provides: bundled(nodejs-jest) = 24.9.0 +Provides: bundled(nodejs-jest-date-mock) = 1.0.8 Provides: bundled(nodejs-jsdom) = 9.12.0 Provides: bundled(nodejs-lodash) = 4.17.15 -Provides: bundled(nodejs-memoize-one) = 5.1.1 -Provides: bundled(nodejs-mocha) = 6.2.0 +Provides: bundled(nodejs-memoize-one) = 4.1.0 +Provides: bundled(nodejs-mocha) = 6.2.2 Provides: bundled(nodejs-prunk) = 1.3.1 Provides: bundled(nodejs-q) = 1.5.1 -Provides: bundled(nodejs-regenerator-runtime) = 0.12.1 +Provides: bundled(nodejs-regenerator-runtime) = 0.11.1 Provides: bundled(nodejs-request) = 2.88.0 Provides: bundled(nodejs-style-loader) = 0.22.1 -Provides: bundled(nodejs-ts-jest) = 24.0.2 +Provides: bundled(nodejs-ts-jest) = 24.3.0 Provides: bundled(nodejs-ts-loader) = 4.5.0 -Provides: bundled(nodejs-tslint) = 5.18.0 -Provides: bundled(nodejs-tslint-config-airbnb) = 5.11.1 -Provides: bundled(nodejs-typescript) = 3.5.3 -Provides: bundled(nodejs-webpack) = 4.39.1 -Provides: bundled(nodejs-webpack-cli) = 3.3.6 +Provides: bundled(nodejs-tslint) = 5.20.1 +Provides: bundled(nodejs-tslint-config-airbnb) = 5.11.2 +Provides: bundled(nodejs-typescript) = 3.7.5 +Provides: bundled(nodejs-webpack) = 4.41.5 +Provides: bundled(nodejs-webpack-cli) = 3.3.10 %description @@ -85,7 +85,6 @@ bpftrace scripts from pmdabpftrace(1), as well as several dashboards. %prep %setup -q %setup -q -a 1 -%patch0 -p1 %patch1 -p1 %patch2 -p1 @@ -110,6 +109,15 @@ cp -a dist/* %{buildroot}/%{install_dir} %doc README.md %changelog +* Fri Aug 07 2020 Andreas Gerstmayr 2.0.2-3 +- bpftrace: show process name and PID in flame graphs + +* Fri Jul 24 2020 Andreas Gerstmayr 2.0.2-2 +- vector: do not show all cgroups in the container overview dashboard in case no containers are present on the system + +* Mon May 11 2020 Andreas Gerstmayr 2.0.2-1 +- update to upstream version 2.0.2, see CHANGELOG + * Tue Jan 28 2020 Andreas Gerstmayr 1.0.5-3 - redis: pass correct timespec to pmproxy (fixes empty graphs for large time ranges)