diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9244c22 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +SOURCES/grafana-pcp-1.0.5.tar.gz +SOURCES/grafana-pcp-deps-1.0.5.tar.xz diff --git a/.grafana-pcp.metadata b/.grafana-pcp.metadata new file mode 100644 index 0000000..4056d29 --- /dev/null +++ b/.grafana-pcp.metadata @@ -0,0 +1,2 @@ +cad0edd0cf8126b104a3caa5daca2a286a07ddce SOURCES/grafana-pcp-1.0.5.tar.gz +ab4710bc6471ed6af38bc4180cd05d14333866a3 SOURCES/grafana-pcp-deps-1.0.5.tar.xz diff --git a/SOURCES/000-redis-support-wildcards-in-metric-names.patch b/SOURCES/000-redis-support-wildcards-in-metric-names.patch new file mode 100644 index 0000000..f088b76 --- /dev/null +++ b/SOURCES/000-redis-support-wildcards-in-metric-names.patch @@ -0,0 +1,92 @@ +diff --git a/src/datasources/redis/datasource.ts b/src/datasources/redis/datasource.ts +index 6076585..aea11fc 100644 +--- a/src/datasources/redis/datasource.ts ++++ b/src/datasources/redis/datasource.ts +@@ -96,6 +96,7 @@ export class PCPRedisDatasource { + } + + async handleTarget(instancesValuesGroupedBySeries: Record, ++ metricNames: Record, + descriptions: any, labels: any, target: QueryTarget): Promise { + const metrics: Metric[] = []; + +@@ -125,7 +126,7 @@ export class PCPRedisDatasource { + } + + metrics.push({ +- name: target.expr, // TODO: metric, not expression ++ name: metricNames[series], + instances: metricInstances + }); + } +@@ -179,10 +180,11 @@ export class PCPRedisDatasource { + + const instances = await this.pmSeriesSrv.getValues(seriesList, { start, finish, samples }); + const descriptions = await this.pmSeriesSrv.getDescriptions(seriesList); ++ const metricNames = await this.pmSeriesSrv.getMetricNames(seriesList); + const instanceValuesGroupedBySeries = _.groupBy(instances, "series"); + const labels = this.pmSeriesSrv.getMetricAndInstanceLabels(seriesList); + const targetResults = await Promise.all(targets.map(target => this.handleTarget( +- _.pick(instanceValuesGroupedBySeries, seriesByExpr[target.expr]), descriptions, labels, target ++ _.pick(instanceValuesGroupedBySeries, seriesByExpr[target.expr]), metricNames, descriptions, labels, target + ))); + const panelData = this.transformations.transform(query, targetResults, PCPRedisDatasource.defaultLegendFormatter); + return { +diff --git a/src/datasources/redis/pmseries_srv.ts b/src/datasources/redis/pmseries_srv.ts +index e3c59c6..f3686a5 100644 +--- a/src/datasources/redis/pmseries_srv.ts ++++ b/src/datasources/redis/pmseries_srv.ts +@@ -8,6 +8,11 @@ export interface LabelsResponse { + labels: Labels; + } + ++export interface MetricNamesResponse { ++ series: string; ++ name: string; ++} ++ + class PmSeriesApi { + + constructor(private datasourceRequest: DatasourceRequestFn, private url: string) { +@@ -67,6 +72,15 @@ class PmSeriesApi { + return _.isArray(metrics) ? metrics : []; // TODO: on error (no metrics found), pmproxy returns an object (should be an empty array) + } + ++ async metricsSeries(series: string[]): Promise { ++ const response = await this.datasourceRequest({ ++ url: `${this.url}/series/metrics`, ++ params: { series: series.join(',') } ++ }); ++ const metricNames = response.data; ++ return _.isArray(metricNames) ? metricNames : []; // TODO: on error (no metrics found), pmproxy returns an object (should be an empty array) ++ } ++ + async labels(series: string[]): Promise { + const response = await this.datasourceRequest({ + url: `${this.url}/series/labels`, +@@ -84,6 +98,7 @@ export class PmSeriesSrv { + private instanceCache: Record> = {}; // instanceCache[series][instance] = instance; + private labelCache: Record = {}; // labelCache[series_or_instance] = labels; + private metricNamesCache: Record = {}; // metricNamesCache[prefix] = name[]; ++ private metricNameOfSeriesCache: Record = {}; + + constructor(datasourceRequest: DatasourceRequestFn, url: string) { + this.pmSeriesApi = new PmSeriesApi(datasourceRequest, url); +@@ -108,6 +123,17 @@ export class PmSeriesSrv { + return _.pick(this.descriptionCache, series); + } + ++ async getMetricNames(series: string[]): Promise> { ++ const requiredSeries = _.difference(series, Object.keys(this.metricNameOfSeriesCache)); ++ if (requiredSeries.length > 0) { ++ const metricNames = await this.pmSeriesApi.metricsSeries(requiredSeries); ++ for (const metricName of metricNames) { ++ this.metricNameOfSeriesCache[metricName.series] = metricName.name; ++ } ++ } ++ return _.pick(this.metricNameOfSeriesCache, series); ++ } ++ + async getInstances(series: string[], ignoreCache = false): Promise>> { + const requiredSeries = ignoreCache ? series : _.difference(series, Object.keys(this.instanceCache)); + if (requiredSeries.length > 0) { diff --git a/SOURCES/001-redis-fix-legend-and-label-support.patch b/SOURCES/001-redis-fix-legend-and-label-support.patch new file mode 100644 index 0000000..61f749d --- /dev/null +++ b/SOURCES/001-redis-fix-legend-and-label-support.patch @@ -0,0 +1,22 @@ +diff --git a/src/datasources/redis/datasource.ts b/src/datasources/redis/datasource.ts +index aea11fc..5980ffa 100644 +--- a/src/datasources/redis/datasource.ts ++++ b/src/datasources/redis/datasource.ts +@@ -138,7 +138,7 @@ export class PCPRedisDatasource { + } + + static defaultLegendFormatter(metric: string, instance: MetricInstance | undefined, labels: Record) { +- let label = instance && instance.id !== null ? instance.name : metric; ++ let label = instance && instance.id !== "" ? instance.name : metric; + if (!_.isEmpty(labels)) { + const pairs: string[] = []; + for (const label of ["hostname", "source"]) { +@@ -182,7 +182,7 @@ export class PCPRedisDatasource { + const descriptions = await this.pmSeriesSrv.getDescriptions(seriesList); + const metricNames = await this.pmSeriesSrv.getMetricNames(seriesList); + const instanceValuesGroupedBySeries = _.groupBy(instances, "series"); +- const labels = this.pmSeriesSrv.getMetricAndInstanceLabels(seriesList); ++ const labels = await this.pmSeriesSrv.getMetricAndInstanceLabels(seriesList); + const targetResults = await Promise.all(targets.map(target => this.handleTarget( + _.pick(instanceValuesGroupedBySeries, seriesByExpr[target.expr]), metricNames, descriptions, labels, target + ))); diff --git a/SOURCES/002-redis-pass-correct-timespec.patch b/SOURCES/002-redis-pass-correct-timespec.patch new file mode 100644 index 0000000..c9a2464 --- /dev/null +++ b/SOURCES/002-redis-pass-correct-timespec.patch @@ -0,0 +1,28 @@ +diff --git a/src/datasources/redis/datasource.ts b/src/datasources/redis/datasource.ts +index 5980ffa..1d108a9 100644 +--- a/src/datasources/redis/datasource.ts ++++ b/src/datasources/redis/datasource.ts +@@ -170,15 +170,14 @@ export class PCPRedisDatasource { + } + } + +- const sampleIntervalSec = 60; // guessed sample interval ++ const interval = query.intervalMs / 1000; // seconds + // request a bigger time frame to fill the chart (otherwise left and right border of chart is empty) + // because of the rate conversation of counters first datapoint is "lost" -> expand timeframe at the beginning +- const start = Math.round(query.range.from.valueOf() / 1000) - 2 * sampleIntervalSec; +- const finish = Math.round(query.range.to.valueOf() / 1000) + sampleIntervalSec; +- const samples = Math.round((query.range.to.valueOf() - query.range.from.valueOf()) / query.intervalMs); +- // const interval = query.interval; ++ const additionalTimeRange = Math.max(interval, 60); // 60s is the default sample interval of pmlogger ++ const start = Math.floor(query.range.from.valueOf() / 1000 - 2 * additionalTimeRange); // seconds ++ const finish = Math.ceil(query.range.to.valueOf() / 1000 + additionalTimeRange); // seconds + +- const instances = await this.pmSeriesSrv.getValues(seriesList, { start, finish, samples }); ++ const instances = await this.pmSeriesSrv.getValues(seriesList, { start, finish, interval }); + const descriptions = await this.pmSeriesSrv.getDescriptions(seriesList); + const metricNames = await this.pmSeriesSrv.getMetricNames(seriesList); + const instanceValuesGroupedBySeries = _.groupBy(instances, "series"); +-- +2.21.1 + diff --git a/SOURCES/create_dependency_bundle.sh b/SOURCES/create_dependency_bundle.sh new file mode 100755 index 0000000..041fbb6 --- /dev/null +++ b/SOURCES/create_dependency_bundle.sh @@ -0,0 +1,37 @@ +#!/bin/sh -eu + +SRC=$(readlink -f "${1:?Usage: $0 source destination}") +DEST=$(readlink -f "${2:?Usage: $0 source destination}") + +if [ -f "$DEST" ]; then + echo "File $DEST exists already." + exit 0 +fi +if [ "$#" -gt 2 ]; then + PATCHES=$(readlink -f "${@:3}") +else + PATCHES="" +fi + +pushd $(mktemp -d) + +echo Extracting sources... +tar xfz $SRC +cd grafana-pcp-* + +echo Applying patches... +for patch in $PATCHES +do + patch -p1 < $patch +done + +echo Installing dependencies... +yarn install + +echo Removing files with licensing issues... +rm -rf node_modules/node-notifier + +echo Compressing... +XZ_OPT=-9 tar cJf $DEST node_modules + +popd diff --git a/SPECS/grafana-pcp.spec b/SPECS/grafana-pcp.spec new file mode 100644 index 0000000..e931fbe --- /dev/null +++ b/SPECS/grafana-pcp.spec @@ -0,0 +1,177 @@ +Name: grafana-pcp +Version: 1.0.5 +Release: 3%{?dist} +Summary: Performance Co-Pilot Grafana Plugin + +%global github https://github.com/performancecopilot/grafana-pcp +%global install_dir %{_sharedstatedir}/grafana/plugins/grafana-pcp + +BuildArch: noarch +ExclusiveArch: %{nodejs_arches} + +License: ASL 2.0 +URL: %{github} + +Source0: %{github}/archive/v%{version}/%{name}-%{version}.tar.gz +Source1: grafana-pcp-deps-%{version}.tar.xz +Source2: create_dependency_bundle.sh + +Patch0: 000-redis-support-wildcards-in-metric-names.patch +Patch1: 001-redis-fix-legend-and-label-support.patch +Patch2: 002-redis-pass-correct-timespec.patch + +BuildRequires: nodejs +Requires: grafana >= 6.2.2, grafana < 6.4.0 +Suggests: pcp >= 5.0.0 +Suggests: redis >= 5.0.0 +Suggests: bpftrace >= 0.9.2 + +# Obsolete old webapps +Obsoletes: pcp-webjs +Obsoletes: pcp-webapp-blinkenlights +Obsoletes: pcp-webapp-grafana +Obsoletes: pcp-webapp-graphite +Obsoletes: pcp-webapp-vector + +# Bundled npm packages +Provides: bundled(nodejs-@babel/cli) = 7.5.5 +Provides: bundled(nodejs-@babel/core) = 7.5.5 +Provides: bundled(nodejs-@babel/preset-env) = 7.5.5 +Provides: bundled(nodejs-@babel/preset-react) = 7.0.0 +Provides: bundled(nodejs-@babel/preset-typescript) = 7.3.3 +Provides: bundled(nodejs-@grafana/data) = 6.4.0 +Provides: bundled(nodejs-@grafana/ui) = 6.4.0 +Provides: bundled(nodejs-@types/benchmark) = 1.0.31 +Provides: bundled(nodejs-@types/d3) = 5.7.2 +Provides: bundled(nodejs-@types/grafana) = 4.6.3 +Provides: bundled(nodejs-@types/jest) = 24.0.17 +Provides: bundled(nodejs-@types/lodash) = 4.14.136 +Provides: bundled(nodejs-babel-jest) = 24.8.0 +Provides: bundled(nodejs-babel-loader) = 8.0.6 +Provides: bundled(nodejs-babel-plugin-angularjs-annotate) = 0.10.0 +Provides: bundled(nodejs-benchmark) = 2.1.4 +Provides: bundled(nodejs-clean-webpack-plugin) = 0.1.19 +Provides: bundled(nodejs-copy-webpack-plugin) = 5.1.1 +Provides: bundled(nodejs-core-js) = 3.1.4 +Provides: bundled(nodejs-css-loader) = 1.0.1 +Provides: bundled(nodejs-d3-flame-graph) = 2.1.8 +Provides: bundled(nodejs-d3-selection) = 1.4.0 +Provides: bundled(nodejs-expr-eval) = 1.2.3 +Provides: bundled(nodejs-jest) = 24.8.0 +Provides: bundled(nodejs-jest-date-mock) = 1.0.7 +Provides: bundled(nodejs-jsdom) = 9.12.0 +Provides: bundled(nodejs-lodash) = 4.17.15 +Provides: bundled(nodejs-memoize-one) = 5.1.1 +Provides: bundled(nodejs-mocha) = 6.2.0 +Provides: bundled(nodejs-prunk) = 1.3.1 +Provides: bundled(nodejs-q) = 1.5.1 +Provides: bundled(nodejs-regenerator-runtime) = 0.12.1 +Provides: bundled(nodejs-request) = 2.88.0 +Provides: bundled(nodejs-style-loader) = 0.22.1 +Provides: bundled(nodejs-ts-jest) = 24.0.2 +Provides: bundled(nodejs-ts-loader) = 4.5.0 +Provides: bundled(nodejs-tslint) = 5.18.0 +Provides: bundled(nodejs-tslint-config-airbnb) = 5.11.1 +Provides: bundled(nodejs-typescript) = 3.5.3 +Provides: bundled(nodejs-webpack) = 4.39.1 +Provides: bundled(nodejs-webpack-cli) = 3.3.6 + + +%description +This Grafana plugin for Performance Co-Pilot includes datasources for +scalable time series from pmseries(1) and Redis, live PCP metrics and +bpftrace scripts from pmdabpftrace(1), as well as several dashboards. + +%prep +%setup -q +%setup -q -a 1 +%patch0 -p1 +%patch1 -p1 +%patch2 -p1 + +%build +rm -rf dist +./node_modules/webpack/bin/webpack.js --config webpack.config.prod.js + +# webpack/copy-webpack-plugin sometimes outputs files with mode = 666 due to reasons unknown (race condition/umask issue afaics) +chmod -Rf a+rX,u+w,g-w,o-w dist + +%check +./node_modules/jest/bin/jest.js --silent + +%install +install -d -m 755 %{buildroot}/%{install_dir} +cp -a dist/* %{buildroot}/%{install_dir} + +%files +%{install_dir} + +%license LICENSE NOTICE +%doc README.md + +%changelog +* Tue Jan 28 2020 Andreas Gerstmayr 1.0.5-3 +- redis: pass correct timespec to pmproxy (fixes empty graphs for large time ranges) + +* Tue Jan 07 2020 Andreas Gerstmayr 1.0.5-2 +- redis: support wildcards in metric names +- redis: fix legend and label support + +* Mon Dec 16 2019 Andreas Gerstmayr 1.0.5-1 +- upgrade to upstream 1.0.5 +- flame graphs: clean flame graph stacks every 5s (reduces CPU load) +- general: implement PCP version checks +- redis: set default sample interval to 60s (fixes empty graph borders) + +* Mon Dec 16 2019 Andreas Gerstmayr 1.0.3-2 +- remove node_modules/node-notifier directory from webpack (due to licensing issues) +- upgrade copy-webpack-plugin, terser-webpack-plugin and remove uglifyjs-webpack-plugin to mitigate XSS vulnerability in serialize-javascript dependency + +* Tue Nov 26 2019 Nathan Scott 1.0.3-1 +- fix flame graph dependency (flamegraph.destroy error in javascript console) + +* Tue Nov 12 2019 Andreas Gerstmayr 1.0.2-1 +- handle counter wraps (overflows) +- convert time based counters to time utilization +- flame graphs: aggregate stack counts by selected time range in the Grafana UI +- flame graphs: add option to hide idle stacks +- vector: fix container dropdown in query editor +- vector: remove container setting from datasource settings page +- redis: fix value transformations (e.g. rate conversation of counters) +- request more datapoints from the datasource to fill the borders of the graph panel + +* Fri Oct 11 2019 Andreas Gerstmayr 1.0.0-1 +- bpftrace: support for Flame Graphs +- bpftrace: context-sensitive auto completion for bpftrace probes, builtin variables and functions incl. help texts +- bpftrace: parse output of bpftrace scripts (e.g. using `printf()`) as CSV and display it in the Grafana table panel +- bpftrace: sample dashboards (BPFtrace System Analysis, BPFtrace Flame Graphs) +- vector: table output: show instance name in left column +- vector: table output: support non-matching instance names (cells of metrics which don't have the specific instance will be blank) +- vector & bpftrace: if the metric/script gets changed in the query editor, immeditately stop polling the old metric/deregister the old script +- vector & bpftrace: improve pmwebd compatibility +- misc: help texts for all datasources (visible with the **[ ? ]** button in the query editor) +- misc: renamed PCP Live to PCP Vector +- misc: logos for all datasources +- misc: improved error handling + +* Fri Aug 16 2019 Andreas Gerstmayr 0.0.7-1 +- converted into a Grafana app plugin, renamed to grafana-pcp +- redis: support for instance domains, labels, autocompletion, automatic rate conversation +- live and bpftrace: initial commit of datasources + +* Tue Jun 11 2019 Mark Goodwin 0.0.6-1 +- renamed package to grafana-pcp-redis, updated README, etc + +* Wed Jun 05 2019 Mark Goodwin 0.0.5-1 +- renamed package to grafana-pcp-datasource, README, etc + +* Fri May 17 2019 Mark Goodwin 0.0.4-1 +- add suggested pmproxy URL in config html +- updated instructions and README.md now that grafana is in Fedora + +* Fri Apr 12 2019 Mark Goodwin 0.0.3-1 +- require grafana v6.1.3 or later +- install directory is now below /var/lib/grafana/plugins + +* Wed Mar 20 2019 Mark Goodwin 0.0.2-1 +- initial version