malmond / rpms / unbound

Forked from rpms/unbound 3 years ago
Clone
8ba890
#!/bin/sh
8ba890
#
8ba890
# plugin for munin to monitor usage of unbound servers.
8ba890
#
8ba890
# (C) 2008 W.C.A. Wijngaards.  BSD Licensed.
8ba890
#
8ba890
# To install; enable statistics and unbound-control in unbound.conf
8ba890
#	server:		extended-statistics: yes
8ba890
#			statistics-cumulative: no
8ba890
#			statistics-interval: 0
8ba890
#	remote-control:	control-enable: yes
8ba890
# Run the command unbound-control-setup to generate the key files.
8ba890
#
8ba890
# Environment variables for this script
8ba890
#	statefile	- where to put temporary statefile.
8ba890
#	unbound_conf	- where the unbound.conf file is located.
8ba890
#	unbound_control	- where to find unbound-control executable.
8ba890
#	spoof_warn	- what level to warn about spoofing
8ba890
#	spoof_crit	- what level to crit about spoofing
8ba890
#
8ba890
# You can set them in your munin/plugin-conf.d/plugins.conf file
8ba890
# with:
8ba890
# [unbound*]
8ba890
# user root
8ba890
# env.statefile /usr/local/var/munin/plugin-state/unbound-state
8ba890
# env.unbound_conf /usr/local/etc/unbound/unbound.conf
8ba890
# env.unbound_control /usr/local/sbin/unbound-control
8ba890
# env.spoof_warn 1000
8ba890
# env.spoof_crit 100000
8ba890
#
8ba890
# This plugin can create different graphs depending on what name
8ba890
# you link it as (with ln -s) into the plugins directory
8ba890
# You can link it multiple times.
8ba890
# If you are only a casual user, the _hits and _by_type are most interesting,
8ba890
# possibly followed by _by_rcode.
8ba890
#
8ba890
#	unbound_munin_hits	- base volume, cache hits, unwanted traffic
8ba890
#	unbound_munin_queue	- to monitor the internal requestlist
8ba890
#	unbound_munin_memory	- memory usage
8ba890
#	unbound_munin_by_type	- incoming queries by type
8ba890
#	unbound_munin_by_class	- incoming queries by class
8ba890
#	unbound_munin_by_opcode	- incoming queries by opcode
8ba890
#	unbound_munin_by_rcode	- answers by rcode, validation status
8ba890
#	unbound_munin_by_flags	- incoming queries by flags
8ba890
#	unbound_munin_histogram	- histogram of query resolving times
8ba890
#
8ba890
# Magic markers - optional - used by installation scripts and
8ba890
# munin-config:
8ba890
#
8ba890
#%# family=contrib
8ba890
#%# capabilities=autoconf suggest
8ba890
8ba890
# POD documentation
8ba890
: <<=cut
8ba890
=head1 NAME
8ba890
8ba890
unbound_munin_ - Munin plugin to monitor the Unbound DNS resolver.
8ba890
8ba890
=head1 APPLICABLE SYSTEMS
8ba890
8ba890
System with unbound daemon.
8ba890
8ba890
=head1 CONFIGURATION
8ba890
8ba890
  [unbound*]
8ba890
  user root
8ba890
  env.statefile /var/lib/munin/plugin-state/unbound-state
8ba890
  env.unbound_conf /etc/unbound/unbound.conf
8ba890
  env.unbound_control /usr/sbin/unbound-control
8ba890
  env.spoof_warn 1000
8ba890
  env.spoof_crit 100000
8ba890
8ba890
Use the .env settings to override the defaults.
8ba890
8ba890
=head1 USAGE
8ba890
8ba890
Can be used to present different graphs. Use ln -s for that name in
8ba890
the plugins directory to enable the graph.
8ba890
unbound_munin_hits	- base volume, cache hits, unwanted traffic
8ba890
unbound_munin_queue	- to monitor the internal requestlist
8ba890
unbound_munin_memory	- memory usage
8ba890
unbound_munin_by_type	- incoming queries by type
8ba890
unbound_munin_by_class	- incoming queries by class
8ba890
unbound_munin_by_opcode	- incoming queries by opcode
8ba890
unbound_munin_by_rcode	- answers by rcode, validation status
8ba890
unbound_munin_by_flags	- incoming queries by flags
8ba890
unbound_munin_histogram - histogram of query resolving times
8ba890
8ba890
=head1 AUTHOR
8ba890
8ba890
Copyright 2008 W.C.A. Wijngaards
8ba890
8ba890
=head1 LICENSE
8ba890
8ba890
BSD
8ba890
8ba890
=cut
8ba890
8ba890
state=${statefile:-/var/lib/munin/plugin-state/unbound-state}
8ba890
conf=${unbound_conf:-/etc/unbound/unbound.conf}
8ba890
ctrl=${unbound_control:-/usr/sbin/unbound-control}
8ba890
warn=${spoof_warn:-1000}
8ba890
crit=${spoof_crit:-100000}
8ba890
lock=$state.lock
8ba890
8ba890
# number of seconds between polling attempts.
8ba890
# makes the statefile hang around for at least this many seconds,
8ba890
# so that multiple links of this script can share the results.
8ba890
lee=55
8ba890
8ba890
# to keep things within 19 characters
8ba890
ABBREV="-e s/total/t/ -e s/thread/t/ -e s/num/n/ -e s/query/q/ -e s/answer/a/ -e s/unwanted/u/ -e s/requestlist/ql/ -e s/type/t/ -e s/class/c/ -e s/opcode/o/ -e s/rcode/r/ -e s/edns/e/ -e s/mem/m/ -e s/cache/c/ -e s/mod/m/"
8ba890
8ba890
# get value from $1 into return variable $value
8ba890
get_value ( ) {
8ba890
	value="`grep '^'$1'=' $state | sed -e 's/^.*=//'`"
8ba890
	if test "$value"x = ""x; then
8ba890
		value="0"
8ba890
	fi
8ba890
}
8ba890
8ba890
# download the state from the unbound server.
8ba890
get_state ( ) {
8ba890
	# obtain lock for fetching the state
8ba890
	# because there is a race condition in fetching and writing to file
8ba890
8ba890
	# see if the lock is stale, if so, take it 
8ba890
	if test -f $lock ; then
8ba890
		pid="`cat $lock 2>&1`"
8ba890
		kill -0 "$pid" >/dev/null 2>&1
8ba890
		if test $? -ne 0 -a "$pid" != $$ ; then
8ba890
			echo $$ >$lock
8ba890
		fi
8ba890
	fi
8ba890
8ba890
	i=0
8ba890
	while test ! -f $lock || test "`cat $lock 2>&1`" != $$; do
8ba890
		while test -f $lock; do
8ba890
			# wait
8ba890
			i=`expr $i + 1`
8ba890
			if test $i -gt 1000; then
8ba890
				sleep 1;
8ba890
			fi
8ba890
			if test $i -gt 1500; then
8ba890
				echo "error locking $lock" "=" `cat $lock`
8ba890
				rm -f $lock
8ba890
				exit 1
8ba890
			fi
8ba890
		done
8ba890
		# try to get it
8ba890
		echo $$ >$lock
8ba890
	done
8ba890
	# do not refetch if the file exists and only LEE seconds old
8ba890
	if test -f $state; then
8ba890
		now=`date +%s`
8ba890
		get_value "time.now"
8ba890
		value="`echo $value | sed -e 's/\..*$//'`"
8ba890
		if test $now -lt `expr $value + $lee`; then
8ba890
			rm -f $lock
8ba890
			return
8ba890
		fi
8ba890
	fi
8ba890
	$ctrl -c $conf stats > $state
8ba890
	if test $? -ne 0; then
8ba890
		echo "error retrieving data from unbound server"
8ba890
		rm -f $lock
8ba890
		exit 1
8ba890
	fi
8ba890
	rm -f $lock
8ba890
}
8ba890
8ba890
if test "$1" = "autoconf" ; then
8ba890
	if test ! -f $conf; then
8ba890
		echo no "($conf does not exist)"
8ba890
		exit 1
8ba890
	fi
8ba890
	if test ! -d `dirname $state`; then
8ba890
		echo no "($state directory does not exist)"
8ba890
		exit 1
8ba890
	fi
8ba890
	echo yes
8ba890
	exit 0
8ba890
fi
8ba890
8ba890
if test "$1" = "suggest" ; then
8ba890
	echo "hits"
8ba890
	echo "queue"
8ba890
	echo "memory"
8ba890
	echo "by_type"
8ba890
	echo "by_class"
8ba890
	echo "by_opcode"
8ba890
	echo "by_rcode"
8ba890
	echo "by_flags"
8ba890
	echo "histogram"
8ba890
	exit 0
8ba890
fi
8ba890
8ba890
# determine my type, by name
8ba890
id=`echo $0 | sed -e 's/^.*unbound_munin_//'`
8ba890
if test "$id"x = ""x; then
8ba890
	# some default to keep people sane.
8ba890
	id="hits"
8ba890
fi
8ba890
8ba890
# if $1 exists in statefile, config is echoed with label $2
8ba890
exist_config ( ) {
8ba890
	mn=`echo $1 | sed $ABBREV | tr . _`
8ba890
	if grep '^'$1'=' $state >/dev/null 2>&1; then
8ba890
		echo "$mn.label $2"
8ba890
		echo "$mn.min 0"
8ba890
	fi
8ba890
}
8ba890
8ba890
# print label and min 0 for a name $1 in unbound format
8ba890
p_config ( ) {
8ba890
	mn=`echo $1 | sed $ABBREV | tr . _`
8ba890
	echo $mn.label "$2"
8ba890
	echo $mn.min 0
8ba890
}
8ba890
8ba890
if test "$1" = "config" ; then
8ba890
	if test ! -f $state; then
8ba890
		get_state
8ba890
	fi
8ba890
	case $id in
8ba890
	hits)
8ba890
		echo "graph_title Unbound DNS traffic and cache hits"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel queries / second"
8ba890
		echo "graph_category DNS"
8ba890
		for x in thread0.num.queries thread1.num.queries \
8ba890
		thread2.num.queries thread3.num.queries thread4.num.queries \
8ba890
		thread5.num.queries thread6.num.queries thread7.num.queries; do
8ba890
			exist_config $x "queries handled by `basename $x .num.queries`"
8ba890
		done
8ba890
		p_config "total.num.queries" "total queries from clients"
8ba890
		p_config "total.num.cachehits" "cache hits"
8ba890
		p_config "total.num.prefetch" "cache prefetch"
8ba890
		p_config "num.query.tcp" "TCP queries"
8ba890
		p_config "num.query.ipv6" "IPv6 queries"
8ba890
		p_config "unwanted.queries" "queries that failed acl"
8ba890
		p_config "unwanted.replies" "unwanted or unsolicited replies"
8ba890
		echo "u_replies.warning $warn"
8ba890
		echo "u_replies.critical $crit"
8ba890
		echo "graph_info DNS queries to the recursive resolver. The unwanted replies could be innocent duplicate packets, late replies, or spoof threats."
8ba890
		;;
8ba890
	queue)
8ba890
		echo "graph_title Unbound requestlist size"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel number of queries"
8ba890
		echo "graph_category DNS"
8ba890
		p_config "total.requestlist.avg" "Average size of queue on insert"
8ba890
		p_config "total.requestlist.max" "Max size of queue (in 5 min)"
8ba890
		p_config "total.requestlist.overwritten" "Number of queries replaced by new ones"
8ba890
		p_config "total.requestlist.exceeded" "Number of queries dropped due to lack of space"
8ba890
		echo "graph_info The queries that did not hit the cache and need recursion service take up space in the requestlist. If there are too many queries, first queries get overwritten, and at last resort dropped."
8ba890
		;;
8ba890
	memory)
8ba890
		echo "graph_title Unbound memory usage"
8ba890
		echo "graph_args --base 1024 -l 0"
8ba890
		echo "graph_vlabel memory used in bytes"
8ba890
		echo "graph_category DNS"
8ba890
		p_config "mem.total.sbrk" "Total memory"
8ba890
		p_config "mem.cache.rrset" "RRset cache memory"
8ba890
		p_config "mem.cache.message" "Message cache memory"
8ba890
		p_config "mem.mod.iterator" "Iterator module memory"
8ba890
		p_config "mem.mod.validator" "Validator module and key cache memory"
8ba890
		echo "graph_info The memory used by unbound."
8ba890
		;;
8ba890
	by_type)
8ba890
		echo "graph_title Unbound DNS queries by type"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel queries / second"
8ba890
		echo "graph_category DNS"
8ba890
		for x in `grep "^num.query.type" $state`; do
8ba890
			nm=`echo $x | sed -e 's/=.*$//'`
8ba890
			tp=`echo $nm | sed -e s/num.query.type.//`
8ba890
			p_config "$nm" "$tp"
8ba890
		done
8ba890
		echo "graph_info queries by DNS RR type queried for"
8ba890
		;;
8ba890
	by_class)
8ba890
		echo "graph_title Unbound DNS queries by class"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel queries / second"
8ba890
		echo "graph_category DNS"
8ba890
		for x in `grep "^num.query.class" $state`; do
8ba890
			nm=`echo $x | sed -e 's/=.*$//'`
8ba890
			tp=`echo $nm | sed -e s/num.query.class.//`
8ba890
			p_config "$nm" "$tp"
8ba890
		done
8ba890
		echo "graph_info queries by DNS RR class queried for."
8ba890
		;;
8ba890
	by_opcode)
8ba890
		echo "graph_title Unbound DNS queries by opcode"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel queries / second"
8ba890
		echo "graph_category DNS"
8ba890
		for x in `grep "^num.query.opcode" $state`; do
8ba890
			nm=`echo $x | sed -e 's/=.*$//'`
8ba890
			tp=`echo $nm | sed -e s/num.query.opcode.//`
8ba890
			p_config "$nm" "$tp"
8ba890
		done
8ba890
		echo "graph_info queries by opcode in the query packet."
8ba890
		;;
8ba890
	by_rcode)
8ba890
		echo "graph_title Unbound DNS answers by return code"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel answer packets / second"
8ba890
		echo "graph_category DNS"
8ba890
		for x in `grep "^num.answer.rcode" $state`; do
8ba890
			nm=`echo $x | sed -e 's/=.*$//'`
8ba890
			tp=`echo $nm | sed -e s/num.answer.rcode.//`
8ba890
			p_config "$nm" "$tp"
8ba890
		done
8ba890
		p_config "num.answer.secure" "answer secure"
8ba890
		p_config "num.answer.bogus" "answer bogus"
8ba890
		p_config "num.rrset.bogus" "num rrsets marked bogus"
8ba890
		echo "graph_info answers sorted by return value. rrsets bogus is the number of rrsets marked bogus per second by the validator"
8ba890
		;;
8ba890
	by_flags)
8ba890
		echo "graph_title Unbound DNS incoming queries by flags"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel queries / second"
8ba890
		echo "graph_category DNS"
8ba890
		p_config "num.query.flags.QR" "QR (query reply) flag"
8ba890
		p_config "num.query.flags.AA" "AA (auth answer) flag"
8ba890
		p_config "num.query.flags.TC" "TC (truncated) flag"
8ba890
		p_config "num.query.flags.RD" "RD (recursion desired) flag"
8ba890
		p_config "num.query.flags.RA" "RA (rec avail) flag"
8ba890
		p_config "num.query.flags.Z" "Z (zero) flag"
8ba890
		p_config "num.query.flags.AD" "AD (auth data) flag"
8ba890
		p_config "num.query.flags.CD" "CD (check disabled) flag"
8ba890
		p_config "num.query.edns.present" "EDNS OPT present"
8ba890
		p_config "num.query.edns.DO" "DO (DNSSEC OK) flag"
8ba890
		echo "graph_info This graphs plots the flags inside incoming queries. For example, if QR, AA, TC, RA, Z flags are set, the query can be rejected. RD, AD, CD and DO are legitimately set by some software."
8ba890
		;;
8ba890
	histogram)
8ba890
		echo "graph_title Unbound DNS histogram of reply time"
8ba890
		echo "graph_args --base 1000 -l 0"
8ba890
		echo "graph_vlabel queries / second"
8ba890
		echo "graph_category DNS"
8ba890
		echo hcache.label "cache hits"
8ba890
		echo hcache.min 0
8ba890
		echo hcache.draw AREA
8ba890
		echo hcache.colour 999999
8ba890
		echo h64ms.label "0 msec - 66 msec"
8ba890
		echo h64ms.min 0
8ba890
		echo h64ms.draw STACK
8ba890
		echo h64ms.colour 0000FF
8ba890
		echo h128ms.label "66 msec - 131 msec"
8ba890
		echo h128ms.min 0
8ba890
		echo h128ms.colour 1F00DF
8ba890
		echo h128ms.draw STACK
8ba890
		echo h256ms.label "131 msec - 262 msec"
8ba890
		echo h256ms.min 0
8ba890
		echo h256ms.draw STACK
8ba890
		echo h256ms.colour 3F00BF
8ba890
		echo h512ms.label "262 msec - 524 msec"
8ba890
		echo h512ms.min 0
8ba890
		echo h512ms.draw STACK
8ba890
		echo h512ms.colour 5F009F
8ba890
		echo h1s.label "524 msec - 1 sec"
8ba890
		echo h1s.min 0
8ba890
		echo h1s.draw STACK
8ba890
		echo h1s.colour 7F007F
8ba890
		echo h2s.label "1 sec - 2 sec"
8ba890
		echo h2s.min 0
8ba890
		echo h2s.draw STACK
8ba890
		echo h2s.colour 9F005F
8ba890
		echo h4s.label "2 sec - 4 sec"
8ba890
		echo h4s.min 0
8ba890
		echo h4s.draw STACK
8ba890
		echo h4s.colour BF003F
8ba890
		echo h8s.label "4 sec - 8 sec"
8ba890
		echo h8s.min 0
8ba890
		echo h8s.draw STACK
8ba890
		echo h8s.colour DF001F
8ba890
		echo h16s.label "8 sec - ..."
8ba890
		echo h16s.min 0
8ba890
		echo h16s.draw STACK
8ba890
		echo h16s.colour FF0000
8ba890
		echo "graph_info Histogram of the reply times for queries."
8ba890
		;;
8ba890
	esac
8ba890
8ba890
	exit 0
8ba890
fi
8ba890
8ba890
# do the stats itself
8ba890
get_state
8ba890
8ba890
# get the time elapsed
8ba890
get_value "time.elapsed"
8ba890
if test $value = 0 || test $value = "0.000000"; then
8ba890
	echo "error: time elapsed 0 or could not retrieve data"
8ba890
	exit 1
8ba890
fi
8ba890
elapsed="$value"
8ba890
8ba890
# print value for $1 / elapsed
8ba890
print_qps ( ) {
8ba890
	mn=`echo $1 | sed $ABBREV | tr . _`
8ba890
	get_value $1
8ba890
	echo "$mn.value" `echo scale=6';' $value / $elapsed | bc `
8ba890
}
8ba890
8ba890
# print qps if line already found in $2
8ba890
print_qps_line ( ) {
8ba890
	mn=`echo $1 | sed $ABBREV | tr . _`
8ba890
	value="`echo $2 | sed -e 's/^.*=//'`"
8ba890
	echo "$mn.value" `echo scale=6';' $value / $elapsed | bc `
8ba890
}
8ba890
8ba890
# print value for $1
8ba890
print_value ( ) {
8ba890
	mn=`echo $1 | sed $ABBREV | tr . _`
8ba890
	get_value $1
8ba890
	echo "$mn.value" $value
8ba890
}
8ba890
8ba890
case $id in
8ba890
hits)
8ba890
	for x in thread0.num.queries thread1.num.queries thread2.num.queries \
8ba890
		thread3.num.queries thread4.num.queries thread5.num.queries \
8ba890
		thread6.num.queries thread7.num.queries total.num.queries \
8ba890
		total.num.cachehits total.num.prefetch num.query.tcp \
8ba890
		num.query.ipv6 unwanted.queries unwanted.replies; do
8ba890
		if grep "^"$x"=" $state >/dev/null 2>&1; then
8ba890
			print_qps $x
8ba890
		fi
8ba890
	done
8ba890
	;;
8ba890
queue)
8ba890
	for x in total.requestlist.avg total.requestlist.max \
8ba890
		total.requestlist.overwritten total.requestlist.exceeded; do
8ba890
		print_value $x
8ba890
	done
8ba890
	;;
8ba890
memory)
8ba890
	mn=`echo mem.total.sbrk | sed $ABBREV | tr . _`
8ba890
	get_value 'mem.total.sbrk'
8ba890
	if test $value -eq 0; then
8ba890
		chk=`echo $ctrl | sed -e 's/-control$/-checkconf/'`
8ba890
		pidf=`$chk -o pidfile $conf 2>&1`
8ba890
		pid=`cat $pidf 2>&1`
8ba890
		value=`ps -p "$pid" -o rss= 2>&1`
8ba890
		if test "`expr $value + 1 - 1 2>&1`" -eq "$value" 2>&1; then
8ba890
			value=`expr $value \* 1024` 
8ba890
		else
8ba890
			value=0
8ba890
		fi
8ba890
	fi
8ba890
	echo "$mn.value" $value
8ba890
	for x in mem.cache.rrset mem.cache.message \
8ba890
		mem.mod.iterator mem.mod.validator; do
8ba890
		print_value $x
8ba890
	done
8ba890
	;;
8ba890
by_type)
8ba890
	for x in `grep "^num.query.type" $state`; do
8ba890
		nm=`echo $x | sed -e 's/=.*$//'`
8ba890
		print_qps_line $nm $x
8ba890
	done
8ba890
	;;
8ba890
by_class)
8ba890
	for x in `grep "^num.query.class" $state`; do
8ba890
		nm=`echo $x | sed -e 's/=.*$//'`
8ba890
		print_qps_line $nm $x
8ba890
	done
8ba890
	;;
8ba890
by_opcode)
8ba890
	for x in `grep "^num.query.opcode" $state`; do
8ba890
		nm=`echo $x | sed -e 's/=.*$//'`
8ba890
		print_qps_line $nm $x
8ba890
	done
8ba890
	;;
8ba890
by_rcode)
8ba890
	for x in `grep "^num.answer.rcode" $state`; do
8ba890
		nm=`echo $x | sed -e 's/=.*$//'`
8ba890
		print_qps_line $nm $x
8ba890
	done
8ba890
	print_qps "num.answer.secure"
8ba890
	print_qps "num.answer.bogus"
8ba890
	print_qps "num.rrset.bogus"
8ba890
	;;
8ba890
by_flags)
8ba890
	for x in num.query.flags.QR num.query.flags.AA num.query.flags.TC num.query.flags.RD num.query.flags.RA num.query.flags.Z num.query.flags.AD num.query.flags.CD num.query.edns.present num.query.edns.DO; do
8ba890
		print_qps $x
8ba890
	done
8ba890
	;;
8ba890
histogram)
8ba890
	get_value total.num.cachehits
8ba890
	echo hcache.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	r=0
8ba890
	for x in histogram.000000.000000.to.000000.000001 \
8ba890
		histogram.000000.000001.to.000000.000002 \
8ba890
		histogram.000000.000002.to.000000.000004 \
8ba890
		histogram.000000.000004.to.000000.000008 \
8ba890
		histogram.000000.000008.to.000000.000016 \
8ba890
		histogram.000000.000016.to.000000.000032 \
8ba890
		histogram.000000.000032.to.000000.000064 \
8ba890
		histogram.000000.000064.to.000000.000128 \
8ba890
		histogram.000000.000128.to.000000.000256 \
8ba890
		histogram.000000.000256.to.000000.000512 \
8ba890
		histogram.000000.000512.to.000000.001024 \
8ba890
		histogram.000000.001024.to.000000.002048 \
8ba890
		histogram.000000.002048.to.000000.004096 \
8ba890
		histogram.000000.004096.to.000000.008192 \
8ba890
		histogram.000000.008192.to.000000.016384 \
8ba890
		histogram.000000.016384.to.000000.032768 \
8ba890
		histogram.000000.032768.to.000000.065536; do
8ba890
		get_value $x
8ba890
		r=`expr $r + $value`
8ba890
	done
8ba890
	echo h64ms.value `echo scale=6';' $r / $elapsed | bc `
8ba890
	get_value histogram.000000.065536.to.000000.131072
8ba890
	echo h128ms.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	get_value histogram.000000.131072.to.000000.262144
8ba890
	echo h256ms.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	get_value histogram.000000.262144.to.000000.524288
8ba890
	echo h512ms.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	get_value histogram.000000.524288.to.000001.000000
8ba890
	echo h1s.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	get_value histogram.000001.000000.to.000002.000000
8ba890
	echo h2s.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	get_value histogram.000002.000000.to.000004.000000
8ba890
	echo h4s.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	get_value histogram.000004.000000.to.000008.000000
8ba890
	echo h8s.value `echo scale=6';' $value / $elapsed | bc `
8ba890
	r=0
8ba890
	for x in histogram.000008.000000.to.000016.000000 \
8ba890
		histogram.000016.000000.to.000032.000000 \
8ba890
		histogram.000032.000000.to.000064.000000 \
8ba890
		histogram.000064.000000.to.000128.000000 \
8ba890
		histogram.000128.000000.to.000256.000000 \
8ba890
		histogram.000256.000000.to.000512.000000 \
8ba890
		histogram.000512.000000.to.001024.000000 \
8ba890
		histogram.001024.000000.to.002048.000000 \
8ba890
		histogram.002048.000000.to.004096.000000 \
8ba890
		histogram.004096.000000.to.008192.000000 \
8ba890
		histogram.008192.000000.to.016384.000000 \
8ba890
		histogram.016384.000000.to.032768.000000 \
8ba890
		histogram.032768.000000.to.065536.000000 \
8ba890
		histogram.065536.000000.to.131072.000000 \
8ba890
		histogram.131072.000000.to.262144.000000 \
8ba890
		histogram.262144.000000.to.524288.000000; do
8ba890
		get_value $x
8ba890
		r=`expr $r + $value`
8ba890
	done
8ba890
	echo h16s.value `echo scale=6';' $r / $elapsed | bc `
8ba890
	;;
8ba890
esac