pmrep: support archive/zabbix filtering
"Marko Myllynen" <[email protected]>
| Newsgroups | gmane.comp.sysutils.pcp |
|---|---|
| Organization | Red Hat |
| Message-ID | <[email protected]> |
Hi,
After rereading https://bugzilla.redhat.com/show_bug.cgi?id=1345821 I
think it's best to take the following approach with pmrep(1), archives,
and Zabbix:
1) When no instances specified, log all the current instances (so no
change to the current situation)
2) Support instance filtering also with archives and Zabbix - the only
minor difference wrt archives with pmlogger(1) is that pmrep(1) does
not log anything about the other instances so no information leak when
recording e.g. process data
There's not a huge amount of app-level code to enable archive writing
with pmfg so perhaps API extension is not hugely needed. Reviewed csv,
stdout outputs while at, still good, slipped in few tine cleanups.
Patch below; man page tweaked as well, passing existing and new QA.
---
qa/1072 | 17 +++++++-
qa/1072.out | 40 ++++++++++++++++++
src/pmrep/pmrep.1 | 8 +---
src/pmrep/pmrep.py | 120 +++++++++++++++++++++++++++++++----------------------
4 files changed, 128 insertions(+), 57 deletions(-)
diff --git a/qa/1072 b/qa/1072
index 8da371d..741109c 100755
--- a/qa/1072
+++ b/qa/1072
@@ -81,7 +81,22 @@ echo "Diff of outputs start (only path / minor timestamp diff expected):"
diff -u $tmp.a1-out $tmp.a1-out-new | grep -v +++ | grep -v -- ---
diff -u $tmp.a2-out $tmp.a2-out-new | grep -v +++ | grep -v -- ---
echo "Diff of outputs end"
-rm -f $tmp.a1-out* $tmp.a2-out* $tmp.new_archive
+rm -f $tmp.a1-out* $tmp.a2-out* $tmp.new_archive*
+
+echo "== pmrep archive creation with instance filtering"
+echo "Filtering output from the original archive to files"
+pmrep -u -s 5 -p -P 4 -w 12 -x -z $log2 -i "5 minute" kernel.all.load | _path_filter | _archive_filter >$tmp.a2-out
+pmrep -u -s 5 -o archive -F $tmp.new_archive2 -z $log2 -i "5 minute" kernel.all.load | _pid_filter
+echo "Displaying archive contents"
+pmrep -u -s 5 -p -P 4 -w 12 -x -z -a $tmp.new_archive2 kernel.all.load | \
+ _path_filter | _archive_filter | _pid_filter
+echo "Writing output from the created archive to files"
+pmrep -u -s 5 -p -P 4 -w 12 -x -z -a $tmp.new_archive2 kernel.all.load | \
+ _path_filter | _archive_filter | _pid_filter >$tmp.a2-out-new
+echo "Diff of outputs start (only path / minor timestamp diff expected):"
+diff -u $tmp.a2-out $tmp.a2-out-new | grep -v +++ | grep -v -- ---
+echo "Diff of outputs end"
+rm -f $tmp.a1-out* $tmp.a2-out* $tmp.new_archive*
# success, all done
echo "== done"
diff --git a/qa/1072.out b/qa/1072.out
index 635df66..222232f 100644
--- a/qa/1072.out
+++ b/qa/1072.out
@@ -28,4 +28,44 @@ Diff of outputs start (only path / minor timestamp diff expected):
samples: 5 (requested)
interval: N/A
Diff of outputs end
+== pmrep archive creation with instance filtering
+Filtering output from the original archive to files
+Recording 1 metrics to /tmp/1072.new_archive2:
+5 samples(s) with 1.0 sec interval ~ 4 sec duration.
+Displaying archive contents
+
+ archive: temp-archive
+ host: billing02
+ timezone: ...-11
+ start: Sat Jul 6 00:28:01 2013
+ end: Sat Jul 6 00:32:01 2013
+ metrics: 1
+ samples: 5 (requested)
+ interval: N/A
+ duration: N/A
+
+ k.a.load
+ 5 minute
+
+00:28:01 0.4200
+00:29:01 0.4500
+00:30:01 0.4700
+00:31:01 0.4700
+00:32:01 0.5300
+Writing output from the created archive to files
+Diff of outputs start (only path / minor timestamp diff expected):
+@@ -1,9 +1,9 @@
+
+- archive: QAPATH/archives/20130706
++ archive: temp-archive
+ host: billing02
+ timezone: ...-11
+- start: Sat Jul 6 00:27:01 2013
+- end: Sun Jul 7 00:15:43 2013
++ start: Sat Jul 6 00:28:01 2013
++ end: Sat Jul 6 00:32:01 2013
+ metrics: 1
+ samples: 5 (requested)
+ interval: N/A
+Diff of outputs end
== done
diff --git a/src/pmrep/pmrep.1 b/src/pmrep/pmrep.1
index 11b67dc..7ccfb08 100644
--- a/src/pmrep/pmrep.1
+++ b/src/pmrep/pmrep.1
@@ -166,12 +166,8 @@ and
see below).
The optional
.I instance
-definition restricts
-.I csv
-and
-.I stdout
-reporting to the specified instances (so non-matching instances
-will be filtered).
+definition restricts reporting to the specified instances
+(so non-matching instances will be filtered).
An optional
.I unit/scale
is applicable for dimension-compatible, non-string, and (currently)
diff --git a/src/pmrep/pmrep.py b/src/pmrep/pmrep.py
index 4468b04..05110c0 100755
--- a/src/pmrep/pmrep.py
+++ b/src/pmrep/pmrep.py
@@ -460,8 +460,8 @@ class PMReporter(object):
insts = [None]
if spec.count(",") < 2:
return spec + ",,", insts
- # User may supply quoted or unquoted instance specification
- # Conf file preservers outer quotes, command line does not
+ # User may supply quoted or unquoted instance specification.
+ # Conf file preserves outer quotes, command line does not.
# We need to detect which is the case here. What a mess.
quoted = 0
s = spec.split(",")[2]
@@ -478,8 +478,6 @@ class PMReporter(object):
spec = spec.replace(inststr, "")
else:
insts = [s]
- if spec.count(",") < 2:
- spec += ",,"
return spec, insts
def parse_metric_info(self, metrics, key, value):
@@ -1184,6 +1182,23 @@ class PMReporter(object):
else:
self.writer.write("...\n(Ctrl-C to stop)\n")
+ def add_archive_value(self, metric, i, inst, name, value):
+ """ Add a value to archive """
+ if inst != PM_IN_NULL and inst not in self.recorded[metric]:
+ self.recorded[metric].append(inst)
+ try:
+ self.pmi.pmiAddInstance(self.descs[i].contents.indom, name, inst)
+ except pmi.pmiErr as error:
+ if error.args[0] == PMI_ERR_DUPINSTNAME:
+ pass
+ if self.descs[i].contents.type == PM_TYPE_STRING:
+ self.pmi.pmiPutValue(metric, name, value)
+ elif self.descs[i].contents.type == PM_TYPE_FLOAT or \
+ self.descs[i].contents.type == PM_TYPE_DOUBLE:
+ self.pmi.pmiPutValue(metric, name, "%f" % value)
+ else:
+ self.pmi.pmiPutValue(metric, name, "%d" % value)
+
def write_archive(self, timestamp):
""" Write an archive record """
if timestamp == None:
@@ -1200,49 +1215,43 @@ class PMReporter(object):
self.pmi.pmiSetHostname(self.context.pmGetArchiveLabel().hostname)
self.pmi.pmiSetTimezone(self.context.pmGetArchiveLabel().tz)
for i, metric in enumerate(self.metrics):
+ self.recorded[metric] = []
self.pmi.pmiAddMetric(metric,
self.pmids[i],
self.descs[i].contents.type,
self.descs[i].contents.indom,
self.descs[i].contents.sem,
self.descs[i].contents.units)
- ins = 0 if self.insts[i][0][0] == PM_IN_NULL else len(self.insts[i][0])
- for j in range(ins):
- if metric not in self.recorded:
- self.recorded[metric] = []
- self.recorded[metric].append(self.insts[i][0][j])
- try:
- self.pmi.pmiAddInstance(self.descs[i].contents.indom, self.insts[i][1][j], self.insts[i][0][j])
- except pmi.pmiErr as error:
- if error.args[0] == PMI_ERR_DUPINSTNAME:
- continue
# Add current values
data = 0
for i, metric in enumerate(self.metrics):
- try:
+ if not self.instances and \
+ (not self.metrics[metric][1] or \
+ self.metrics[metric][1] is None or \
+ self.metrics[metric][1][0] is None):
+ # Unfiltered
for inst, name, val in self.metrics[metric][5]():
try:
- value = val()
- if inst != PM_IN_NULL and inst not in self.recorded[metric]:
- self.recorded[metric].append(inst)
- try:
- self.pmi.pmiAddInstance(self.descs[i].contents.indom, name, inst)
- except pmi.pmiErr as error:
- if error.args[0] == PMI_ERR_DUPINSTNAME:
- pass
- if self.descs[i].contents.type == PM_TYPE_STRING:
- self.pmi.pmiPutValue(metric, name, value)
- elif self.descs[i].contents.type == PM_TYPE_FLOAT or \
- self.descs[i].contents.type == PM_TYPE_DOUBLE:
- self.pmi.pmiPutValue(metric, name, "%f" % value)
- else:
- self.pmi.pmiPutValue(metric, name, "%d" % value)
+ self.add_archive_value(metric, i, inst, name, val())
data = 1
- except Exception as e:
+ except:
pass
- except:
- pass
+ continue
+ for j in range(len(self.insts[i][0])):
+ # Filtered
+ found = 0
+ try:
+ for inst, name, val in self.metrics[metric][5]():
+ if inst == PM_IN_NULL or inst == self.insts[i][0][j]:
+ found = 1
+ break
+ if not found:
+ continue
+ self.add_archive_value(metric, i, inst, name, val())
+ data = 1
+ except:
+ pass
# Flush
if data:
@@ -1311,23 +1320,14 @@ class PMReporter(object):
for j in range(len(self.insts[i][0])):
k += 1
- found = 0
+ value = NO_VAL
try:
for inst, name, val in self.metrics[metric][5]():
if inst == PM_IN_NULL or inst == self.insts[i][0][j]:
- found = 1
+ value = val()
break
except:
pass
- if not found:
- value = NO_VAL
- else:
- try:
- value = val()
- if type(value) is list:
- value = value[0]
- except:
- value = NO_VAL
# Make sure the value fits
if type(value) is int or type(value) is long:
@@ -1387,18 +1387,38 @@ class PMReporter(object):
if self.zabbix_prevsend == None:
self.zabbix_prevsend = ts
for i, metric in enumerate(self.metrics):
- try:
+ if not self.instances and \
+ (not self.metrics[metric][1] or \
+ self.metrics[metric][1] is None or \
+ self.metrics[metric][1][0] is None):
+ # Unfiltered
for inst, name, val in self.metrics[metric][5]():
- key = ZBXPRFX + metric
- if name:
- key += "[" + name + "]"
try:
+ key = ZBXPRFX + metric
+ if name:
+ key += "[" + name + "]"
value = str(val())
self.zabbix_metrics.append(ZabbixMetric(self.zabbix_host, key, value, ts))
except:
pass
- except:
- pass
+ continue
+ for j in range(len(self.insts[i][0])):
+ # Filtered
+ found = 0
+ try:
+ for inst, name, val in self.metrics[metric][5]():
+ if inst == PM_IN_NULL or inst == self.insts[i][0][j]:
+ found = 1
+ break
+ if not found:
+ continue
+ key = ZBXPRFX + metric
+ if name:
+ key += "[" + name + "]"
+ value = str(val())
+ self.zabbix_metrics.append(ZabbixMetric(self.zabbix_host, key, value, ts))
+ except:
+ pass
# Send when needed
if self.context.type == PM_CONTEXT_ARCHIVE:
Thanks,
--
Marko Myllynen
-=-=-=-=-=-=-=-=-=-=-=-
pcp mailing list
[email protected]
https://groups.io/g/pcp/messages
-=-=-
Groups.io Links:
You receive all messages sent to this group.
View/Reply Online (#14632): https://groups.io/g/pcp/message/14632
View All Messages In Topic (1): https://groups.io/g/pcp/topic/2993028
Mute This Topic: https://groups.io/mt/2993028?uid=174580
New Topic: https://groups.io/g/pcp/post
Change Your Subscription: https://groups.io/g/pcp/editsub?uid=174580
Group Home: https://groups.io/g/pcp
Contact Group Owner: [email protected]
Terms of Service: https://groups.io/static/tos
Unsubscribe: https://groups.io/g/pcp/leave/354243/563757577/xyzzy
-=-=-=-=-=-=-=-=-=-=-=-