Re: RFC: metric labels as first-class PCP concept

diff --git a/man/man1/pmcd.1 b/man/man1/pmcd.1
index 595ae71..f9b6d52 100644
--- a/man/man1/pmcd.1
+++ b/man/man1/pmcd.1
@@ -649,14 +649,17 @@ Following this line, the remainder of the configuration file should contain
 lines that allow or disallow operations from particular hosts or groups of
 hosts.
 .PP
-There are two kinds of operations that occur via
+There are three kinds of operations that occur via
 .BR pmcd :
 .TP 15
 .B fetch
 allows retrieval of information from
 .BR pmcd .
-This may be information about a metric (e.g. its description, instance domain
-or help text) or a value for a metric.
+This may be information about a metric (e.g. its description, instance domain,
+labels or help text) or a value for a metric.
+See
+.BR pminfo (1)
+for further information.
 .TP 15
 .B store
 allows
@@ -667,6 +670,16 @@ zero).  Alternatively, it may be a value used by the PMDA to introduce a
 change to some aspect of monitoring of that metric (e.g. server side event
 filtering) \- possibly even only for the active client tool performing the
 store operation, and not others.
+See
+.BR pmstore (1)
+for further information.
+.TP 15
+.B label
+allows creation, modification and deletion of custom labels for performance
+information (metrics, instance domains and individual values).
+See
+.BR pmlabel (1)
+for further information.
 .PP
 Access to
 .B pmcd
@@ -831,7 +844,7 @@ had an IP address of 129.127.112.2, specifying the following two rules would be
 erroneous:
 .PP
 .CS
-allow host clank : fetch, store;
+allow host clank : fetch, store, label;
 disallow host 129.127.112.2 : all except fetch;
 .CE
 .PP
@@ -1155,17 +1168,30 @@ additional environment variables that will be set when
 executes.
 Only settings of the form "PMCD_VARIABLE=value" will be honoured.
 .TP
+.B $PCP_SYSCONF_DIR/labels.d
+global metric labels that will be set when
+.B pmcd
+executes.
+These files are ``JSONB'' formatted, and the merged set can be
+queried via the
+.B pmcd.labels
+metric.
+Global labels are applied universally to all values.
+See
+.BR pmlabel (1)
+for further disucssion about metric value labeling.
+.TP
 .B \&./pmcd.log
 (or
 .B $PCP_LOG_DIR/pmcd/pmcd.log
 when started automatically)
+.br
+All messages and diagnostics are directed here.
 .TP
 .B $PCP_RUN_DIR/pmcd.pid
 contains an ascii decimal representation of the process ID of 
-.B pmcd
-, when it's running.
-.br
-All messages and diagnostics are directed here
+.BR pmcd ,
+when it's running.
 .TP
 .B /etc/pki/nssdb
 default Network Security Services (NSS) certificate database
diff --git a/man/man1/pminfo.1 b/man/man1/pminfo.1
index 4ec8e88..10d1f31 100644
--- a/man/man1/pminfo.1
+++ b/man/man1/pminfo.1
@@ -19,7 +19,7 @@
 \f3pminfo\f1 \- display information about performance metrics
 .SH SYNOPSIS
 \f3pminfo\f1
-[\f3\-dfFLmMtTvxz\f1]
+[\f3\-dfFlLmMtTvxz\f1]
 [\f3\-a\f1 \f2archive\f1]
 [\f3\-b\f1 \f2batchsize\f1]
 [\f3\-c\f1 \f2dmfile\f1]
@@ -174,6 +174,9 @@ More than one
 .B \-K
 option may be used.
 .TP
+.B \-l
+Print any labels associated with the values of each metric.
+.TP
 .B \-m
 Print the PMID in terse mode.
 .TP
@@ -245,6 +248,7 @@ as described in
 .BR pmchart (1),
 .BR pmdumplog (1),
 .BR pmdumptext (1),
+.BR pmlabel (1),
 .BR pmprobe (1),
 .BR pmrep (1),
 .BR pmval (1),
diff --git a/man/man1/pmlabel.1 b/man/man1/pmlabel.1
new file mode 100644
index 0000000..111a0bd
--- /dev/null
+++ b/man/man1/pmlabel.1
@@ -0,0 +1,157 @@
+'\"macro stdmacro
+.\"
+.\" Copyright (c) 2016 Red Hat.  All Rights Reserved.
+.\"
+.\" This program is free software; you can redistribute it and/or modify it
+.\" under the terms of the GNU General Public License as published by the
+.\" Free Software Foundation; either version 2 of the License, or (at your
+.\" option) any later version.
+.\"
+.\" This program is distributed in the hope that it will be useful, but
+.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+.\" or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+.\" for more details.
+.\"
+.TH PMLABEL 1 "PCP" "Performance Co-Pilot"
+.SH NAME
+\f3pmlabel\f1 \- modify performance metric labels
+.\" literals use .B or \f3
+.\" arguments use .I or \f2
+.SH SYNOPSIS
+\f3pmlabel\f1
+[\f3\-h\f1/\f3\-\-host\f1 \f2host\f1]
+\f2tree\f1|\f2metric\f1|\f2pmid\f1|\f2indom\f1 \f2name\f1:\f2value\f1
+.br
+\f3pmlabel\f1
+[\f3\-h\f1/\f3\-\-host\f1 \f2host\f1]
+[\f3\-d\f1/\f3\-\-delete\f1]
+\f2tree\f1|\f2metric\f1|\f2pmid\f1|\f2indom\f1 \f2name\f1\f1
+.SH DESCRIPTION
+In many situations it is useful to associate custom
+metadata (\f2name\f1:\f2value\f1 pairs) with individual
+performance metrics, instance domains, or instances of a
+metric.
+.PP
+.B pmlabel
+changes the current labels for the nominated instances
+of one or more performance metrics, as identified by a
+.BR PMNS (5)
+sub-\c
+.IR tree ,
+an individually named
+.I metric
+or
+.IR pmid
+(dotted notation),
+or
+instance domain identified by
+.I indom
+(dotted notation).
+.PP
+The
+.I metric
+argument is evaluated by
+.BR pmParseMetricSpec (3)
+and as a result can be restricted to individual instances
+of the metric if desired.
+.PP
+Unless directed to another host by the
+.BR \-h / \-\-host
+option,
+.B pmlabel
+will interact with the Performance Metric Collector Daemon (PMCD)
+on the local host.
+.PP
+The output reports the new set of labels associated with each
+affected metric or instance, according to the precedence rules
+outlined below.
+The
+.BR pminfo (1)
+tool also reports labels according to these rules.
+.SH SYNTAX
+Labels are stored and communicated within PCP using JSONB format.
+This format is a restricted form of JSON suitable for indexing
+and other operations.
+In JSONB form, insignificant whitespace is discarded,
+and the order of object keys is not preserved.
+Duplicate object keys are not kept - the later value for a given
+key is the only one stored.
+.PP
+The given \f2name\f1:\f2value\f1 pair(s) will be converted to
+JSONB form and merged with the existing set of labels for the
+requested entity (metric, indom or instance).
+.PP
+The label names are further constrained to the same set of rules
+defined for
+.BR PMNS
+subtree names.
+.PP
+Each component in a label
+.I name
+must begin with an alphabetic character, and be followed by zero
+or more characters drawn from the alphabetics, the digits and the
+underscore (``_'') character.  For alphabetic characters in a
+.IR name ,
+upper and lower case are distinguished.
+.PP
+The
+.I value
+of a label offers significantly more freedom, and may be any valid
+value as defined by the JSON (\c
+.IR http://json.org )
+specification.
+.SH PRECEDENCE
+The complete set of labels associated with any metric value is
+built from several sources and duplicate label names may exist
+at any point in the source hierarchy.
+However, when evaluating the label set (merging labels from the
+different sources) we use the JSONB concept of presenting only
+unique labels.
+It is therefore important to define precedence rules in order
+that a deterministic set of uniquely named labels can be defined.
+.PP
+As a rule of thumb, the labels set closest to the PMNS leaf nodes
+take precedence.
+.PP
+Note that labels set by the Performance Metric Collection System
+(PMCS \- i.e. PMCD and all PMDAs) itself cannot be modified by
+.B pmlabel
+- only custom labels may be created, updated, and deleted.
+However, any PMCS labels that are set on metrics, instances, or
+indoms can be overridden by other labels following the precedence
+rules above.
+.PP
+Global context labels (as reported by the
+.I pmcd.labels
+metric) are the lowest precedence.
+Metric labels are a step higher, then instance domain labels,
+and finally labels set on individual instances have highest
+precedence.
+.PP
+At any given level in this hierarchy, a custom label set by
+.B pmlabel
+will take precedence over a PMCS label at the same level.
+.PP
+For further details and examples, refer to the discussion on the
+.BR pmLookupLabels (3)
+manual page.
+.SH "PCP ENVIRONMENT"
+Environment variables with the prefix
+.B PCP_
+are used to parameterize the file and directory names
+used by PCP.
+On each installation, the file
+.I /etc/pcp.conf
+contains the local values for these variables.
+The
+.B $PCP_CONF
+variable may be used to specify an alternative
+configuration file,
+as described in
+.BR pcp.conf (5).
+.SH SEE ALSO
+.BR pmcd (1),
+.BR pminfo (1),
+.BR pmLookupLabels (3)
+and
+.BR PMNS (5).
diff --git a/man/man3/pmdacache.3 b/man/man3/pmdacache.3
index 1a05b10..1245dc3 100644
--- a/man/man3/pmdacache.3
+++ b/man/man3/pmdacache.3
@@ -144,7 +144,7 @@ This rule is perhaps the responsibility of the PMDA, but is often
 a characteristic of the environment in which the PMDA runs.
 .IP 4. 3n
 It is preferable, although not mandatory, for the association between
-and external instance name and an internal instance identifier to
+an external instance name and an internal instance identifier to
 be persistent.
 This rule is supported by the
 .B pmdaCache
@@ -717,6 +717,7 @@ directory.
 .BR PMDA (3),
 .BR pmdaInit (3),
 .BR pmdaInstance (3),
+.BR pmdaLabels (3),
 .BR pmdaFetch (3),
 .BR pmdaSetFetchCallback (3),
 .BR pmErrStr (3)
diff --git a/man/man3/pmdainstance.3 b/man/man3/pmdainstance.3
index 8b7fd97..195961a 100644
--- a/man/man3/pmdainstance.3
+++ b/man/man3/pmdainstance.3
@@ -105,6 +105,118 @@ structure is allocated with
 .BR malloc (3)
 and should be released by the caller with
 .BR free (3).
+.SH MULTI-DIMENSIONAL INSTANCE NAMING
+Further to the above description of
+.IR name ,
+the set of rules describing external instance names is provided
+in the
+.BR pmdaCacheStore (3)
+manual page.
+.PP
+Instance domains adds another dimension (set of values) to metrics.
+However, this may not suffice to describe complex multi-dimensional
+instance domain situations.
+For this case the approach used by a number of PMDAs is to structure
+the external instance names
+using a delimiter (``/'' or ``::'' are most commonly used) to allow
+separation of the other dimensions.
+In this situation, instance domain labels should be used to define
+names for each instance name component.
+This allows
+.BR PMAPI (3)
+client tools to identify and refine value fetches
+to specific dimensions of interest.
+.PP
+For example, some of the Linux kernel cgroup (control group) metric
+instance domains are multi-dimensional.
+The instance domain represents individual values across both control
+groups and CPUs, making this a two-dimensional instance domain.
+The instance names associated with this cgroup metrics indom have
+been structured using the ``::'' delimiter to separate the two
+dimensions.
+The instance domain itself has been labeled accordingly, as follows.
+.P
+.ft CW
+.nf
+.B "$ pminfo \-\-desc \-\-fetch \-\-labels cgroup.cpuacct.usage_percpu"
+cgroup.cpuacct.usage_percpu
+.in +0.5i
+Data Type: 64-bit unsigned int  InDom: 3.22 0xc00016
+Semantics: counter  Units: nanosec
+inst [0 or "/::cpu0"] value 713787
+inst [1 or "/::cpu1"] value 353969
+inst [2 or "/app::cpu0"] value 407816
+inst [3 or "/app::cpu1"] value 202747
+Labels: {}
+inst [0 or "/::cpu0"] labels {"domain":["cgroup","cpu"],"cgroup":"/","cpu":"cpu0"}
+inst [1 or "/::cpu1"] labels {"domain":["cgroup","cpu"],"cgroup":"/","cpu":"cpu1"}
+inst [2 or "/app::cpu0"] labels {"domain":["cgroup","cpu"],"cgroup":"/app","cpu":"cpu0"}
+inst [3 or "/app::cpu1"] labels {"domain":["cgroup","cpu"],"cgroup:"/app","cpu":"cpu1"}
+.in
+.fi
+.ft 1
+.P
+.ft CW
+.nf
+.B "$ pminfo \-\-labels 3.22"
+InDom: 3.22 0xc00016
+.in +0.5i
+Labels: {"domain":["cgroup","cpu"]}
+.in
+.fi
+.ft 1
+.P
+As shown above the individual instances inherit the labels from
+the instance domain, and the PMDA also applies additional
+per-instance labels describing individual cgroup and CPU names.
+When this model has been used by the PMDA, PMAPI clients are
+able to restrict their fetches to the cgroup metric instances \-
+in the example, restricting to processor "cpu0" using the "cpu"
+label, perhaps, or to just the "/app" cgroup metrics using the
+"cgroup" label.
+.PP
+Furthermore, using this labeling scheme client tools can also
+correlate related instances across different instance domains.
+.P
+.ft CW
+.nf
+.B "$ pminfo \-\-desc \-\-fetch \-\-labels kernel.percpu.cpu.irq.soft"
+kernel.percpu.cpu.irq.soft
+.in +0.5i
+Data Type: 64-bit unsigned int  InDom: 60.0 0xf000000
+Semantics: counter  Units: millisec
+inst [0 or "cpu0"] value 6770
+inst [1 or "cpu1"] value 100
+Labels: {}
+inst [0 or "cpu0"] labels {"domain":"cpu"}
+inst [1 or "cpu1"] labels {"domain":"cpu"}
+.in
+.fi
+.ft 1
+.P
+.ft CW
+.nf
+.B "$ pminfo \-\-labels 60.0"
+InDom: 60.0 0xf000000
+.in +0.5i
+Labels: {"domain":"cpu"}
+.in
+.fi
+.ft 1
+.P
+Although these two metrics have different instance domains (\c
+.I 60.0
+and
+.I 3.22
+respectively) and are sourced from different PMDAs, the "domain"
+label identifies the shared (sub-)domain \- "cpu".
+.PP
+The PMAPI
+.BR pmAddLabelProfile (3),
+.BR pmDelLabelProfile (3)
+family of interfaces can then setup the fetch profile for this
+sub-domain. (?)
+[TODO]
 .SH DIAGNOSTICS
 If any errors occur during the execution of
 .BR pmdaInstance ,
@@ -134,6 +246,10 @@ or later, as specified in the call to
 .BR pmdaDSO (3)
 or 
 .BR pmdaDaemon (3).
+If labeling of multi-dimensional instance names is performed,
+the PMDA must use
+.B PMDA_INTERFACE_7
+or later.
 .PP
 Because of optional partial matching up to the first space in the instance 
 name, the
@@ -143,6 +259,8 @@ spaces, the names are unique up to the first space.
 .SH SEE ALSO
 .BR malloc (3),
 .BR PMAPI (3),
-.BR PMDA (3)
+.BR PMDA (3),
+.BR pmdaCacheStore (3),
+.BR pmdaLabels (3)
 and
 .BR pmGetInDom (3).
diff --git a/man/man3/pmdalabels.3 b/man/man3/pmdalabels.3
new file mode 100644
index 0000000..5e3081d
--- /dev/null
+++ b/man/man3/pmdalabels.3
@@ -0,0 +1,108 @@
+'\"macro stdmacro
+.\"
+.\" Copyright (c) 2016 Red Hat.  All Rights Reserved.
+.\"
+.\" This program is free software; you can redistribute it and/or modify it
+.\" under the terms of the GNU General Public License as published by the
+.\" Free Software Foundation; either version 2 of the License, or (at your
+.\" option) any later version.
+.\"
+.\" This program is distributed in the hope that it will be useful, but
+.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+.\" or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+.\" for more details.
+.\"
+.TH PMDALABELS 3 "PCP" "Performance Co-Pilot"
+.SH NAME
+\f3pmdaLabels\f1 \- manipulate metric labels for a PMDA
+.SH "C SYNOPSIS"
+.ft 3
+#include <pcp/pmapi.h>
+.br
+#include <pcp/impl.h>
+.br
+#include <pcp/pmda.h>
+.sp
+int pmdaLabels(int \fIident\fP, int \fItype\fP, char **\fIbuffer\fP, pmdaExt *\fIpmda\fP);
+.sp
+cc ... \-lpcp_pmda \-lpcp
+.ft 1
+.SH DESCRIPTION
+As part of the Performance Metrics Domain Agent (PMDA) API (see
+.BR PMDA (3)),
+.B pmdaLabels
+uses the standard
+.BR PMDA (3)
+data structures to return the labels for metric, instance domain
+or instance
+.I ident
+in ``JSONB'' format in the given
+.IR buffer .
+.PP
+The
+.I type
+argument is a bit mask that encodes the interpretation of
+.I ident
+and the requested form of help text,
+as follows:
+either
+.B PM_TYPE_PMID
+if
+.I ident
+is a metric identifier,
+.B PM_TYPE_INDOM
+if
+.I ident
+is an instance domain identifier, or
+.B PM_TYPE_INSTANCE
+if
+.I ident
+is an internal instance identifier,
+plus the operation to perform (create, update, delete).
+.PP
+The
+.I buffer
+is managed internally (usually it is cached),
+and it should
+.B not
+be released or freed by the caller of
+.BR pmdaLabels .
+.PP
+Note that any custom (user-defined) labels created by the
+.BR pmlabel (1)
+utility are automatically added to the returned set, and
+hence should not be included in the set returned by the
+PMDA interface labels callback.
+.SH FILES
+Custom label persistence uses JSONB formatted files, with names
+constructed from the PMID, PMID-with-instance-ID, or InDom
+ID within the
+.BI $PCP_VAR_DIR/config/labels
+directory.
+These files are named using dotted notation, distinguishing
+metric identifiers (X.Y.Z), instance domain identifiers (X.Z),
+and individual instances of a metric (X.Y.Z.N).
+.SH DIAGNOSTICS
+If the requested labels
+could not be obtained, 
+.B pmdaLabels
+will return
+.BR PM_ERR_LABELS .
+.SH CAVEAT
+The PMDA must be using 
+.B PMDA_PROTOCOL_7
+or later, as specified in the call to 
+.BR pmdaDSO (3)
+or 
+.BR pmdaDaemon (3).
+.SH SEE ALSO
+.BR pmlabels (1),
+.BR newhelp (1),
+.BR malloc (3),
+.BR PMAPI (3),
+.BR PMDA (3),
+.BR pmdaDaemon (3),
+.BR pmdaDSO (3),
+.BR pmdaInit (3),
+and
+.BR pmLookupLabels (3).
diff --git a/man/man3/pmfetch.3 b/man/man3/pmfetch.3
index 13be92a..d4b36e8 100644
--- a/man/man3/pmfetch.3
+++ b/man/man3/pmfetch.3
@@ -292,6 +292,8 @@ An attempt has been made to restart at least one failed PMDA.
 At least one PMDA has been started.
 .IP \fBPMCD_DROP_AGENT\fR
 PMCD has noticed the termination of at least one PMDA.
+.IP \fBPMCD_LABEL_CHANGE\fR
+At least one metric has been affected by a change in label.
 .PP
 The default is to return zero to indicate
 no change in state, however
@@ -312,6 +314,7 @@ zero or greater than zero.
 .BR pmFreeResult (3),
 .BR pmGetInDom (3),
 .BR pmLookupDesc (3),
+.BR pmLookupLabels (3),
 .BR pmLookupName (3),
 .BR pmNewContext (3),
 .BR pmSetMode (3),
diff --git a/man/man3/pmlookuplabels.3 b/man/man3/pmlookuplabels.3
new file mode 100644
index 0000000..bc2bd5b
--- /dev/null
+++ b/man/man3/pmlookuplabels.3
@@ -0,0 +1,374 @@
+'\"macro stdmacro
+.\"
+.\" Copyright (c) 2016 Red Hat.
+.\"
+.\" This program is free software; you can redistribute it and/or modify it
+.\" under the terms of the GNU General Public License as published by the
+.\" Free Software Foundation; either version 2 of the License, or (at your
+.\" option) any later version.
+.\"
+.\" This program is distributed in the hope that it will be useful, but
+.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+.\" or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+.\" for more details.
+.\"
+.\" add in the -me strings for super and subscripts
+.ie n \{\
+.       ds [ \u\x'-0.25v'
+.       ds ] \d
+.       ds { \d\x'0.25v'
+.       ds } \u
+.\}
+.el \{\
+.       ds [ \v'-0.4m'\x'-0.2m'\s-3
+.       ds ] \s0\v'0.4m'
+.       ds { \v'0.4m'\x'0.2m'\s-3
+.       ds } \s0\v'-0.4m'
+.\}
+.TH PMLOOKUPLABELS 3 "PCP" "Performance Co-Pilot"
+.SH NAME
+\f3pmLookupLabels\f1 \- obtain labels associated with performance metrics
+.SH "C SYNOPSIS"
+.ft 3
+#include <pcp/pmapi.h>
+.sp
+.nf
+int pmLookupLabels(int \fInumpmid\fP, pmID *\fIpmidlist\fP, pmAnnotation **\fIannotation\fP);
+.br
+void pmFreeAnnotation(pmAnnotation **\fIannotation\fP);
+.fi
+.sp
+.nf
+int pmGetContextLabels(char **\fIlabels\fP);
+.br
+int pmGetInDomLabels(pmInDom \fIindom\fP, char **\fIlabels\fP);
+.br
+int pmGetMetricLabels(pmID \fIpmid\fP, char **\fIlabels\fP);
+.br
+int pmGetValueLabels(pmID *\fIpmid\fP, pmLabeledSet **\fIlabeledset\fP);
+.sp
+cc ... \-lpcp
+.ft 1
+.SH DESCRIPTION
+.de CW
+.ie t \f(CW\\$1\fR\\$2
+.el \fI\\$1\fR\\$2
+..
+Given a list of Performance Metric Identifiers (PMID)s,
+e.g. as constructed by
+.BR pmLookupName (3),
+via
+.I pmidlist
+and
+.IR numpmid ,
+lookup the merged label set for these performance metrics.
+.PP
+These value labels (name:value pairs) are collated according to
+the precedence rules described on
+.BR pmlabel (1).
+They follow the naming conventions and ``JSONB'' formatting also
+described there.
+.PP
+The call to
+.B pmLookupLabels
+is executed in the context of a source of metrics,
+instance profile and collection time,
+previously established by calls to
+the appropriate context and profile functions, namely some of
+.BR pmNewContext (3),
+.BR pmDupContext (3),
+.BR pmUseContext (3),
+.BR pmAddProfile (3),
+.BR pmDelProfile (3)
+and
+.BR pmSetMode (3).
+.PP
+The principal result from
+.B pmLookupLabels
+is returned in the
+argument
+.I annotation
+as a tree, using the following component data structures;
+.PP
+.ft CW
+.nf
+.in +0.5i
+typedef struct pmLabels {
+      unsigned int      id;          /* instance/indom/NULL ID */
+      unsigned int      llen;        /* length of JSONB string */
+      char              *labels;     /* JSONB name:value labels */
+} pmLabels;
+
+typedef struct pmLabeledSet {
+      pmID              pmid;        /* instance identifier */
+      int               numval;      /* value count or error code */
+      pmLabels          *vlist;      /* set of values with labels */
+} pmLabeledSet;
+
+typedef struct pmAnnotation {
+      int               numpmid;
+      pmLabeledSet      *labeledset;
+} pmAnnotation;
+.in
+.fi
+.ft 1
+.PP
+To accommodate metrics with multiple value instances, the
+.CW numval
+field indicates how many labeledsets are returned for each
+requested PMID.
+.PP
+If one value (i.e. associated with a particular instance)
+for a requested metric has no labels at the requested time,
+then there is no associated 
+.CW pmLabels
+structure in the 
+.IR annotation .
+If there are no available labels for a metric,
+then
+.CW numval
+will be zero and the associated 
+.CW pmLabels[]
+instance will be empty (however 
+.CW pmid
+will be correctly set to the PMID of the metric with no labels).
+.PP
+As an extension of this protocol,
+if the Performance Metrics Collection System (PMCS)
+is able to provide a reason why no labels are available 
+for a particular metric,
+this is encoded as a standard error code in the corresponding 
+.CW numval .
+Since the error codes are all negative,
+values for a requested metric are `unavailable' if
+.CW numval
+is less than, or equal to, zero.
+A performance metrics value may be `unavailable'
+for a number of reasons; the following list is illustrative but
+not exhaustive:
+of the software for the associated Performance Metric Domain
+.IP "+"
+Collection is not currently activated
+in the software for the associated Performance Metric Domain
+.IP "+"
+The associated PMID is not known
+.IP "+"
+The current system configuration does not include
+the associated hardware component and/or the associated software module,
+e.g. a disk is not installed, or off-line, or a database is not installed
+.PP
+In general, we may not be able to differentiate between the various cases,
+and if differentiation is not possible, 
+.CW numval
+will simply be zero.
+.PP
+The argument definition and the annotation specifications have been
+constructed to ensure that for each PMID in the requested 
+.I pmidlist
+there is exactly one 
+.CW pmLabeledSet
+in the 
+.IR annotation ,
+and further the PMIDs appear in exactly the same sequence in both
+.I pmidlist
+and 
+.IR annotation .
+This makes the number 
+and order of entries in 
+.I annotation
+completely deterministic,
+and greatly simplifies the application programming logic
+after the call to 
+.BR pmLookupLabels .
+.PP
+The 
+.I annotation
+structure returned by 
+.B pmLookupLabels
+is dynamically allocated using
+.BR malloc (3)
+calls,
+and should be released when no longer required by calling
+.BR pmFreeAnnotation (3)
+\- under no circumstances should 
+.BR free (3)
+be called directly to release this space.
+.PP
+As common error conditions are encoded
+in the 
+.I annotation
+data structure, we'd expect only cataclysmic events
+to cause an error value to be returned.
+One example would be if the metrics source context was a remote host,
+and that host or the PMCS on that host became unreachable.
+Otherwise the value returned by the 
+.B pmLookupLabels
+function will be non-negative.
+.SH UNMERGED LABELS
+The
+.B pmLookupLabels
+interface returns the fully merged sets of metric labels as most
+PMAPI applications would need to use them.
+To minimise duplication and for efficiency in communication and
+storage, within the PMCS labels are maintained as a hierarchy.
+The complete set of labels associated with any individual metric
+value consists of labels from these separate sets:
+.TP 4n
+1. Global labels (apply to every metric in the PMCS)
+.TP 4n
+2. Instance Domain labels (apply to all metrics sharing that indom)
+.TP 4n
+3. Per-metric labels (apply to a metric and any instances it has)
+.TP 4n
+4. Per-value labels (apply to individual instances)
+.P
+The third and fourth label levels are equivalent for singleton
+metrics (without an instance domain) such as
+.IR hinv.ncpu .
+.PP
+Some low level PMAPI utilities (such as
+.BR pmlogger (1)
+itself)
+need to access the label hierarchy before label merging takes place.
+A second group of interfaces is available for this class of tool.
+.PP
+.TP 4n
+1.
+.B pmGetContextLabels
+returns the global context labels as a JSONB formatted string.
+.TP 4n
+2.
+.B pmGetInDomLabels
+returns JSONB formatted
+.I labels
+associated with instance domain
+.IR indom .
+.TP 4n
+3.
+.B pmGetMetricLabels
+returns JSONB formatted
+.I labels
+associated with metric
+.IR pmid .
+.TP 4n
+4.
+.B pmGetValueLabels
+returns a
+.IR labeledset
+of JSONB formatted labels stored for the (potentially multiple)
+values associated with metric
+.IR pmid .
+.PP
+Use these lower level interfaces if you need to extract the labels
+that form the input to the merge process and precedence rules
+described on the
+.BR pmlabel (1)
+manual page.
+.SH EXAMPLES
+Consider a deployment with global labels as follows:
+.P
+.ft CW
+.nf
+.B "$ cat $PCP_SYSCONF_DIR/labels.d/*"
+{"environment":"production"}
+{"datacenter":"hong kong"}
+.fi
+.ft 1
+.P
+An unlabeled context switch metric (\c
+.IR kernel.all.pswitch ,
+PMID
+.IR 60.2.0 )
+and the kernel load average metric (\c
+.IR kernel.all.load ,
+PMID
+.IR 60.0.13 )
+labeled as:
+.P
+.ft CW
+.nf
+.B "$ pminfo --labels kernel.all.load"
+kernel.all.load
+.in +0.5i
+Labels: {"statistic":"average"}
+.in
+.fi
+.ft 1
+.P
+In this scenario, a call to
+.B pmLookupLabels
+for these two metrics will return the following merged label
+.IR annotations :
+.P
+.BR pmLookupLabels "(\fI2\fP, [\fI60.2.0\fP, \fI60.0.13\fP], &\fIannotations\fP);"
+.br
+\fIannotations\fP = pmAnnotation {
+    .numpmid = 2
+    .labeledset[0] = {
+        .pmid = 60.2.0          /* kernel.all.load */
+        .numval = 3
+        .vlist[0] = {
+            .id = 1
+            .length = 75
+            .labels = {"datacenter":"hong kong","environment":"production","statistic":"average"}
+        }
+        .vlist[1] = {
+            .id = 5
+            .length = 75
+            .labels = {"datacenter":"hong kong","environment":"production","statistic":"average"}
+        }
+        .vlist[2] = {
+            .id = 15
+            .length = 75
+            .labels = {"datacenter":"hong kong","environment":"production","statistic":"average"}
+        }
+    }
+    .labeledset[1] = {
+        .pmid = 60.0.13         /* kernel.all.pswitch */
+        .numval = 1
+        .vlist[0] = {
+            .id = 0xffffffff
+            .length = 53
+            .labels = {"datacenter":"hong kong","environment":"production"}
+        }
+    }
+.br
+}
+.fi
+.ft 1
+.P
+.SH SEE ALSO
+.BR pmcd (1),
+.BR pmAddProfile (3),
+.BR PMAPI (3),
+.BR pmDelProfile (3),
+.BR pmDupContext (3),
+.BR pmFetch (3),
+.BR pmGetInDom (3),
+.BR pmLookupDesc (3),
+.BR pmLookupName (3),
+.BR pmNewContext (3),
+.BR pmSetMode (3),
+.BR pmUseContext (3)
+and
+.BR pmWhichContext (3).
+.SH DIAGNOSTICS
+As mentioned above,
+.B pmLookupLabels
+returns error codes
+.I insitu
+in the argument
+.IR annotation .
+If no result is returned,
+e.g. due to IPC failure using the current PMAPI context, or
+end of file on an archive log,
+then
+.B pmLookupLabels
+will return a negative error code which may be examined using
+.BR pmErrStr (3).
+.IP \f3PM_ERR_EOL\f1
+When fetching records from an archive log,
+.B pmLookupLabels
+returns this error code to indicate the end of the log has been
+passed (or the start of the log has been passed, if the direction
+of traversal is backwards in time).
diff --git a/src/include/pcp/pmapi.h b/src/include/pcp/pmapi.h
index c0e59b0..c9a549d 100644
--- a/src/include/pcp/pmapi.h
+++ b/src/include/pcp/pmapi.h
@@ -195,6 +195,7 @@ typedef struct pmDesc {
 #define PM_ERR_LOGCHANGEINDOM	(-PM_ERR_BASE-61)   /* The instance domain of a metric has changed in an archive */
 #define PM_ERR_LOGCHANGEUNITS	(-PM_ERR_BASE-62)   /* The units of a metric have changed in an archive */
 #define PM_ERR_NEEDCLIENTCERT	(-PM_ERR_BASE-63)   /* PMCD requires a client certificate */
+#define PM_ERR_LABELS		(-PM_ERR_BASE-64)    /* Missing metric label(s) */
 
 /* retired PM_ERR_CTXBUSY (-PM_ERR_BASE-97) Context is busy */
 #define PM_ERR_TOOSMALL		(-PM_ERR_BASE-98)   /* Insufficient elements in list */
@@ -478,6 +479,37 @@ PCP_CALL extern int pmFetch(int, pmID *, pmResult **);
 PCP_CALL extern int pmFetchArchive(pmResult **);
 
 /*
+ * Support for annotating metric values with labels (name:value pairs).
+ * The full set of labels for a given metric instance is the union of
+ * those found at the source (host/archive) level, those set at the
+ * (optional) indom level, and those set at the level of metric values.
+ */
+typedef struct pmLabels {
+    unsigned int 	id;		/* instance ID, InDom ID, PM_I*_NULL */
+    unsigned int	llen;		/* length of a complete JSONB string */
+    char		*labels;	/* JSONB string of name:value labels */
+} pmLabels;
+
+typedef struct pmLabeledSet {
+    pmID		pmid;		/* metric identifier */
+    int			numval;		/* number of values or error code */
+    pmLabels		*vlist;		/* set of values with label lists */
+} pmLabeledSet;
+
+typedef struct pmAnnotation {
+    int			numpmid;
+    pmLabeledSet	*labeledset;
+} pmAnnotation;
+
+PCP_CALL extern int pmLookupLabels(int, pmID *, pmAnnotation **);
+PCP_CALL extern void pmFreeAnnotation(pmAnnotation *);
+
+PCP_CALL extern int pmGetContextLabels(char **);
+PCP_CALL extern int pmGetMetricLabels(pmID, pmLabels **);
+PCP_CALL extern int pmGetInDomLabels(pmInDom, pmLabels **);
+PCP_CALL extern int pmGetValueLabels(int, pmID *, pmAnnotation **);
+
+/*
  * struct timeval is sometimes 2 x 64-bit ... we use a 2 x 32-bit format for
  * PDUs, internally within libpcp and for (external) archive logs
  */
diff --git a/src/include/pcp/pmda.h b/src/include/pcp/pmda.h
index 7e58de6..8d3571e 100644
--- a/src/include/pcp/pmda.h
+++ b/src/include/pcp/pmda.h
@@ -30,7 +30,8 @@ extern "C" {
 #define PMDA_INTERFACE_5	5	/* client context in pmda and */
 					/* 4-state return from fetch callback */
 #define PMDA_INTERFACE_6	6	/* client security attributes in pmda */
-#define PMDA_INTERFACE_LATEST	6
+#define PMDA_INTERFACE_7	7	/* metric value label support in pmda */
+#define PMDA_INTERFACE_LATEST	7
 
 /*
  * Type of I/O connection to PMCD (pmdaUnknown defaults to pmdaPipe)
@@ -256,6 +257,25 @@ typedef struct pmdaInterface {
 	    int     (*attribute)(int, int, const char *, int, pmdaExt *);
 	} six;
 
+/*
+ * Interface Version 7 (metric value label support in PMDA).
+ * PMDA_INTERFACE_7
+ */
+	struct {
+	    pmdaExt *ext;
+	    int	    (*profile)(pmdaInProfile *, pmdaExt *);
+	    int	    (*fetch)(int, pmID *, pmResult **, pmdaExt *);
+	    int	    (*desc)(pmID, pmDesc *, pmdaExt *);
+	    int	    (*instance)(pmInDom, int, char *, pmdaInResult **, pmdaExt *);
+	    int	    (*text)(int, int, char **, pmdaExt *);
+	    int	    (*store)(pmResult *, pmdaExt *);
+	    int     (*pmid)(const char *, pmID *, pmdaExt *);
+	    int     (*name)(pmID, char ***, pmdaExt *);
+	    int     (*children)(const char *, int, char ***, int **, pmdaExt *);
+	    int     (*attribute)(int, int, const char *, int, pmdaExt *);
+	    int	    (*labels)(int, int, char **, pmdaExt *);
+	} seven;
+
     } version;
 
 } pmdaInterface;
@@ -455,10 +475,10 @@ PMDA_CALL extern void pmdaSetEndContextCallBack(pmdaInterface *, pmdaEndContextC
  *	Return description of instances and instance domains.
  *
  * pmdaDesc
- *	Return the metric desciption.
+ *	Return the metric description.
  *
  * pmdaText
- *	Return the help text for the metric.
+ *	Return the help text for the metric or instance domain.
  *
  * pmdaStore
  *	Store a value into a metric. This is a no-op.
@@ -483,6 +503,9 @@ PMDA_CALL extern void pmdaSetEndContextCallBack(pmdaInterface *, pmdaEndContextC
  *	such as the authenticated userid.  Passed in a client identifier,
  *	numeric PCP_ATTR, pointer to the associated value, and the length
  *	of the value.
+ *
+ * pmdaLabels
+ *	Return the JSONB labels for the metric, instance domain or instance.
  */
 
 PMDA_CALL extern int pmdaProfile(pmdaInProfile *, pmdaExt *);
@@ -495,6 +518,7 @@ PMDA_CALL extern int pmdaPMID(const char *, pmID *, pmdaExt *);
 PMDA_CALL extern int pmdaName(pmID, char ***, pmdaExt *);
 PMDA_CALL extern int pmdaChildren(const char *, int, char ***, int **, pmdaExt *);
 PMDA_CALL extern int pmdaAttribute(int, int, const char *, int, pmdaExt *);
+PMDA_CALL extern int pmdaLabels(int, int, char **, pmdaExt *);
 
 /*
  * PMDA "help" text manipulation
@@ -503,6 +527,8 @@ PMDA_CALL extern int pmdaOpenHelp(const char *);
 PMDA_CALL extern void pmdaCloseHelp(int);
 PMDA_CALL extern char *pmdaGetHelp(int, pmID, int);
 PMDA_CALL extern char *pmdaGetInDomHelp(int, pmInDom, int);
+PMDA_CALL extern char *pmdaGetLabels(int, pmID, int);
+PMDA_CALL extern char *pmdaGetInDomLabels(int, pmInDom, int);
 
 /*
  * Dynamic metric table manipulation