Re: How to create a new Data Type and Consolidation Function?

Index: src/rrd_dump.c
===================================================================
--- src/rrd_dump.c	(revision 2180)
+++ src/rrd_dump.c	(working copy)
@@ -271,6 +271,7 @@
         case CF_MAXIMUM:
         case CF_MINIMUM:
         case CF_LAST:
+        case CF_SUM:
         default:
             CB_FMTS("\t\t<xff>%0.10e</xff>\n",
                 rrd.rra_def[i].par[RRA_cdp_xff_val].u_val);
@@ -390,6 +391,7 @@
             case CF_MAXIMUM:
             case CF_MINIMUM:
             case CF_LAST:
+            case CF_SUM:
             default:
                 value = rrd.cdp_prep[i * rrd.stat_head->ds_cnt + ii].scratch[CDP_val].u_val;
                 if (isnan(value)) {
Index: src/rrd_create.c
===================================================================
--- src/rrd_create.c	(revision 2180)
+++ src/rrd_create.c	(working copy)
@@ -255,6 +255,7 @@
             case DST_ABSOLUTE:
             case DST_GAUGE:
             case DST_DERIVE:
+            case DST_DIFFERENCE:
                 parseGENERIC_DS(&argv[i][offset + 3], &rrd,
                                 rrd.stat_head->ds_cnt);
                 break;
Index: src/rrd_update.c
===================================================================
--- src/rrd_update.c	(revision 2180)
+++ src/rrd_update.c	(working copy)
@@ -165,7 +165,8 @@
     double post_int,
     long diff_pdp_st,
     rrd_value_t *pdp_new,
-    rrd_value_t *pdp_temp);
+    rrd_value_t *pdp_temp,
+    unsigned long elapsed_pdp_st);
 
 static int update_all_cdp_prep(
     rrd_t *rrd,
@@ -1056,6 +1057,7 @@
              */
             switch (dst_idx) {
             case DST_COUNTER:
+            case DST_DIFFERENCE:
             case DST_DERIVE:
                 /* Check if this is a valid integer. `U' is already handled in
                  * another branch. */
@@ -1077,7 +1079,7 @@
                     pdp_new[ds_idx] =
                         rrd_diff(updvals[ds_idx + 1],
                                  rrd->pdp_prep[ds_idx].last_ds);
-                    if (dst_idx == DST_COUNTER) {
+                    if (dst_idx == DST_COUNTER || dst_idx == DST_DIFFERENCE) {
                         /* simple overflow catcher. This will fail
                          * terribly for non 32 or 64 bit counters
                          * ... are there any others in SNMP land?
@@ -1291,7 +1293,7 @@
     for (ds_idx = 0; ds_idx < rrd->stat_head->ds_cnt; ds_idx++) {
         if (process_pdp_st(rrd, ds_idx, interval, pre_int, post_int,
                            elapsed_pdp_st * rrd->stat_head->pdp_step,
-                           pdp_new, pdp_temp) == -1) {
+                           pdp_new, pdp_temp,elapsed_pdp_st) == -1) {
             return -1;
         }
 #ifdef DEBUG
@@ -1325,7 +1327,8 @@
     double post_int,
     long diff_pdp_st,   /* number of seconds in full steps passed since last update */
     rrd_value_t *pdp_new,
-    rrd_value_t *pdp_temp)
+    rrd_value_t *pdp_temp,
+    unsigned long elapsed_pdp_st)
 {
     int       i;
 
@@ -1338,6 +1341,8 @@
 
     rpnstack_init(&rpnstack);
 
+    enum dst_en dst_idx;
+    dst_idx = dst_conv(rrd->ds_def[ds_idx].dst);
 
     if (isnan(pdp_new[ds_idx])) {
         /* a final bit of unknown to be added before calculation
@@ -1358,9 +1363,20 @@
          (signed) scratch[PDP_unkn_sec_cnt].u_cnt)) {
         pdp_temp[ds_idx] = DNAN;
     } else {
-        pdp_temp[ds_idx] = scratch[PDP_val].u_val /
-            ((double) (diff_pdp_st - scratch[PDP_unkn_sec_cnt].u_cnt) -
-             pre_unknown);
+        pdp_temp[ds_idx] = scratch[PDP_val].u_val;
+        if ( dst_idx != DST_DIFFERENCE ) {
+            pdp_temp[ds_idx] /= 
+              ((double) (diff_pdp_st - scratch[PDP_unkn_sec_cnt].u_cnt) - 
+               pre_unknown);
+        }
+        else {
+            // DST_DIFFERENCE, have to divide by elapsed_pdp_st to 
+            // distribute the value over the elapsed period
+            // otherwise, if elapsed_pdp_st == 2, 
+            // pdp_temp == twice the value already and 
+            // we will be adding this doubled value twice
+            pdp_temp[ds_idx] /= elapsed_pdp_st;
+        }
     }
 
     /* process CDEF data sources; remember each CDEF DS can
@@ -1664,12 +1680,16 @@
     rrd_value_t cum_val, cur_val;
 
     switch (current_cf) {
+    case CF_SUM:
     case CF_AVERAGE:
         cum_val = IFDNAN(scratch[CDP_val].u_val, 0.0);
         cur_val = IFDNAN(pdp_temp_val, 0.0);
         scratch[CDP_primary_val].u_val =
-            (cum_val + cur_val * start_pdp_offset) /
+            (cum_val + cur_val * start_pdp_offset);
+        if ( current_cf == CF_AVERAGE ) {
+          scratch[CDP_primary_val].u_val /= 
             (pdp_cnt - scratch[CDP_unkn_pdp_cnt].u_cnt);
+        }
         break;
     case CF_MAXIMUM: 
         cum_val = IFDNAN(scratch[CDP_val].u_val, -DINF);
@@ -1734,6 +1754,7 @@
     unival   *scratch = rrd->cdp_prep[cdp_idx].scratch;
 
     switch (current_cf) {
+    case CF_SUM:
     case CF_AVERAGE:
     default:
         scratch[CDP_primary_val].u_val = pdp_temp[ds_idx];
@@ -1786,6 +1807,7 @@
             return -DINF;
         case CF_MINIMUM:
             return DINF;
+        case CF_SUM:
         case CF_AVERAGE:
             return 0;
         default:
@@ -1794,6 +1816,7 @@
     } 
     else {
         switch (current_cf) {
+        case CF_SUM:
         case CF_AVERAGE:
             return pdp_temp_val *  pdp_into_cdp_cnt ;
         default:
@@ -1823,7 +1846,7 @@
     )
 {
     if (isnan(cdp_val)) {
-        if (current_cf == CF_AVERAGE) {
+        if (current_cf == CF_AVERAGE || current_cf == CF_SUM) {
             pdp_temp_val *= elapsed_pdp_st;
         }
 #ifdef DEBUG
@@ -1832,7 +1855,7 @@
 #endif
         return pdp_temp_val;
     }
-    if (current_cf == CF_AVERAGE)
+    if (current_cf == CF_AVERAGE || current_cf == CF_SUM)
         return cdp_val + pdp_temp_val * elapsed_pdp_st;
     if (current_cf == CF_MINIMUM)
         return (pdp_temp_val < cdp_val) ? pdp_temp_val : cdp_val;
Index: src/rrd_format.c
===================================================================
--- src/rrd_format.c	(revision 2180)
+++ src/rrd_format.c	(working copy)
@@ -61,6 +61,7 @@
         converter(GAUGE, DST_GAUGE)
         converter(DERIVE, DST_DERIVE)
         converter(COMPUTE, DST_CDEF)
+        converter(DIFFERENCE, DST_DIFFERENCE)
         rrd_set_error("unknown data acquisition function '%s'", string);
     return (enum dst_en)(-1);
 }
@@ -80,6 +81,7 @@
         converter(SEASONAL, CF_SEASONAL)
         converter(DEVSEASONAL, CF_DEVSEASONAL)
         converter(FAILURES, CF_FAILURES)
+        converter(SUM, CF_SUM)
         rrd_set_error("unknown consolidation function '%s'", string);
     return (enum cf_en)(-1);
 }
@@ -98,6 +100,7 @@
         case CF_DEVSEASONAL: return "DEVSEASONAL";
         case CF_FAILURES:    return "FAILURES";
         case CF_MHWPREDICT:  return "MHWPREDICT";
+        case CF_SUM:         return "SUM";
 
         default:
             return NULL;
Index: src/rrd_format.h
===================================================================
--- src/rrd_format.h	(revision 2180)
+++ src/rrd_format.h	(working copy)
@@ -138,6 +138,7 @@
     DST_ABSOLUTE,
     DST_GAUGE,
     DST_DERIVE,
+    DST_DIFFERENCE,
     DST_CDEF
 };
 
@@ -192,7 +193,8 @@
      * */
     CF_FAILURES,
     /* HWPREDICT that follows a moving baseline */
-    CF_MHWPREDICT
+    CF_MHWPREDICT,
+    CF_SUM
         /* new entries must come last !!! */
 };
 
Index: src/rrd_graph.c
===================================================================
--- src/rrd_graph.c	(revision 2180)
+++ src/rrd_graph.c	(working copy)
@@ -749,6 +749,7 @@
                     case CF_DEVSEASONAL:
                     case CF_DEVPREDICT:
                     case CF_SEASONAL:
+                    case CF_SUM:
                     case CF_AVERAGE:
                         newval += srcptr[i * (*ds_cnt) + col];
                         break;
@@ -782,6 +783,7 @@
                 case CF_FAILURES:
                 case CF_MAXIMUM:
                 case CF_LAST:
+                case CF_SUM:
                     break;
                 }
             }
@@ -1571,6 +1573,9 @@
                         validsteps++;
                         printval += im->gdes[vidx].data[ii];
                         break;
+                    case CF_SUM:
+                        printval += im->gdes[vidx].data[ii];
+                        break;
                     case CF_MINIMUM:
                         printval = min(printval, im->gdes[vidx].data[ii]);
                         break;
@@ -1582,7 +1587,7 @@
                         printval = im->gdes[vidx].data[ii];
                     }
                 }
-                if (im->gdes[i].cf == CF_AVERAGE || im->gdes[i].cf > CF_LAST) {
+                if ( (im->gdes[i].cf == CF_AVERAGE || im->gdes[i].cf > CF_LAST) && im->gdes[i].cf != CF_SUM ) {
                     if (validsteps > 1) {
                         printval = (printval / validsteps);
                     }
Index: CONTRIBUTORS
===================================================================
--- CONTRIBUTORS	(revision 2180)
+++ CONTRIBUTORS	(working copy)
@@ -45,6 +45,7 @@
 McCreary mccreary with xoanon.colorado.edu
 Mike Mitchell <mcm with unx.sas.com>
 Mike Slifcak <slif with bellsouth.net> many rrdtool-1.1.x fixes
+Long Vu <long_at_work at yahoo.ca> (DST_DIFFERENCE and CF_SUM)
 Oleg Cherevko <olwi with icyb.kiev.ua>
 Otmar Lendl <O.Lendl with Austria.EU.net> (lots of bugfixes)
 Pablo Sanchez <pablo at blueoakdb.com> (CDEF vs VDEF)
Index: doc/rrdcreate.pod
===================================================================
--- doc/rrdcreate.pod	(revision 2180)
+++ doc/rrdcreate.pod	(working copy)
@@ -62,9 +62,9 @@
 
 I<DST> defines the Data Source Type. The remaining arguments of a
 data source entry depend on the data source type. For GAUGE, COUNTER,
-DERIVE, and ABSOLUTE the format for a data source entry is:
+DERIVE, ABSOLUTE and DIFFERENCE the format for a data source entry is:
 
-B<DS:>I<ds-name>B<:>I<GAUGE | COUNTER | DERIVE | ABSOLUTE>B<:>I<heartbeat>B<:>I<min>B<:>I<max>
+B<DS:>I<ds-name>B<:>I<GAUGE | COUNTER | DERIVE | ABSOLUTE | DIFFERENCE>B<:>I<heartbeat>B<:>I<min>B<:>I<max>
 
 For COMPUTE data sources, the format is:
 
@@ -138,6 +138,12 @@
 to generate PDPs). In database software, such data sets are referred
 to as "virtual" or "computed" columns.
 
+=item B<DIFFERENCE>
+
+is for counters which is not stored as rate. A usage could be to count
+the number of transaction since the last update stored as a count, not
+as a rate per second like COUNTER.
+
 =back
 
 I<heartbeat> defines the maximum number of seconds that may pass
@@ -179,7 +185,7 @@
 The data is also processed with the consolidation function (I<CF>) of
 the archive. There are several consolidation functions that
 consolidate primary data points via an aggregate function: B<AVERAGE>,
-B<MIN>, B<MAX>, B<LAST>. 
+B<MIN>, B<MAX>, B<LAST>, B<SUM>.
 
 =over
 
@@ -199,6 +205,10 @@
 
 the last data points is used.
 
+=item SUM
+
+the sum of the data points is stored.
+
 =back
 
 Note that data aggregation inevitably leads to loss of precision and
@@ -210,7 +220,7 @@
 The format of B<RRA> line for these
 consolidation functions is:
 
-B<RRA:>I<AVERAGE | MIN | MAX | LAST>B<:>I<xff>B<:>I<steps>B<:>I<rows>
+B<RRA:>I<AVERAGE | MIN | MAX | LAST | SUM>B<:>I<xff>B<:>I<steps>B<:>I<rows>
 
 I<xff> The xfiles factor defines what part of a consolidation interval may
 be made up from I<*UNKNOWN*> data while the consolidated value is still
Index: doc/rrd-beginners.pod
===================================================================
--- doc/rrd-beginners.pod	(revision 2180)
+++ doc/rrd-beginners.pod	(working copy)
@@ -116,7 +116,7 @@
 this is not a problem, RRDtool will interpolate the data accordingly.
 
 B<DST> (Data Source Type) defines the type of the DS. It can be
-COUNTER, DERIVE, ABSOLUTE, GAUGE. A DS declared as COUNTER will save
+COUNTER, DERIVE, ABSOLUTE, GAUGE, DIFFERENCE. A DS declared as COUNTER will save
 the rate of change of the value over a step period. This assumes that
 the value is always increasing (the difference between the current and
 the previous value is greater than 0). Traffic counters on a router
@@ -130,15 +130,19 @@
 the step interval (300 seconds in our example). GAUGE does not save
 the rate of change. It saves the actual value itself. There are no
 divisions or calculations. Memory consumption in a server is a typical
-example of gauge. The difference between the different types DSTs can be
+example of gauge. DIFFERENCE behave exactly like COUNTER but do not
+devide by the step period. So DIFFERENCE is not a rate of change but 
+the actual difference between the current reading and the previous
+reading. The difference between the different types DSTs can be
 explained better with the following example:
 
- Values       = 300, 600, 900, 1200
- Step         = 300 seconds
- COUNTER DS   =    1,  1,   1,    1
- DERIVE DS    =    1,  1,   1,    1
- ABSOLUTE DS  =    1,  2,   3,    4
- GAUGE DS     = 300, 600, 900, 1200
+ Values         = 300, 600, 900, 1200
+ Step           = 300 seconds
+ COUNTER DS     =    1,  1,   1,    1
+ DERIVE DS      =    1,  1,   1,    1
+ ABSOLUTE DS    =    1,  2,   3,    4
+ GAUGE DS       = 300, 600, 900, 1200
+ DIFFERENCE DS  = 300, 300, 300,  300
 
 The next parameter is B<heartbeat>. In our example, heartbeat is 600
 seconds. If the database does not get a new PDP within 300 seconds, it
@@ -164,10 +168,10 @@
  RRA:CF:xff:step:rows
 
 RRA is the keyword to declare RRAs. The consolidation function (CF)
-can be AVERAGE, MINIMUM, MAXIMUM, and LAST. The concept of the
+can be AVERAGE, MINIMUM, MAXIMUM, LAST and SUM. The concept of the
 consolidated data point (CDP) comes into the picture here. A CDP is
-CFed (averaged, maximum/minimum value or last value) from I<step>
-number of PDPs. This RRA will hold I<rows> CDPs.
+CFed (averaged, maximum/minimum value, last value or sum of all the
+values) from I<step> number of PDPs. This RRA will hold I<rows> CDPs.
 
 Lets have a look at the example above. For the first RRA, 12 (steps)
 PDPs (DS variables) are AVERAGEed (CF) to form one CDP. 24 (rows) of