root/src/noit_check_log.c

Revision b09464ce1fe8fb09f4ad58414507f5b965735967, 15.1 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 3 years ago)

add metric dtrace logging

  • Property mode set to 100644
Line 
1 /*
2  * Copyright (c) 2007, OmniTI Computer Consulting, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are
7  * met:
8  *
9  *     * Redistributions of source code must retain the above copyright
10  *       notice, this list of conditions and the following disclaimer.
11  *     * Redistributions in binary form must reproduce the above
12  *       copyright notice, this list of conditions and the following
13  *       disclaimer in the documentation and/or other materials provided
14  *       with the distribution.
15  *     * Neither the name OmniTI Computer Consulting, Inc. nor the names
16  *       of its contributors may be used to endorse or promote products
17  *       derived from this software without specific prior written
18  *       permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #include "noit_defines.h"
34 #include "dtrace_probes.h"
35
36 #include <uuid/uuid.h>
37 #include <netinet/in.h>
38
39 #include "noit_check.h"
40 #include "noit_filters.h"
41 #include "utils/noit_log.h"
42 #include "jlog/jlog.h"
43
44 #include "bundle.pb-c.h"
45 #include "noit_check_log_helpers.h"
46
47 /* Log format is tab delimited:
48  * NOIT CONFIG (implemented in noit_conf.c):
49  *  'n' TIMESTAMP strlen(xmlconfig) base64(gzip(xmlconfig))
50  *
51  * DELETE:
52  *  'D' TIMESTAMP UUID
53  *
54  * CHECK:
55  *  'C' TIMESTAMP UUID TARGET MODULE NAME
56  *
57  * STATUS:
58  *  'S' TIMESTAMP UUID STATE AVAILABILITY DURATION STATUS_MESSAGE
59  *
60  * METRICS:
61  *  'M' TIMESTAMP UUID NAME TYPE VALUE
62  *
63  * BUNDLE
64  *  'B#' TIMESTAMP UUID TARGET MODULE NAME strlen(base64(gzipped(payload))) base64(gzipped(payload))
65  * 
66  */
67
68 static noit_log_stream_t check_log = NULL;
69 static noit_log_stream_t status_log = NULL;
70 static noit_log_stream_t metrics_log = NULL;
71 static noit_log_stream_t delete_log = NULL;
72 static noit_log_stream_t bundle_log = NULL;
73 static noit_boolean use_compression = noit_true;
74
75 #define SECPART(a) ((unsigned long)(a)->tv_sec)
76 #define MSECPART(a) ((unsigned long)((a)->tv_usec / 1000))
77 #define MAKE_CHECK_UUID_STR(uuid_str, len, ls, check) do { \
78   noit_boolean extended_id = noit_false; \
79   const char *v; \
80   v = noit_log_stream_get_property(ls, "extended_id"); \
81   if(v && !strcmp(v, "on")) extended_id = noit_true; \
82   uuid_str[0] = '\0'; \
83   if(extended_id) { \
84     strlcat(uuid_str, check->target, len-37); \
85     strlcat(uuid_str, "`", len-37); \
86     strlcat(uuid_str, check->module, len-37); \
87     strlcat(uuid_str, "`", len-37); \
88     strlcat(uuid_str, check->name, len-37); \
89     strlcat(uuid_str, "`", len-37); \
90   } \
91   uuid_unparse_lower(check->checkid, uuid_str + strlen(uuid_str)); \
92 } while(0)
93
94 static void
95 handle_extra_feeds(noit_check_t *check,
96                    int (*log_f)(noit_log_stream_t ls, noit_check_t *check)) {
97   noit_log_stream_t ls;
98   noit_skiplist_node *curr, *next;
99   const char *feed_name;
100
101   if(!check->feeds) return;
102   curr = next = noit_skiplist_getlist(check->feeds);
103   while(curr) {
104     /* We advance next here (before we try to use curr).
105      * We may need to remove the node we're looking at and that would
106      * disturb the iterator, so advance in advance. */
107     noit_skiplist_next(check->feeds, &next);
108     feed_name = (char *)curr->data;
109     ls = noit_log_stream_find(feed_name);
110     if(!ls || log_f(ls, check)) {
111       noit_check_transient_remove_feed(check, feed_name);
112       /* noit_skiplisti_remove(check->feeds, curr, free); */
113     }
114     curr = next;
115   }
116   /* We're done... we may have destroyed the last feed.
117    * that combined with transience means we should kill the check */
118   /* noit_check_transient_remove_feed(check, NULL); */
119 }
120
121 static int
122 _noit_check_log_delete(noit_log_stream_t ls,
123                        noit_check_t *check) {
124   stats_t *c;
125   char uuid_str[256*3+37];
126   SETUP_LOG(delete, );
127   MAKE_CHECK_UUID_STR(uuid_str, sizeof(uuid_str), status_log, check);
128
129   c = &check->stats.current;
130   return noit_log(ls, &c->whence, __FILE__, __LINE__,
131                   "D\t%lu.%03lu\t%s\n",
132                   SECPART(&c->whence), MSECPART(&c->whence), uuid_str);
133 }
134 void
135 noit_check_log_delete(noit_check_t *check) {
136   if(!(check->flags & NP_TRANSIENT)) {
137     handle_extra_feeds(check, _noit_check_log_delete);
138     SETUP_LOG(delete, return);
139     _noit_check_log_delete(delete_log, check);
140   }
141 }
142
143 static int
144 _noit_check_log_check(noit_log_stream_t ls,
145                       noit_check_t *check) {
146   struct timeval __now;
147   char uuid_str[256*3+37];
148   SETUP_LOG(check, );
149   MAKE_CHECK_UUID_STR(uuid_str, sizeof(uuid_str), check_log, check);
150
151   gettimeofday(&__now, NULL);
152   return noit_log(ls, &__now, __FILE__, __LINE__,
153                   "C\t%lu.%03lu\t%s\t%s\t%s\t%s\n",
154                   SECPART(&__now), MSECPART(&__now),
155                   uuid_str, check->target, check->module, check->name);
156 }
157
158 void
159 noit_check_log_check(noit_check_t *check) {
160   if(!(check->flags & NP_TRANSIENT)) {
161     handle_extra_feeds(check, _noit_check_log_check);
162     SETUP_LOG(check, return);
163     _noit_check_log_check(check_log, check);
164   }
165 }
166
167 static int
168 _noit_check_log_status(noit_log_stream_t ls,
169                        noit_check_t *check) {
170   stats_t *c;
171   char uuid_str[256*3+37];
172   SETUP_LOG(status, );
173   MAKE_CHECK_UUID_STR(uuid_str, sizeof(uuid_str), status_log, check);
174
175   c = &check->stats.current;
176   return noit_log(ls, &c->whence, __FILE__, __LINE__,
177                   "S\t%lu.%03lu\t%s\t%c\t%c\t%d\t%s\n",
178                   SECPART(&c->whence), MSECPART(&c->whence), uuid_str,
179                   (char)c->state, (char)c->available, c->duration, c->status);
180 }
181 void
182 noit_check_log_status(noit_check_t *check) {
183   handle_extra_feeds(check, _noit_check_log_status);
184   if(!(check->flags & (NP_TRANSIENT | NP_SUPPRESS_STATUS))) {
185     SETUP_LOG(status, return);
186     _noit_check_log_status(status_log, check);
187   }
188 }
189 static int
190 _noit_check_log_metric(noit_log_stream_t ls, noit_check_t *check,
191                        const char *uuid_str,
192                        struct timeval *whence, metric_t *m) {
193   char our_uuid_str[256*3+37];
194   int srv = 0;
195   if(!noit_apply_filterset(check->filterset, check, m)) return 0;
196   if(!ls->enabled) return 0;
197
198   if(!uuid_str) {
199     MAKE_CHECK_UUID_STR(our_uuid_str, sizeof(our_uuid_str), metrics_log, check);
200     uuid_str = our_uuid_str;
201   }
202
203   if(!m->metric_value.s) { /* they are all null */
204     srv = noit_log(ls, whence, __FILE__, __LINE__,
205                    "M\t%lu.%03lu\t%s\t%s\t%c\t[[null]]\n",
206                    SECPART(whence), MSECPART(whence), uuid_str,
207                    m->metric_name, m->metric_type);
208   }
209   else {
210     switch(m->metric_type) {
211       case METRIC_INT32:
212         srv = noit_log(ls, whence, __FILE__, __LINE__,
213                        "M\t%lu.%03lu\t%s\t%s\t%c\t%d\n",
214                        SECPART(whence), MSECPART(whence), uuid_str,
215                        m->metric_name, m->metric_type, *(m->metric_value.i));
216         break;
217       case METRIC_UINT32:
218         srv = noit_log(ls, whence, __FILE__, __LINE__,
219                        "M\t%lu.%03lu\t%s\t%s\t%c\t%u\n",
220                        SECPART(whence), MSECPART(whence), uuid_str,
221                        m->metric_name, m->metric_type, *(m->metric_value.I));
222         break;
223       case METRIC_INT64:
224         srv = noit_log(ls, whence, __FILE__, __LINE__,
225                        "M\t%lu.%03lu\t%s\t%s\t%c\t%lld\n",
226                        SECPART(whence), MSECPART(whence), uuid_str,
227                        m->metric_name, m->metric_type,
228                        (long long int)*(m->metric_value.l));
229         break;
230       case METRIC_UINT64:
231         srv = noit_log(ls, whence, __FILE__, __LINE__,
232                        "M\t%lu.%03lu\t%s\t%s\t%c\t%llu\n",
233                        SECPART(whence), MSECPART(whence), uuid_str,
234                        m->metric_name, m->metric_type,
235                        (long long unsigned int)*(m->metric_value.L));
236         break;
237       case METRIC_DOUBLE:
238         srv = noit_log(ls, whence, __FILE__, __LINE__,
239                        "M\t%lu.%03lu\t%s\t%s\t%c\t%.12e\n",
240                        SECPART(whence), MSECPART(whence), uuid_str,
241                        m->metric_name, m->metric_type, *(m->metric_value.n));
242         break;
243       case METRIC_STRING:
244         srv = noit_log(ls, whence, __FILE__, __LINE__,
245                        "M\t%lu.%03lu\t%s\t%s\t%c\t%s\n",
246                        SECPART(whence), MSECPART(whence), uuid_str,
247                        m->metric_name, m->metric_type, m->metric_value.s);
248         break;
249       default:
250         noitL(noit_error, "Unknown metric type '%c' 0x%x\n",
251               m->metric_type, m->metric_type);
252     }
253   }
254   return srv;
255 }
256 static int
257 _noit_check_log_metrics(noit_log_stream_t ls, noit_check_t *check) {
258   int rv = 0;
259   int srv;
260   char uuid_str[256*3+37];
261   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
262   const char *key;
263   int klen;
264   stats_t *c;
265   void *vm;
266   SETUP_LOG(metrics, );
267   MAKE_CHECK_UUID_STR(uuid_str, sizeof(uuid_str), metrics_log, check);
268
269   c = &check->stats.current;
270   while(noit_hash_next(&c->metrics, &iter, &key, &klen, &vm)) {
271     /* If we apply the filter set and it returns false, we don't log */
272     metric_t *m = (metric_t *)vm;
273     srv = _noit_check_log_metric(ls, check, uuid_str, &c->whence, m);
274     if(srv) rv = srv;
275   }
276   return rv;
277 }
278 void
279 noit_check_log_metrics(noit_check_t *check) {
280   handle_extra_feeds(check, _noit_check_log_metrics);
281   if(!(check->flags & (NP_TRANSIENT | NP_SUPPRESS_METRICS))) {
282     SETUP_LOG(metrics, return);
283     _noit_check_log_metrics(metrics_log, check);
284   }
285 }
286
287
288 static int
289 _noit_check_log_bundle_metric(noit_log_stream_t ls, Metric *metric, metric_t *m) {
290   metric->metrictype = (int)m->metric_type;
291
292   metric->name = m->metric_name;
293   if(m->metric_value.vp != NULL) {
294     switch (m->metric_type) {
295       case METRIC_INT32:
296         metric->has_valuei32 = noit_true;
297         metric->valuei32 = *(m->metric_value.i); break;
298       case METRIC_UINT32:
299         metric->has_valueui32 = noit_true;
300         metric->valueui32 = *(m->metric_value.I); break;
301       case METRIC_INT64:
302         metric->has_valuei64 = noit_true;
303         metric->valuei64 = *(m->metric_value.l); break;
304       case METRIC_UINT64:
305         metric->has_valueui64 = noit_true;
306         metric->valueui64 = *(m->metric_value.L); break;
307       case METRIC_DOUBLE:
308         metric->has_valuedbl = noit_true;
309         metric->valuedbl = *(m->metric_value.n); break;
310       case METRIC_STRING:
311         metric->valuestr = m->metric_value.s; break;
312       default:
313         return -1;
314     }
315   }
316   return 0;
317 }
318
319 static int
320 noit_check_log_bundle_serialize(noit_log_stream_t ls, noit_check_t *check) {
321   int rv = 0;
322   char uuid_str[256*3+37];
323   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
324   noit_hash_iter iter2 = NOIT_HASH_ITER_ZERO;
325   const char *key;
326   int klen, i=0, size, j;
327   unsigned int out_size;
328   stats_t *c;
329   void *vm;
330   char *buf, *out_buf;
331   noit_compression_type_t comp;
332   Bundle bundle = BUNDLE__INIT;
333   SETUP_LOG(bundle, );
334   MAKE_CHECK_UUID_STR(uuid_str, sizeof(uuid_str), bundle_log, check);
335
336   // Get a bundle
337   c = &check->stats.current;
338
339   // Set attributes
340   bundle.status = malloc(sizeof(Status));
341   status__init(bundle.status);
342   bundle.status->available = c->available;
343   bundle.status->state = c->state;
344   bundle.status->duration = c->duration;
345   bundle.status->status = c->status;
346
347
348   // Just count
349   while(noit_hash_next(&c->metrics, &iter, &key, &klen, &vm)) {
350     bundle.n_metrics++;
351   }
352
353   if(bundle.n_metrics > 0) {
354     bundle.metrics = malloc(bundle.n_metrics * sizeof(Metric*));
355
356     // Now convert
357     while(noit_hash_next(&c->metrics, &iter2, &key, &klen, &vm)) {
358       /* If we apply the filter set and it returns false, we don't log */
359       metric_t *m = (metric_t *)vm;
360       bundle.metrics[i] = malloc(sizeof(Metric));
361       metric__init(bundle.metrics[i]);
362       _noit_check_log_bundle_metric(ls, bundle.metrics[i], m);
363       if(NOIT_CHECK_METRIC_ENABLED()) {
364         char buff[256];
365         noit_stats_snprint_metric(buff, sizeof(buff), m);
366         NOIT_CHECK_METRIC(uuid_str, check->module, check->name, check->target,
367                           m->metric_name, m->metric_type, buff);
368       }
369       i++;
370     }
371   }
372
373   size = bundle__get_packed_size(&bundle);
374   buf = malloc(size);
375   bundle__pack(&bundle, (uint8_t*)buf);
376
377   // Compress + B64
378   comp = use_compression ? NOIT_COMPRESS_ZLIB : NOIT_COMPRESS_NONE;
379   noit_check_log_bundle_compress_b64(comp, buf, size, &out_buf, &out_size);
380   rv = noit_log(ls, &(c->whence), __FILE__, __LINE__,
381                 "B%c\t%lu.%03lu\t%s\t%s\t%s\t%s\t%d\t%.*s\n",
382                 use_compression ? '1' : '2',
383                 SECPART(&(c->whence)), MSECPART(&(c->whence)),
384                 uuid_str, check->target, check->module, check->name, size,
385                 (unsigned int)out_size, out_buf);
386
387   free(buf);
388   free(out_buf);
389   // Free all the resources
390   for (j=0; j<i; j++) {
391     free(bundle.metrics[j]);
392   }
393   free(bundle.metrics);
394   return rv;
395 }
396
397 void
398 noit_check_log_bundle(noit_check_t *check) {
399   handle_extra_feeds(check, noit_check_log_bundle_serialize);
400   if(!(check->flags & (NP_TRANSIENT | NP_SUPPRESS_STATUS | NP_SUPPRESS_METRICS))) {
401     SETUP_LOG(bundle, return);
402     noit_check_log_bundle_serialize(bundle_log, check);
403   }
404 }
405
406 void
407 noit_check_log_metric(noit_check_t *check, struct timeval *whence,
408                       metric_t *m) {
409   char uuid_str[256*3+37];
410   MAKE_CHECK_UUID_STR(uuid_str, sizeof(uuid_str), metrics_log, check);
411
412   /* handle feeds -- hust like handle_extra_feeds, but this
413    * is with different arguments.
414    */
415   if(check->feeds) {
416     noit_skiplist_node *curr, *next;
417     curr = next = noit_skiplist_getlist(check->feeds);
418     while(curr) {
419       const char *feed_name = (char *)curr->data;
420       noit_log_stream_t ls = noit_log_stream_find(feed_name);
421       noit_skiplist_next(check->feeds, &next);
422       if(!ls || _noit_check_log_metric(ls, check, uuid_str, whence, m))
423         noit_check_transient_remove_feed(check, feed_name);
424       curr = next;
425     }
426   }
427   if(!(check->flags & NP_TRANSIENT)) {
428     SETUP_LOG(metrics, return);
429     _noit_check_log_metric(metrics_log, check, uuid_str, whence, m);
430     if(NOIT_CHECK_METRIC_ENABLED()) {
431       char buff[256];
432       noit_stats_snprint_metric(buff, sizeof(buff), m);
433       NOIT_CHECK_METRIC(uuid_str, check->module, check->name, check->target,
434                         m->metric_name, m->metric_type, buff);
435     }
436   }
437 }
438
439 int
440 noit_stats_snprint_metric(char *b, int l, metric_t *m) {
441   int rv, nl;
442   nl = snprintf(b, l, "%s[%c] = ", m->metric_name, m->metric_type);
443   if(nl >= l || nl <= 0) return nl;
444   rv = noit_stats_snprint_metric_value(b+nl, l-nl, m);
445   if(rv == -1)
446     rv = snprintf(b+nl, l-nl, "[[unknown type]]");
447   return rv + nl;
448 }
449
Note: See TracBrowser for help on using the browser.