root/src/noit_check_tools.c

Revision 928ddde811578a89dff7e5c7efd86581462147c9, 13.1 kB (checked in by Phil Maddox <philip.maddox@circonus.com>, 3 days ago)

Write To Debug Log When Check Has 0ms To Complete

If we determine a check has 0ms to complete, the check has probably been
recycled. Log to the debug log when this happens, not the error log.

  • Property mode set to 100644
Line 
1 /*
2  * Copyright (c) 2007, OmniTI Computer Consulting, Inc.
3  * All rights reserved.
4  * Copyright (c) 2015, Circonus, Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are
8  * met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above
13  *       copyright notice, this list of conditions and the following
14  *       disclaimer in the documentation and/or other materials provided
15  *       with the distribution.
16  *     * Neither the name OmniTI Computer Consulting, Inc. nor the names
17  *       of its contributors may be used to endorse or promote products
18  *       derived from this software without specific prior written
19  *       permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <mtev_defines.h>
35 #include <mtev_str.h>
36 #include <mtev_json.h>
37
38 #include "noit_mtev_bridge.h"
39 #include "noit_dtrace_probes.h"
40 #include "noit_check_tools.h"
41 #include "noit_check_tools_shared.h"
42
43 MTEV_HOOK_IMPL(check_preflight,
44   (noit_module_t *self, noit_check_t *check, noit_check_t *cause),
45   void *, closure,
46   (void *closure, noit_module_t *self, noit_check_t *check, noit_check_t *cause),
47   (closure,self,check,cause))
48 MTEV_HOOK_IMPL(check_postflight,
49   (noit_module_t *self, noit_check_t *check, noit_check_t *cause),
50   void *, closure,
51   (void *closure, noit_module_t *self, noit_check_t *check, noit_check_t *cause),
52   (closure,self,check,cause))
53
54 typedef struct {
55   noit_module_t *self;
56   noit_check_t *check;
57   noit_check_t *cause;
58   dispatch_func_t dispatch;
59 } recur_closure_t;
60
61 static void
62 noit_check_recur_name_details(char *buf, int buflen,
63                               eventer_t e, void *closure) {
64   char id_str[UUID_STR_LEN+1];
65   recur_closure_t *rcl = e ? e->closure : NULL;
66   if(!e) {
67     snprintf(buf, buflen, "noit_check_recur_handler");
68     return;
69   }
70   uuid_unparse_lower(rcl->check->checkid, id_str);
71   snprintf(buf, buflen, "fire(%s)", id_str);
72   return;
73 }
74 static int
75 noit_check_recur_handler(eventer_t e, int mask, void *closure,
76                               struct timeval *now) {
77   recur_closure_t *rcl = closure;
78   int ms;
79
80   if(e != rcl->check->fire_event) return 0;
81
82   noit_check_resolve(rcl->check);
83   ms = noit_check_schedule_next(rcl->self, NULL, rcl->check, now,
84                                 rcl->dispatch, NULL);
85   if(ms == 0)
86     rcl->check->fire_event = NULL; /* This is us, we get free post-return */
87   if(NOIT_CHECK_RESOLVED(rcl->check)) {
88     if(MTEV_HOOK_CONTINUE ==
89        check_preflight_hook_invoke(rcl->self, rcl->check, rcl->cause)) {
90       if(NOIT_CHECK_DISPATCH_ENABLED()) {
91         char id[UUID_STR_LEN+1];
92         uuid_unparse_lower(rcl->check->checkid, id);
93         NOIT_CHECK_DISPATCH(id, rcl->check->module, rcl->check->name,
94                             rcl->check->target);
95       }
96       if(ms < rcl->check->timeout && !(rcl->check->flags & NP_TRANSIENT))
97         mtevL(((ms == 0) ? noit_debug : noit_error),
98               "%s might not finish in %dms (timeout %dms)\n",
99               rcl->check->name, ms, rcl->check->timeout);
100       rcl->dispatch(rcl->self, rcl->check, rcl->cause);
101     }
102     check_postflight_hook_invoke(rcl->self, rcl->check, rcl->cause);
103   }
104   else
105     mtevL(noit_debug, "skipping %s`%s`%s, unresolved\n",
106           rcl->check->target, rcl->check->module, rcl->check->name);
107   free(rcl);
108   return 0;
109 }
110
111 int
112 noit_check_schedule_next(noit_module_t *self,
113                          struct timeval *last_check, noit_check_t *check,
114                          struct timeval *now, dispatch_func_t dispatch,
115                          noit_check_t *cause) {
116   eventer_t newe;
117   struct timeval period, earliest, diff;
118   int64_t diffms, periodms, offsetms;
119   recur_closure_t *rcl;
120   int initial = last_check ? 1 : 0;
121
122   mtevAssert(cause == NULL);
123   if(check->period == 0) return 0;
124
125   /* if last_check is not passed, we use the initial_schedule_time
126    * otherwise, we set the initial_schedule_time
127    */
128   if(!last_check) last_check = &check->initial_schedule_time;
129   else memcpy(&check->initial_schedule_time, last_check, sizeof(*last_check));
130
131   if(NOIT_CHECK_DISABLED(check) || NOIT_CHECK_KILLED(check)) {
132     if(!(check->flags & NP_TRANSIENT)) check_slots_dec_tv(last_check);
133     memset(&check->initial_schedule_time, 0, sizeof(struct timeval));
134     return 0;
135   }
136
137   /* If we have an event, we know when we intended it to fire.  This means
138    * we should schedule that point + period.
139    */
140   if(now)
141     memcpy(&earliest, now, sizeof(earliest));
142   else
143     gettimeofday(&earliest, NULL);
144
145   /* If the check is unconfigured and needs resolving, we'll set the
146    * period down a bit lower so we can pick up the resolution quickly.
147    * The one exception is if this is the initial run.
148    */
149   if(!initial &&
150      !NOIT_CHECK_RESOLVED(check) && NOIT_CHECK_SHOULD_RESOLVE(check) &&
151      check->period > 1000) {
152     period.tv_sec = 1;
153     period.tv_usec = 0;
154   }
155   else {
156     period.tv_sec = check->period / 1000;
157     period.tv_usec = (check->period % 1000) * 1000;
158   }
159   periodms = period.tv_sec * 1000 + period.tv_usec / 1000;
160
161   newe = eventer_alloc();
162   /* calculate the differnet between the initial schedule time and "now" */
163   if(compare_timeval(earliest, *last_check) >= 0) {
164     sub_timeval(earliest, *last_check, &diff);
165     diffms = (int64_t)diff.tv_sec * 1000 + diff.tv_usec / 1000;
166   }
167   else {
168     mtevFatal(noit_error, "time is going backwards. abort.\n");
169   }
170   /* determine the offset from initial schedule time that would place
171    * us at the next period-aligned point past "now" */
172   offsetms = ((diffms / periodms) + 1) * periodms;
173   diff.tv_sec = offsetms / 1000;
174   diff.tv_usec = (offsetms % 1000) * 1000;
175
176   memcpy(&newe->whence, last_check, sizeof(*last_check));
177   add_timeval(newe->whence, diff, &newe->whence);
178
179   sub_timeval(newe->whence, earliest, &diff);
180   diffms = (int64_t)diff.tv_sec * 1000 + (int)diff.tv_usec / 1000;
181   mtevAssert(compare_timeval(newe->whence, earliest) > 0);
182   newe->mask = EVENTER_TIMER;
183   newe->callback = noit_check_recur_handler;
184   rcl = calloc(1, sizeof(*rcl));
185   rcl->self = self;
186   rcl->check = check;
187   rcl->cause = cause;
188   rcl->dispatch = dispatch;
189   newe->closure = rcl;
190
191   /* knuth's golden ratio approach */
192   if(!self->thread_unsafe) {
193     newe->thr_owner = CHOOSE_EVENTER_THREAD_FOR_CHECK(check);
194   }
195   check->fire_event = newe;
196   eventer_add(newe);
197   return diffms;
198 }
199
200 void
201 noit_check_run_full_asynch_opts(noit_check_t *check, eventer_func_t callback,
202                                 int mask) {
203   struct timeval __now, p_int;
204   eventer_t e;
205   e = eventer_alloc();
206   e->fd = -1;
207   e->mask = EVENTER_ASYNCH | mask;
208   gettimeofday(&__now, NULL);
209   memcpy(&e->whence, &__now, sizeof(__now));
210   p_int.tv_sec = check->timeout / 1000;
211   p_int.tv_usec = (check->timeout % 1000) * 1000;
212   add_timeval(e->whence, p_int, &e->whence);
213   e->callback = callback;
214   e->closure =  check->closure;
215   eventer_add(e);
216 }
217 void
218 noit_check_run_full_asynch(noit_check_t *check, eventer_func_t callback) {
219   noit_check_run_full_asynch_opts(check, callback,
220                                   EVENTER_DEFAULT_ASYNCH_ABORT);
221 }
222
223 void
224 noit_check_tools_init() {
225   noit_check_tools_shared_init();
226   eventer_name_callback_ext("noit_check_recur_handler",
227                             noit_check_recur_handler,
228                             noit_check_recur_name_details, NULL);
229 }
230
231 static int
232 populate_stats_from_resmon_formatted_json(noit_check_t *check,
233                                           struct json_object *o,
234                                           const char *prefix) {
235   int count = 0;
236   char keybuff[256];
237 #define MKKEY(fmt, arg) do { \
238   if(prefix) snprintf(keybuff, sizeof(keybuff), "%s`" fmt, prefix, arg); \
239   else snprintf(keybuff, sizeof(keybuff), fmt, arg); \
240 } while(0)
241   if(o == NULL) {
242     if(prefix) {
243       noit_stats_set_metric(check, prefix, METRIC_STRING, NULL);
244       count++;
245     }
246     return count;
247   }
248   switch(json_object_get_type(o)) {
249     /* sub callers */
250     case json_type_array:
251     {
252       int i, alen = json_object_array_length(o);
253       for(i=0;i<alen;i++) {
254         struct json_object *item = json_object_array_get_idx(o, i);
255         MKKEY("%d", i);
256         count += populate_stats_from_resmon_formatted_json(check, item, keybuff);
257       }
258     }
259     break;
260     case json_type_object:
261     {
262       struct jl_lh_table *lh;
263       struct jl_lh_entry *el;
264       struct json_object *has_type = NULL, *has_value = NULL;
265       lh = json_object_get_object(o);
266       jl_lh_foreach(lh, el) {
267         if(!strcmp(el->k, "_type")) has_type = (struct json_object *)el->v;
268         else if(!strcmp(el->k, "_value")) has_value = (struct json_object *)el->v;
269         else {
270           struct json_object *item = (struct json_object *)el->v;
271           MKKEY("%s", (const char *)el->k);
272           count += populate_stats_from_resmon_formatted_json(check, item, keybuff);
273         }
274       }
275       if(prefix && has_type &&
276          json_object_is_type(has_type, json_type_string)) {
277         const char *type_str = json_object_get_string(has_type);
278
279 #define COERCE_JSON_OBJECT(type, item) do { \
280   const char *value_str = NULL; \
281   if(json_object_is_type(item, json_type_string)) \
282     value_str = json_object_get_string(item); \
283   else if(!json_object_is_type(item, json_type_null)) \
284     value_str = json_object_to_json_string(item); \
285   switch(type) { \
286     case METRIC_INT32: case METRIC_UINT32: case METRIC_INT64: \
287     case METRIC_UINT64: case METRIC_DOUBLE: case METRIC_STRING: \
288       noit_stats_set_metric_coerce(check, prefix, \
289                                    (metric_type_t)type, value_str); \
290       count++; \
291     default: \
292       break; \
293   } \
294 } while(0)
295
296
297         if (has_value == NULL) {
298           noit_stats_set_metric_coerce(check, prefix, (metric_type_t)*type_str, NULL);
299           count++;
300         }
301         else if(json_object_is_type(has_value, json_type_array)) {
302           int i, alen = json_object_array_length(has_value);
303           for(i=0;i<alen;i++) {
304             struct json_object *item = json_object_array_get_idx(has_value, i);
305             COERCE_JSON_OBJECT(*type_str, item);
306           }
307         }
308         else {
309           COERCE_JSON_OBJECT(*type_str, has_value);
310         }
311       }
312       break;
313     }
314
315     /* directs */
316     case json_type_string:
317       if(prefix) {
318         noit_stats_set_metric(check, prefix, METRIC_GUESS,
319                               (char *)json_object_get_string(o));
320         count++;
321       }
322       break;
323     case json_type_boolean:
324       if(prefix) {
325         int val = json_object_get_boolean(o) ? 1 : 0;
326         noit_stats_set_metric(check, prefix, METRIC_INT32, &val);
327         count++;
328       }
329       break;
330     case json_type_null:
331       if(prefix) {
332         noit_stats_set_metric(check, prefix, METRIC_STRING, NULL);
333         count++;
334       }
335       break;
336     case json_type_double:
337       if(prefix) {
338         double val = json_object_get_double(o);
339         noit_stats_set_metric(check, prefix, METRIC_DOUBLE, &val);
340         count++;
341       }
342       break;
343     case json_type_int:
344       if(prefix) {
345         int64_t i64;
346         uint64_t u64;
347         switch(json_object_get_int_overflow(o)) {
348           case json_overflow_int:
349             i64 = json_object_get_int(o);
350             noit_stats_set_metric(check, prefix, METRIC_INT64, &i64);
351             count++;
352             break;
353           case json_overflow_int64:
354             i64 = json_object_get_int64(o);
355             noit_stats_set_metric(check, prefix, METRIC_INT64, &i64);
356             count++;
357             break;
358           case json_overflow_uint64:
359             u64 = json_object_get_uint64(o);
360             noit_stats_set_metric(check, prefix, METRIC_UINT64, &u64);
361             count++;
362             break;
363         }
364       }
365   }
366   return count;
367 }
368 int
369 noit_check_stats_from_json_str(noit_check_t *check,
370                                const char *json_str, int len) {
371   int rv = -1;
372   struct json_tokener *tok = NULL;
373   struct json_object *root = NULL;
374   tok = json_tokener_new();
375   root = json_tokener_parse_ex(tok, json_str, len);
376   if(root) rv = populate_stats_from_resmon_formatted_json(check, root, NULL);
377   if(tok) json_tokener_free(tok);
378   if(root) json_object_put(root);
379   return rv;
380 }
381
382 void
383 noit_check_make_attrs(noit_check_t *check, mtev_hash_table *attrs) {
384 #define CA_STORE(a,b) mtev_hash_store(attrs, a, strlen(a), b)
385   CA_STORE("target", check->target);
386   CA_STORE("target_ip", check->target_ip);
387   CA_STORE("name", check->name);
388   CA_STORE("module", check->module);
389 }
Note: See TracBrowser for help on using the browser.