root/src/noit_check.c

Revision a8796b087a3ac48076015ed85cad06bff8ea92dc, 70.5 kB (checked in by Phil Maddox <philip.maddox@circonus.com>, 1 month ago)

Reduce Amount Of mtev_memory_safe Stuff Needed For Checks

Change the metric_t data to be malloced with a cleanup callback and make
the internal parts of the metric_t get allocated with regular memory
management, not mtev_memory_safe memory management.

  • Property mode set to 100644
Line 
1 /*
2  * Copyright (c) 2007, OmniTI Computer Consulting, Inc.
3  * All rights reserved.
4  * Copyright (c) 2015, Circonus, Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are
8  * met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above
13  *       copyright notice, this list of conditions and the following
14  *       disclaimer in the documentation and/or other materials provided
15  *       with the distribution.
16  *     * Neither the name OmniTI Computer Consulting, Inc. nor the names
17  *       of its contributors may be used to endorse or promote products
18  *       derived from this software without specific prior written
19  *       permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include "noit_config.h"
35 #include <mtev_defines.h>
36
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <unistd.h>
40 #include <ctype.h>
41 #include <assert.h>
42 #include <errno.h>
43 #include <netinet/in.h>
44 #include <arpa/inet.h>
45 #include <time.h>
46
47 #include <eventer/eventer.h>
48 #include <mtev_memory.h>
49 #include <mtev_log.h>
50 #include <mtev_hash.h>
51 #include <mtev_skiplist.h>
52 #include <mtev_watchdog.h>
53 #include <mtev_conf.h>
54 #include <mtev_console.h>
55 #include <mtev_cluster.h>
56
57 #include "noit_mtev_bridge.h"
58 #include "noit_dtrace_probes.h"
59 #include "noit_check.h"
60 #include "noit_module.h"
61 #include "noit_check_tools.h"
62 #include "noit_check_resolver.h"
63
64 #define DEFAULT_TEXT_METRIC_SIZE_LIMIT  512
65 #define RECYCLE_INTERVAL 60
66
67 MTEV_HOOK_IMPL(check_config_fixup,
68   (noit_check_t *check),
69   void *, closure,
70   (void *closure, noit_check_t *check),
71   (closure,check))
72
73 MTEV_HOOK_IMPL(check_stats_set_metric,
74   (noit_check_t *check, stats_t *stats, metric_t *m),
75   void *, closure,
76   (void *closure, noit_check_t *check, stats_t *stats, metric_t *m),
77   (closure,check,stats,m))
78
79 MTEV_HOOK_IMPL(check_stats_set_metric_coerce,
80   (noit_check_t *check, stats_t *stats, const char *name,
81    metric_type_t type, const char *v, mtev_boolean success),
82   void *, closure,
83   (void *closure, noit_check_t *check, stats_t *stats, const char *name,
84    metric_type_t type, const char *v, mtev_boolean success),
85   (closure,check,stats,name,type,v,success))
86
87 MTEV_HOOK_IMPL(check_passive_log_stats,
88   (noit_check_t *check),
89   void *, closure,
90   (void *closure, noit_check_t *check),
91   (closure,check))
92
93 MTEV_HOOK_IMPL(check_log_stats,
94   (noit_check_t *check),
95   void *, closure,
96   (void *closure, noit_check_t *check),
97   (closure,check))
98
99 #define STATS_INPROGRESS 0
100 #define STATS_CURRENT 1
101 #define STATS_PREVIOUS 2
102
103 void
104 free_metric(metric_t *m) {
105   if(m->metric_name) free(m->metric_name);
106   if(m->metric_value.i) free(m->metric_value.i);
107 }
108
109 #define stats_inprogress(c) ((stats_t **)(c->statistics))[STATS_INPROGRESS]
110 #define stats_current(c) ((stats_t **)(c->statistics))[STATS_CURRENT]
111 #define stats_previous(c) ((stats_t **)(c->statistics))[STATS_PREVIOUS]
112
113 stats_t *
114 noit_check_get_stats_inprogress(noit_check_t *c) {
115   return stats_inprogress(c);
116 }
117 stats_t *
118 noit_check_get_stats_current(noit_check_t *c) {
119   return stats_current(c);
120 }
121 stats_t *
122 noit_check_get_stats_previous(noit_check_t *c) {
123   return stats_previous(c);
124 }
125
126 struct stats_t {
127   struct timeval whence;
128   int8_t available;
129   int8_t state;
130   u_int32_t duration;
131   mtev_hash_table metrics;
132   char status[256];
133 };
134
135 struct timeval *
136 noit_check_stats_whence(stats_t *s, struct timeval *n) {
137   if(n) memcpy(&s->whence, n, sizeof(*n));
138   return &s->whence;
139 }
140 int8_t
141 noit_check_stats_available(stats_t *s, int8_t *n) {
142   if(n) s->available = *n;
143   return s->available;
144 }
145 int8_t
146 noit_check_stats_state(stats_t *s, int8_t *n) {
147   if(n) s->state = *n;
148   return s->state;
149 }
150 u_int32_t
151 noit_check_stats_duration(stats_t *s, u_int32_t *n) {
152   if(n) s->duration = *n;
153   return s->duration;
154 }
155 const char *
156 noit_check_stats_status(stats_t *s, const char *n) {
157   if(n) strlcpy(s->status, n, sizeof(s->status));
158   return s->status;
159 }
160 mtev_hash_table *
161 noit_check_stats_metrics(stats_t *s) {
162   return &s->metrics;
163 }
164 void
165 noit_stats_set_whence(noit_check_t *c, struct timeval *t) {
166   (void)noit_check_stats_whence(noit_check_get_stats_inprogress(c), t);
167 }
168 void
169 noit_stats_set_state(noit_check_t *c, int8_t t) {
170   (void)noit_check_stats_state(noit_check_get_stats_inprogress(c), &t);
171 }
172 void
173 noit_stats_set_duration(noit_check_t *c, u_int32_t t) {
174   (void)noit_check_stats_duration(noit_check_get_stats_inprogress(c), &t);
175 }
176 void
177 noit_stats_set_status(noit_check_t *c, const char *s) {
178   (void)noit_check_stats_status(noit_check_get_stats_inprogress(c), s);
179 }
180 void
181 noit_stats_set_available(noit_check_t *c, int8_t t) {
182   (void)noit_check_stats_available(noit_check_get_stats_inprogress(c), &t);
183 }
184 static void
185 noit_check_safe_free_metric(void *vs) {
186   metric_t *m = vs;
187   if (m) {
188     free_metric(m);
189   }
190 }
191 static void
192 noit_check_safe_free_stats(void *vs) {
193   stats_t *s = vs;
194   mtev_hash_destroy(&s->metrics, NULL, (void (*)(void *))mtev_memory_safe_free);
195 }
196 static stats_t *
197 noit_check_stats_alloc() {
198   stats_t *n;
199   n = mtev_memory_safe_malloc_cleanup(sizeof(*n), noit_check_safe_free_stats);
200   memset(n, 0, sizeof(*n));
201   mtev_hash_init(&n->metrics);
202   return n;
203 }
204 static void *
205 noit_check_stats_set_calloc() {
206   int i;
207   stats_t **s;
208   s = calloc(sizeof(stats_t *), 3);
209   for(i=0;i<3;i++) s[i] = noit_check_stats_alloc();
210   return s;
211 }
212
213 /* 20 ms slots over 60 second for distribution */
214 #define SCHEDULE_GRANULARITY 20
215 #define SLOTS_PER_SECOND (1000/SCHEDULE_GRANULARITY)
216 #define MAX_MODULE_REGISTRATIONS 64
217
218 /* used to manage per-check generic module metadata */
219 struct vp_w_free {
220   void *ptr;
221   void (*freefunc)(void *);
222 };
223
224 static mtev_boolean system_needs_causality = mtev_false;
225 static int text_size_limit = DEFAULT_TEXT_METRIC_SIZE_LIMIT;
226 static int reg_module_id = 0;
227 static char *reg_module_names[MAX_MODULE_REGISTRATIONS] = { NULL };
228 static int reg_module_used = -1;
229 static u_int64_t check_completion_count = 0ULL;
230 static u_int64_t check_metrics_seen = 0ULL;
231 static pthread_mutex_t polls_lock = PTHREAD_MUTEX_INITIALIZER;
232 static mtev_hash_table polls = MTEV_HASH_EMPTY;
233 static mtev_hash_table dns_ignore_list = MTEV_HASH_EMPTY;
234 static mtev_skiplist watchlist = { 0 };
235 static mtev_skiplist polls_by_name = { 0 };
236 static u_int32_t __config_load_generation = 0;
237 static unsigned short check_slots_count[60000 / SCHEDULE_GRANULARITY] = { 0 },
238                       check_slots_seconds_count[60] = { 0 };
239
240 static noit_check_t *
241 noit_poller_lookup__nolock(uuid_t in) {
242   void *vcheck;
243   if(mtev_hash_retrieve(&polls, (char *)in, UUID_SIZE, &vcheck))
244     return (noit_check_t *)vcheck;
245   return NULL;
246 }
247 static noit_check_t *
248 noit_poller_lookup_by_name__nolock(char *target, char *name) {
249   noit_check_t tmp_check;
250   memset(&tmp_check, 0, sizeof(tmp_check));
251   tmp_check.target = target;
252   tmp_check.name = name;
253   return mtev_skiplist_find(&polls_by_name, &tmp_check, NULL);
254 }
255
256 static int
257 noit_console_show_timing_slots(mtev_console_closure_t ncct,
258                                int argc, char **argv,
259                                mtev_console_state_t *dstate,
260                                void *closure) {
261   int i, j;
262   const int upl = (60000 / SCHEDULE_GRANULARITY) / 60;
263   for(i=0;i<60;i++) {
264     nc_printf(ncct, "[%02d] %04d: ", i, check_slots_seconds_count[i]);
265     for(j=i*upl;j<(i+1)*upl;j++) {
266       char cp = '!';
267       if(check_slots_count[j] < 10) cp = '0' + check_slots_count[j];
268       else if(check_slots_count[j] < 36) cp = 'a' + (check_slots_count[j] - 10);
269       nc_printf(ncct, "%c", cp);
270     }
271     nc_printf(ncct, "\n");
272   }
273   return 0;
274 }
275 static int
276 noit_check_add_to_list(noit_check_t *new_check, const char *newname) {
277   char *oldname = NULL, *newnamecopy;
278   if(newname) {
279     /* track this stuff outside the lock to avoid allocs */
280     oldname = new_check->name;
281     newnamecopy = strdup(newname);
282   }
283   pthread_mutex_lock(&polls_lock);
284   if(!(new_check->flags & NP_TRANSIENT)) {
285     assert(new_check->name || newname);
286     /* This remove could fail -- no big deal */
287     if(new_check->name != NULL)
288       mtev_skiplist_remove(&polls_by_name, new_check, NULL);
289
290     /* optional update the name (at the critical point) */
291     if(newname) new_check->name = newnamecopy;
292
293     /* This insert could fail.. which means we have a conflict on
294      * target`name.  That should result in the check being disabled. */
295     if(!mtev_skiplist_insert(&polls_by_name, new_check)) {
296       mtevL(noit_error, "Check %s`%s disabled due to naming conflict\n",
297             new_check->target, new_check->name);
298       new_check->flags |= NP_DISABLED;
299     }
300     if(oldname) free(oldname);
301   }
302   pthread_mutex_unlock(&polls_lock);
303   return 1;
304 }
305
306 u_int64_t noit_check_metric_count() {
307   return check_metrics_seen;
308 }
309 void noit_check_metric_count_add(int add) {
310   mtev_atomic64_t *n = (mtev_atomic64_t *)&check_metrics_seen;
311   mtev_atomic64_t v = (mtev_atomic64_t)add;
312   mtev_atomic_add64(n, v);
313 }
314
315 u_int64_t noit_check_completion_count() {
316   return check_completion_count;
317 }
318 static void register_console_check_commands();
319 static int check_recycle_bin_processor(eventer_t, int, void *,
320                                        struct timeval *);
321
322 static int
323 check_slots_find_smallest(int sec, struct timeval* period) {
324   int i, j, cyclic, random_offset, jbase = 0, mini = 0, minj = 0;
325   unsigned short min_running_i = 0xffff, min_running_j = 0xffff;
326   int period_seconds = period->tv_sec;
327
328   /* If we're greater than sixty seconds, we should do our
329    * initial scheduling as if the period was sixty seconds. */
330   if (period_seconds > 60)
331     period_seconds = 60;
332
333   for(i=0;i<period_seconds;i++) {
334     int adj_i = (i + sec) % 60;
335     if(check_slots_seconds_count[adj_i] < min_running_i) {
336       min_running_i = check_slots_seconds_count[adj_i];
337       mini = adj_i;
338     }
339   }
340   jbase = mini * (1000/SCHEDULE_GRANULARITY);
341   random_offset = drand48() * SLOTS_PER_SECOND;
342   for(cyclic=0;cyclic<SLOTS_PER_SECOND;cyclic++) {
343     j = jbase + ((random_offset + cyclic) % SLOTS_PER_SECOND);
344     if(check_slots_count[j] < min_running_j) {
345       min_running_j = check_slots_count[j];
346       minj = j;
347     }
348   }
349   return (minj * SCHEDULE_GRANULARITY) + drand48() * SCHEDULE_GRANULARITY;
350 }
351 static void
352 check_slots_adjust_tv(struct timeval *tv, short adj) {
353   int offset_ms, idx;
354   offset_ms = (tv->tv_sec % 60) * 1000 + (tv->tv_usec / 1000);
355   idx = offset_ms / SCHEDULE_GRANULARITY;
356   check_slots_count[idx] += adj;
357   check_slots_seconds_count[offset_ms / 1000] += adj;
358 }
359 void check_slots_inc_tv(struct timeval *tv) {
360   check_slots_adjust_tv(tv, 1);
361 }
362 void check_slots_dec_tv(struct timeval *tv) {
363   check_slots_adjust_tv(tv, -1);
364 }
365 static int
366 noit_check_generic_safe_string(const char *p) {
367   if(!p) return 0;
368   for(;*p;p++) {
369     if(!isprint(*p)) return 0;
370   }
371   return 1;
372 }
373 int
374 noit_check_validate_target(const char *p) {
375   if(!noit_check_generic_safe_string(p)) return 0;
376   return 1;
377 }
378 int
379 noit_check_validate_name(const char *p) {
380   if(!noit_check_generic_safe_string(p)) return 0;
381   return 1;
382 }
383 const char *
384 noit_check_available_string(int16_t available) {
385   switch(available) {
386     case NP_AVAILABLE:    return "available";
387     case NP_UNAVAILABLE:  return "unavailable";
388     case NP_UNKNOWN:      return "unknown";
389   }
390   return NULL;
391 }
392 const char *
393 noit_check_state_string(int16_t state) {
394   switch(state) {
395     case NP_GOOD:         return "good";
396     case NP_BAD:          return "bad";
397     case NP_UNKNOWN:      return "unknown";
398   }
399   return NULL;
400 }
401 static int __check_name_compare(const void *a, const void *b) {
402   const noit_check_t *ac = a;
403   const noit_check_t *bc = b;
404   int rv;
405   if((rv = strcmp(ac->target, bc->target)) != 0) return rv;
406   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
407   return 0;
408 }
409 static int __watchlist_compare(const void *a, const void *b) {
410   const noit_check_t *ac = a;
411   const noit_check_t *bc = b;
412   int rv;
413   if((rv = memcmp(ac->checkid, bc->checkid, sizeof(ac->checkid))) != 0) return rv;
414   if(ac->period < bc->period) return -1;
415   if(ac->period == bc->period) return 0;
416   return 1;
417 }
418 static int __check_target_ip_compare(const void *a, const void *b) {
419   const noit_check_t *ac = a;
420   const noit_check_t *bc = b;
421   int rv;
422   if((rv = strcmp(ac->target_ip, bc->target_ip)) != 0) return rv;
423   if (ac->name == NULL) return 1;
424   if (bc->name == NULL) return -1;
425   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
426   return 1;
427 }
428 static int __check_target_compare(const void *a, const void *b) {
429   const noit_check_t *ac = a;
430   const noit_check_t *bc = b;
431   int rv;
432   if (ac->target == NULL) return 1;
433   if (bc->target == NULL) return -1;
434   if((rv = strcmp(ac->target, bc->target)) != 0) return rv;
435   if (ac->name == NULL) return 1;
436   if (bc->name == NULL) return -1;
437   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
438   return 1;
439 }
440 int
441 noit_calc_rtype_flag(char *resolve_rtype) {
442   int flags = 0;
443   if(resolve_rtype) {
444     flags |= strcmp(resolve_rtype, PREFER_IPV6) == 0 ||
445              strcmp(resolve_rtype, FORCE_IPV6) == 0 ? NP_PREFER_IPV6 : 0;
446     flags |= strcmp(resolve_rtype, FORCE_IPV4) == 0 ||
447              strcmp(resolve_rtype, FORCE_IPV6) == 0 ? NP_SINGLE_RESOLVE : 0;
448   }
449   return flags;
450 }
451 void
452 noit_check_fake_last_check(noit_check_t *check,
453                            struct timeval *lc, struct timeval *_now) {
454   struct timeval now, period, lc_copy;
455   int balance_ms;
456
457   if(!_now) {
458     gettimeofday(&now, NULL);
459     _now = &now;
460   }
461   period.tv_sec = check->period / 1000;
462   period.tv_usec = (check->period % 1000) * 1000;
463   sub_timeval(*_now, period, lc);
464
465   /* We need to set the last check value based on the period, but
466    * we also need to store a value that is based around the one-minute
467    * time to properly increment the slots; otherwise, the slots will
468    * get all messed up */
469   if(!(check->flags & NP_TRANSIENT) && check->period) {
470     balance_ms = check_slots_find_smallest(_now->tv_sec+1, &period);
471     lc->tv_sec = (lc->tv_sec / 60) * 60 + balance_ms / 1000;
472     lc->tv_usec = (balance_ms % 1000) * 1000;
473     memcpy(&lc_copy, lc, sizeof(lc_copy));
474     if(compare_timeval(*_now, *lc) < 0) {
475       do {
476         sub_timeval(*lc, period, lc);
477       } while(compare_timeval(*_now, *lc) < 0);
478     }
479     else {
480       struct timeval test;
481       while(1) {
482         add_timeval(*lc, period, &test);
483         if(compare_timeval(*_now, test) < 0) break;
484         memcpy(lc, &test, sizeof(test));
485       }
486     }
487   }
488   else {
489     memcpy(&lc_copy, lc, sizeof(lc_copy));
490   }
491  
492   /* now, we're going to do an even distribution using the slots */
493   if(!(check->flags & NP_TRANSIENT)) check_slots_inc_tv(&lc_copy);
494 }
495 void
496 noit_poller_process_checks(const char *xpath) {
497   int i, flags, cnt = 0, found;
498   mtev_conf_section_t *sec;
499   __config_load_generation++;
500   sec = mtev_conf_get_sections(NULL, xpath, &cnt);
501   for(i=0; i<cnt; i++) {
502     void *vcheck;
503     char uuid_str[37];
504     char target[256] = "";
505     char module[256] = "";
506     char name[256] = "";
507     char filterset[256] = "";
508     char oncheck[1024] = "";
509     char resolve_rtype[16] = "";
510     int ridx;
511     int no_period = 0;
512     int no_oncheck = 0;
513     int period = 0, timeout = 0;
514     mtev_boolean disabled = mtev_false, busted = mtev_false;
515     uuid_t uuid, out_uuid;
516     int64_t config_seq = 0;
517     mtev_hash_table *options;
518     mtev_hash_table **moptions = NULL;
519     mtev_boolean moptions_used = mtev_false, backdated = mtev_false;
520
521     /* We want to heartbeat here... otherwise, if a lot of checks are
522      * configured or if we're running on a slower system, we could
523      * end up getting watchdog killed before we get a chance to run
524      * any checks */
525     mtev_watchdog_child_heartbeat();
526
527     if(reg_module_id > 0) {
528       moptions = alloca(reg_module_id * sizeof(mtev_hash_table *));
529       memset(moptions, 0, reg_module_id * sizeof(mtev_hash_table *));
530       moptions_used = mtev_true;
531     }
532
533 #define NEXT(...) mtevL(noit_stderr, __VA_ARGS__); continue
534 #define MYATTR(type,a,...) mtev_conf_get_##type(sec[i], "@" #a, __VA_ARGS__)
535 #define INHERIT(type,a,...) \
536   mtev_conf_get_##type(sec[i], "ancestor-or-self::node()/@" #a, __VA_ARGS__)
537
538     if(!MYATTR(stringbuf, uuid, uuid_str, sizeof(uuid_str))) {
539       mtevL(noit_stderr, "check %d has no uuid\n", i+1);
540       continue;
541     }
542
543     MYATTR(int64, seq, &config_seq);
544
545     if(uuid_parse(uuid_str, uuid)) {
546       mtevL(noit_stderr, "check uuid: '%s' is invalid\n", uuid_str);
547       continue;
548     }
549
550     if(!INHERIT(stringbuf, target, target, sizeof(target))) {
551       mtevL(noit_stderr, "check uuid: '%s' has no target\n", uuid_str);
552       busted = mtev_true;
553     }
554     if(!noit_check_validate_target(target)) {
555       mtevL(noit_stderr, "check uuid: '%s' has malformed target\n", uuid_str);
556       busted = mtev_true;
557     }
558     if(!INHERIT(stringbuf, module, module, sizeof(module))) {
559       mtevL(noit_stderr, "check uuid: '%s' has no module\n", uuid_str);
560       busted = mtev_true;
561     }
562
563     if(!INHERIT(stringbuf, filterset, filterset, sizeof(filterset)))
564       filterset[0] = '\0';
565    
566     if (!INHERIT(stringbuf, resolve_rtype, resolve_rtype, sizeof(resolve_rtype)))
567       strlcpy(resolve_rtype, PREFER_IPV4, sizeof(resolve_rtype));
568
569     if(!MYATTR(stringbuf, name, name, sizeof(name)))
570       strlcpy(name, module, sizeof(name));
571
572     if(!noit_check_validate_name(name)) {
573       mtevL(noit_stderr, "check uuid: '%s' has malformed name\n", uuid_str);
574       busted = mtev_true;
575     }
576
577     if(!INHERIT(int, period, &period) || period == 0)
578       no_period = 1;
579
580     if(!INHERIT(stringbuf, oncheck, oncheck, sizeof(oncheck)) || !oncheck[0])
581       no_oncheck = 1;
582
583     if(no_period && no_oncheck) {
584       mtevL(noit_stderr, "check uuid: '%s' has neither period nor oncheck\n",
585             uuid_str);
586       busted = mtev_true;
587     }
588     if(!(no_period || no_oncheck)) {
589       mtevL(noit_stderr, "check uuid: '%s' has oncheck and period.\n",
590             uuid_str);
591       busted = mtev_true;
592     }
593     if(!INHERIT(int, timeout, &timeout)) {
594       mtevL(noit_stderr, "check uuid: '%s' has no timeout\n", uuid_str);
595       busted = mtev_true;
596     }
597     if(!no_period && timeout >= period) {
598       mtevL(noit_stderr, "check uuid: '%s' timeout > period\n", uuid_str);
599       timeout = period/2;
600     }
601     options = mtev_conf_get_hash(sec[i], "config");
602     for(ridx=0; ridx<reg_module_id; ridx++) {
603       moptions[ridx] = mtev_conf_get_namespaced_hash(sec[i], "config",
604                                                      reg_module_names[ridx]);
605     }
606
607     INHERIT(boolean, disable, &disabled);
608     flags = 0;
609     if(busted) flags |= (NP_UNCONFIG|NP_DISABLED);
610     else if(disabled) flags |= NP_DISABLED;
611
612     flags |= noit_calc_rtype_flag(resolve_rtype);
613
614     pthread_mutex_lock(&polls_lock);
615     found = mtev_hash_retrieve(&polls, (char *)uuid, UUID_SIZE, &vcheck);
616     if(found) {
617       noit_check_t *check = (noit_check_t *)vcheck;
618       /* Possibly reset the seq */
619       if(config_seq < 0) check->config_seq = 0;
620
621       /* Otherwise note a non-increasing sequence */
622       if(check->config_seq > config_seq) backdated = mtev_true;
623     }
624     pthread_mutex_unlock(&polls_lock);
625     if(found)
626       noit_poller_deschedule(uuid);
627     if(backdated) {
628       mtevL(noit_error, "Check config seq backwards, ignored\n");
629     }
630     else {
631       noit_poller_schedule(target, module, name, filterset, options,
632                            moptions_used ? moptions : NULL,
633                            period, timeout, oncheck[0] ? oncheck : NULL,
634                            config_seq, flags, uuid, out_uuid);
635       mtevL(noit_debug, "loaded uuid: %s\n", uuid_str);
636     }
637     for(ridx=0; ridx<reg_module_id; ridx++) {
638       if(moptions[ridx]) {
639         mtev_hash_destroy(moptions[ridx], free, free);
640         free(moptions[ridx]);
641       }
642     }
643     mtev_hash_destroy(options, free, free);
644     free(options);
645   }
646   if(sec) free(sec);
647 }
648
649 int
650 noit_check_activate(noit_check_t *check) {
651   noit_module_t *mod;
652   if(NOIT_CHECK_LIVE(check)) return 0;
653   mod = noit_module_lookup(check->module);
654   if(mod && mod->initiate_check) {
655     if((check->flags & NP_DISABLED) == 0) {
656       mod->initiate_check(mod, check, 0, NULL);
657       return 1;
658     }
659     else
660       mtevL(noit_debug, "Skipping %s`%s, disabled.\n",
661             check->target, check->name);
662   }
663   else {
664     if(!mod) {
665       mtevL(noit_stderr, "Cannot find module '%s'\n", check->module);
666       check->flags |= NP_DISABLED;
667     }
668   }
669   return 0;
670 }
671
672 void
673 noit_poller_initiate() {
674   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
675   uuid_t key_id;
676   int klen;
677   void *vcheck;
678   /* This is only ever called in the beginning, no lock needed */
679   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
680                        &vcheck)) {
681     noit_check_activate((noit_check_t *)vcheck);
682     mtev_watchdog_child_heartbeat();
683   }
684 }
685
686 void
687 noit_poller_flush_epoch(int oldest_allowed) {
688   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
689   uuid_t key_id;
690   int klen, i;
691   void *vcheck;
692 #define TOFREE_PER_ITER 1024
693   noit_check_t *tofree[TOFREE_PER_ITER];
694
695   /* Cleanup any previous causal map */
696   while(1) {
697     i = 0;
698     pthread_mutex_lock(&polls_lock);
699     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
700                          &vcheck) && i < TOFREE_PER_ITER) {
701       noit_check_t *check = (noit_check_t *)vcheck;
702       if(check->generation < oldest_allowed) {
703         tofree[i++] = check;
704       }
705     }
706     pthread_mutex_unlock(&polls_lock);
707     if(i==0) break;
708     while(i>0) noit_poller_deschedule(tofree[--i]->checkid);
709   }
710 #undef TOFREE_PER_ITER
711 }
712
713 void
714 noit_poller_make_causal_map() {
715   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
716   uuid_t key_id;
717   int klen;
718   void *vcheck;
719
720   if(!system_needs_causality) return;
721
722   /* set it to false, we'll set it to true during the scan if we
723    * find anything causal.  */
724   system_needs_causality = mtev_false;
725
726   /* Cleanup any previous causal map */
727   pthread_mutex_lock(&polls_lock);
728   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
729                        &vcheck)) {
730     noit_check_t *check = (noit_check_t *)vcheck;
731     dep_list_t *dep;
732     while((dep = check->causal_checks) != NULL) {
733       check->causal_checks = dep->next;
734       free(dep);
735     }
736   }
737
738   memset(&iter, 0, sizeof(iter));
739   /* Walk all checks and add check dependencies to their parents */
740   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
741                        &vcheck)) {
742     noit_check_t *check = (noit_check_t *)vcheck, *parent;
743     if(check->oncheck) {
744       /* This service is causally triggered by another service */
745       uuid_t id;
746       char fullcheck[1024];
747       char *name = check->oncheck;
748       char *target = NULL;
749
750       system_needs_causality = mtev_true;
751       mtevL(noit_debug, "Searching for upstream trigger on %s\n", name);
752       parent = NULL;
753       if(uuid_parse(check->oncheck, id) == 0) {
754         target = "";
755         parent = noit_poller_lookup__nolock(id);
756       }
757       else if((target = strchr(check->oncheck, '`')) != NULL) {
758         strlcpy(fullcheck, check->oncheck, target + 1 - check->oncheck);
759         name = target + 1;
760         target = fullcheck;
761         parent = noit_poller_lookup_by_name__nolock(target, name);
762       }
763       else {
764         target = check->target;
765         parent = noit_poller_lookup_by_name__nolock(target, name);
766       }
767
768       if(!parent) {
769         check->flags |= NP_DISABLED;
770         mtevL(noit_stderr, "Disabling check %s`%s, can't find oncheck %s`%s\n",
771               check->target, check->name, target, name);
772       }
773       else {
774         dep_list_t *dep;
775         dep = malloc(sizeof(*dep));
776         dep->check = check;
777         dep->next = parent->causal_checks;
778         parent->causal_checks = dep;
779         mtevL(noit_debug, "Causal map %s`%s --> %s`%s\n",
780               parent->target, parent->name, check->target, check->name);
781       }
782     }
783   }
784   pthread_mutex_unlock(&polls_lock);
785   /* We found some causal checks, so we might need to activate stuff */
786   if(system_needs_causality) noit_poller_initiate();
787 }
788 void
789 noit_poller_reload(const char *xpath)
790 {
791   noit_poller_process_checks(xpath ? xpath : "/noit/checks//check");
792   if(!xpath) {
793     /* Full reload, we need to wipe old checks */
794     noit_poller_flush_epoch(__config_load_generation);
795   }
796   noit_poller_make_causal_map();
797 }
798 void
799 noit_check_dns_ignore_tld(const char* extension, const char* ignore) {
800   mtev_hash_replace(&dns_ignore_list, strdup(extension), strlen(extension), strdup(ignore), NULL, NULL);
801 }
802 static void
803 noit_check_dns_ignore_list_init() {
804   mtev_conf_section_t* dns;
805   int cnt;
806
807   dns = mtev_conf_get_sections(NULL, "/noit/dns/extension", &cnt);
808   if(dns) {
809     int i = 0;
810     for (i = 0; i < cnt; i++) {
811       char* extension;
812       char* ignore;
813       if(!mtev_conf_get_string(dns[i], "self::node()/@value", &extension)) {
814         continue;
815       }
816       if(!mtev_conf_get_string(dns[i], "self::node()/@ignore", &ignore)) {
817         continue;
818       }
819       noit_check_dns_ignore_tld(extension, ignore);
820     }
821   }
822 }
823 void
824 noit_poller_init() {
825   srand48((getpid() << 16) ^ time(NULL));
826   noit_check_resolver_init();
827   noit_check_tools_init();
828   mtev_skiplist_init(&polls_by_name);
829   mtev_skiplist_set_compare(&polls_by_name, __check_name_compare,
830                             __check_name_compare);
831   mtev_skiplist_add_index(&polls_by_name, __check_target_ip_compare,
832                             __check_target_ip_compare);
833   mtev_skiplist_add_index(&polls_by_name, __check_target_compare,
834                             __check_target_compare);
835   mtev_skiplist_init(&watchlist);
836   mtev_skiplist_set_compare(&watchlist, __watchlist_compare,
837                             __watchlist_compare);
838   register_console_check_commands();
839   eventer_name_callback("check_recycle_bin_processor",
840                         check_recycle_bin_processor);
841   eventer_add_in_s_us(check_recycle_bin_processor, NULL, RECYCLE_INTERVAL, 0);
842   mtev_conf_get_int(NULL, "noit/@text_size_limit", &text_size_limit);
843   if (text_size_limit <= 0) {
844     text_size_limit = DEFAULT_TEXT_METRIC_SIZE_LIMIT;
845   }
846   noit_check_dns_ignore_list_init();
847   noit_poller_reload(NULL);
848 }
849
850 int
851 noit_poller_check_count() {
852   return polls_by_name.size;
853 }
854
855 int
856 noit_poller_transient_check_count() {
857   return watchlist.size;
858 }
859
860 noit_check_t *
861 noit_check_clone(uuid_t in) {
862   int i;
863   noit_check_t *checker, *new_check;
864   void *vcheck;
865   if(mtev_hash_retrieve(&polls,
866                         (char *)in, UUID_SIZE,
867                         &vcheck) == 0) {
868     return NULL;
869   }
870   checker = (noit_check_t *)vcheck;
871   if(checker->oncheck) {
872     return NULL;
873   }
874   new_check = calloc(1, sizeof(*new_check));
875   memcpy(new_check, checker, sizeof(*new_check));
876   new_check->target = strdup(new_check->target);
877   new_check->module = strdup(new_check->module);
878   new_check->name = strdup(new_check->name);
879   new_check->filterset = strdup(new_check->filterset);
880   new_check->flags = 0;
881   new_check->fire_event = NULL;
882   memset(&new_check->last_fire_time, 0, sizeof(new_check->last_fire_time));
883   new_check->statistics = noit_check_stats_set_calloc();
884   new_check->closure = NULL;
885   new_check->config = calloc(1, sizeof(*new_check->config));
886   mtev_hash_merge_as_dict(new_check->config, checker->config);
887   new_check->module_configs = NULL;
888   new_check->module_metadata = NULL;
889
890   for(i=0; i<reg_module_id; i++) {
891     void *src_metadata;
892     mtev_hash_table *src_mconfig;
893     src_mconfig = noit_check_get_module_config(checker, i);
894     if(src_mconfig) {
895       mtev_hash_table *t = calloc(1, sizeof(*new_check->config));
896       mtev_hash_merge_as_dict(t, src_mconfig);
897       noit_check_set_module_config(new_check, i, t);
898     }
899     if(checker->flags & NP_PASSIVE_COLLECTION)
900       if(NULL != (src_metadata = noit_check_get_module_metadata(new_check, i)))
901         noit_check_set_module_metadata(new_check, i, src_metadata, NULL);
902   }
903   return new_check;
904 }
905
906 noit_check_t *
907 noit_check_watch(uuid_t in, int period) {
908   /* First look for a copy that is being watched */
909   int minimum_pi = 1000, granularity_pi = 500;
910   mtev_conf_section_t check_node;
911   char uuid_str[UUID_STR_LEN + 1];
912   char xpath[1024];
913   noit_check_t n, *f;
914
915   uuid_unparse_lower(in, uuid_str);
916
917   mtevL(noit_debug, "noit_check_watch(%s,%d)\n", uuid_str, period);
918   if(period == 0) {
919     return noit_poller_lookup(in);
920   }
921
922   /* Find the check */
923   snprintf(xpath, sizeof(xpath), "//checks//check[@uuid=\"%s\"]", uuid_str);
924   check_node = mtev_conf_get_section(NULL, xpath);
925   mtev_conf_get_int(NULL, "//checks/@transient_min_period", &minimum_pi);
926   mtev_conf_get_int(NULL, "//checks/@transient_period_granularity", &granularity_pi);
927   if(check_node) {
928     mtev_conf_get_int(check_node,
929                       "ancestor-or-self::node()/@transient_min_period",
930                       &minimum_pi);
931     mtev_conf_get_int(check_node,
932                       "ancestor-or-self::node()/@transient_period_granularity",
933                       &granularity_pi);
934   }
935
936   /* apply the bounds */
937   period /= granularity_pi;
938   period *= granularity_pi;
939   period = MAX(period, minimum_pi);
940
941   uuid_copy(n.checkid, in);
942   n.period = period;
943
944   f = mtev_skiplist_find(&watchlist, &n, NULL);
945   if(f) return f;
946   f = noit_check_clone(in);
947   if(!f) return NULL;
948   f->period = period;
949   f->timeout = period - 10;
950   f->flags |= NP_TRANSIENT;
951   mtevL(noit_debug, "Watching %s@%d\n", uuid_str, period);
952   mtev_skiplist_insert(&watchlist, f);
953   return f;
954 }
955
956 noit_check_t *
957 noit_check_get_watch(uuid_t in, int period) {
958   noit_check_t n, *f;
959
960   uuid_copy(n.checkid, in);
961   n.period = period;
962
963   f = mtev_skiplist_find(&watchlist, &n, NULL);
964   return f;
965 }
966
967 void
968 noit_check_transient_add_feed(noit_check_t *check, const char *feed) {
969   char *feedcopy;
970   if(!check->feeds) {
971     check->feeds = calloc(1, sizeof(*check->feeds));
972     mtev_skiplist_init(check->feeds);
973     mtev_skiplist_set_compare(check->feeds,
974                               (mtev_skiplist_comparator_t)strcmp,
975                               (mtev_skiplist_comparator_t)strcmp);
976   }
977   feedcopy = strdup(feed);
978   /* No error on failure -- it's already there */
979   if(mtev_skiplist_insert(check->feeds, feedcopy) == NULL) free(feedcopy);
980   mtevL(noit_debug, "check %s`%s @ %dms has %d feed(s): %s.\n",
981         check->target, check->name, check->period, check->feeds->size, feed);
982 }
983 void
984 noit_check_transient_remove_feed(noit_check_t *check, const char *feed) {
985   if(!check->feeds) return;
986   if(feed) {
987     mtevL(noit_debug, "check %s`%s @ %dms removing 1 of %d feeds: %s.\n",
988           check->target, check->name, check->period, check->feeds->size, feed);
989     mtev_skiplist_remove(check->feeds, feed, free);
990   }
991   if(check->feeds->size == 0) {
992     char uuid_str[UUID_STR_LEN + 1];
993     uuid_unparse_lower(check->checkid, uuid_str);
994     mtevL(noit_debug, "Unwatching %s@%d\n", uuid_str, check->period);
995     mtev_skiplist_remove(&watchlist, check, NULL);
996     mtev_skiplist_destroy(check->feeds, free);
997     free(check->feeds);
998     check->feeds = NULL;
999     if(check->flags & NP_TRANSIENT) {
1000       mtevL(noit_debug, "check %s`%s @ %dms has no more listeners.\n",
1001             check->target, check->name, check->period);
1002       check->flags |= NP_KILLED;
1003     }
1004     noit_poller_free_check(check);
1005   }
1006 }
1007
1008 mtev_boolean
1009 noit_check_is_valid_target(const char *target) {
1010   int8_t family;
1011   int rv;
1012   union {
1013     struct in_addr addr4;
1014     struct in6_addr addr6;
1015   } a;
1016
1017   family = AF_INET;
1018   rv = inet_pton(family, target, &a);
1019   if(rv != 1) {
1020     family = AF_INET6;
1021     rv = inet_pton(family, target, &a);
1022     if(rv != 1) {
1023       return mtev_false;
1024     }
1025   }
1026   return mtev_true;
1027 }
1028 int
1029 noit_check_set_ip(noit_check_t *new_check,
1030                   const char *ip_str, const char *newname) {
1031   int8_t family;
1032   int rv, failed = 0;
1033   char old_target_ip[INET6_ADDRSTRLEN];
1034   union {
1035     struct in_addr addr4;
1036     struct in6_addr addr6;
1037   } a;
1038
1039   memset(old_target_ip, 0, INET6_ADDRSTRLEN);
1040   strlcpy(old_target_ip, new_check->target_ip, sizeof(old_target_ip));
1041
1042   family = NOIT_CHECK_PREFER_V6(new_check) ? AF_INET6 : AF_INET;
1043   rv = inet_pton(family, ip_str, &a);
1044   if(rv != 1) {
1045     if (!NOIT_CHECK_SINGLE_RESOLVE(new_check)) {
1046       family = family == AF_INET ? AF_INET6 : AF_INET;
1047       rv = inet_pton(family, ip_str, &a);
1048       if(rv != 1) {
1049         family = AF_INET;
1050         memset(&a, 0, sizeof(a));
1051         failed = -1;
1052       }
1053     } else {
1054       failed = -1;
1055     }
1056   }
1057
1058   new_check->target_family = family;
1059   memcpy(&new_check->target_addr, &a, sizeof(a));
1060   new_check->target_ip[0] = '\0';
1061   if(failed == 0)
1062     if(inet_ntop(new_check->target_family,
1063                  &new_check->target_addr,
1064                  new_check->target_ip,
1065                  sizeof(new_check->target_ip)) == NULL) {
1066       mtevL(noit_error, "inet_ntop failed [%s] -> %d\n", ip_str, errno);
1067     }
1068   /*
1069    * new_check->name could be null if this check is being set for the
1070    * first time.  add_to_list will set it.
1071    */
1072   if (new_check->name == NULL ||
1073       strcmp(old_target_ip, new_check->target_ip) != 0) {
1074     noit_check_add_to_list(new_check, newname);
1075   }
1076
1077   if(new_check->name == NULL && newname != NULL) {
1078     assert(new_check->flags & NP_TRANSIENT);
1079     new_check->name = strdup(newname);
1080   }
1081
1082   return failed;
1083 }
1084 int
1085 noit_check_resolve(noit_check_t *check) {
1086   uint8_t family_pref = NOIT_CHECK_PREFER_V6(check) ? AF_INET6 : AF_INET;
1087   char ipaddr[INET6_ADDRSTRLEN];
1088   if(!NOIT_CHECK_SHOULD_RESOLVE(check)) return 1; /* success, not required */
1089   noit_check_resolver_remind(check->target);
1090   if(noit_check_resolver_fetch(check->target, ipaddr, sizeof(ipaddr),
1091                                family_pref) >= 0) {
1092     check->flags |= NP_RESOLVED;
1093     noit_check_set_ip(check, ipaddr, NULL);
1094     return 0;
1095   }
1096   check->flags &= ~NP_RESOLVED;
1097   return -1;
1098 }
1099 int
1100 noit_check_update(noit_check_t *new_check,
1101                   const char *target,
1102                   const char *name,
1103                   const char *filterset,
1104                   mtev_hash_table *config,
1105                   mtev_hash_table **mconfigs,
1106                   u_int32_t period,
1107                   u_int32_t timeout,
1108                   const char *oncheck,
1109           int64_t seq,
1110                   int flags) {
1111   char uuid_str[37];
1112   int mask = NP_DISABLED | NP_UNCONFIG;
1113
1114   assert(name);
1115   uuid_unparse_lower(new_check->checkid, uuid_str);
1116   if(!new_check->statistics) new_check->statistics = noit_check_stats_set_calloc();
1117   if(seq < 0) new_check->config_seq = seq = 0;
1118   if(new_check->config_seq > seq) {
1119     mtevL(mtev_error, "noit_check_update[%s] skipped: seq backwards\n", uuid_str);
1120     return -1;
1121   }
1122
1123   /* selfcheck will identify this node in a cluster */
1124   if(mtev_cluster_enabled() && !strcmp(new_check->module, "selfcheck")) {
1125     uuid_t cluster_id;
1126     mtev_cluster_get_self(cluster_id);
1127     if(uuid_compare(cluster_id, new_check->checkid)) {
1128       mtevL(mtev_error, "Setting global cluster identity to '%s'\n", uuid_str);
1129       mtev_cluster_set_self(new_check->checkid);
1130     }
1131   }
1132
1133   if(NOIT_CHECK_RUNNING(new_check)) {
1134     char module[256];
1135     uuid_t id, dummy;
1136     uuid_copy(id, new_check->checkid);
1137     strlcpy(module, new_check->module, sizeof(module));
1138     noit_poller_deschedule(id);
1139     return noit_poller_schedule(target, module, name, filterset,
1140                                 config, mconfigs, period, timeout, oncheck,
1141                                 seq, flags, id, dummy);
1142   }
1143
1144   new_check->generation = __config_load_generation;
1145   if(new_check->target) free(new_check->target);
1146   new_check->target = strdup(target);
1147
1148   // apply resolution flags to check.
1149   if (flags & NP_PREFER_IPV6)
1150     new_check->flags |= NP_PREFER_IPV6;
1151   else
1152     new_check->flags &= ~NP_PREFER_IPV6;
1153   if (flags & NP_SINGLE_RESOLVE)
1154     new_check->flags |= NP_SINGLE_RESOLVE;
1155   else
1156     new_check->flags &= ~NP_SINGLE_RESOLVE;
1157   if (flags & NP_RESOLVE)
1158     new_check->flags |= NP_RESOLVE;
1159   else
1160     new_check->flags &= ~NP_RESOLVE;
1161
1162   /* This sets both the name and the target_addr */
1163   if(noit_check_set_ip(new_check, target, name)) {
1164     mtev_boolean should_resolve;
1165     mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1166     const char *key, *value;
1167     int klen;
1168     char* extension = strrchr(target, '.');
1169     new_check->flags |= NP_RESOLVE;
1170     new_check->flags &= ~NP_RESOLVED;
1171     /* If we match any of the extensions we're supposed to ignore,
1172      * don't resolve */
1173     if (extension && (strlen(extension) > 1)) {
1174       while(mtev_hash_next(&dns_ignore_list, &iter, &key, &klen, (void**)&value)) {
1175         if ((!strcmp("true", value)) && (!strcmp(extension+1, key))) {
1176             new_check->flags &= ~NP_RESOLVE;
1177             break;
1178         }
1179       }
1180     }
1181     if(noit_check_should_resolve_targets(&should_resolve) && !should_resolve)
1182       flags |= NP_DISABLED | NP_UNCONFIG;
1183     noit_check_resolve(new_check);
1184   }
1185
1186   if(new_check->filterset) free(new_check->filterset);
1187   new_check->filterset = filterset ? strdup(filterset): NULL;
1188
1189   if(config != NULL) {
1190     mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1191     const char *k;
1192     int klen;
1193     void *data;
1194     if(new_check->config) mtev_hash_delete_all(new_check->config, free, free);
1195     else new_check->config = calloc(1, sizeof(*new_check->config));
1196     while(mtev_hash_next(config, &iter, &k, &klen, &data)) {
1197       mtev_hash_store(new_check->config, strdup(k), klen, strdup((char *)data));
1198     }
1199   }
1200   if(mconfigs != NULL) {
1201     int i;
1202     for(i=0; i<reg_module_id; i++) {
1203       mtev_hash_table *t;
1204       if(NULL != (t = noit_check_get_module_config(new_check, i))) {
1205         noit_check_set_module_config(new_check, i, NULL);
1206         mtev_hash_destroy(t, free, free);
1207         free(t);
1208       }
1209       if(mconfigs[i]) {
1210         mtev_hash_table *t = calloc(1, sizeof(*new_check->config));
1211         mtev_hash_merge_as_dict(t, mconfigs[i]);
1212         noit_check_set_module_config(new_check, i, t);
1213       }
1214     }
1215   }
1216   if(new_check->oncheck) free(new_check->oncheck);
1217   new_check->oncheck = oncheck ? strdup(oncheck) : NULL;
1218   if(new_check->oncheck) system_needs_causality = mtev_true;
1219   new_check->period = period;
1220   new_check->timeout = timeout;
1221   new_check->config_seq = seq;
1222
1223   /* Unset what could be set.. then set what should be set */
1224   new_check->flags = (new_check->flags & ~mask) | flags;
1225
1226   check_config_fixup_hook_invoke(new_check);
1227
1228   if((new_check->flags & NP_TRANSIENT) == 0)
1229     noit_check_activate(new_check);
1230
1231   noit_check_add_to_list(new_check, NULL);
1232   noit_check_log_check(new_check);
1233   return 0;
1234 }
1235 int
1236 noit_poller_schedule(const char *target,
1237                      const char *module,
1238                      const char *name,
1239                      const char *filterset,
1240                      mtev_hash_table *config,
1241                      mtev_hash_table **mconfigs,
1242                      u_int32_t period,
1243                      u_int32_t timeout,
1244                      const char *oncheck,
1245                      int64_t seq,
1246                      int flags,
1247                      uuid_t in,
1248                      uuid_t out) {
1249   noit_check_t *new_check;
1250   new_check = calloc(1, sizeof(*new_check));
1251   if(!new_check) return -1;
1252
1253   /* The module and the UUID can never be changed */
1254   new_check->module = strdup(module);
1255   if(uuid_is_null(in))
1256     uuid_generate(new_check->checkid);
1257   else
1258     uuid_copy(new_check->checkid, in);
1259
1260   new_check->statistics = noit_check_stats_set_calloc();
1261   noit_check_update(new_check, target, name, filterset, config, mconfigs,
1262                     period, timeout, oncheck, seq, flags);
1263   assert(mtev_hash_store(&polls,
1264                          (char *)new_check->checkid, UUID_SIZE,
1265                          new_check));
1266   uuid_copy(out, new_check->checkid);
1267
1268   return 0;
1269 }
1270
1271 /* A quick little list of recycleable checks.  This list never really
1272  * grows large, so no sense in thinking too hard about the algorithmic
1273  * complexity.
1274  */
1275 struct _checker_rcb {
1276   noit_check_t *checker;
1277   struct _checker_rcb *next;
1278 };
1279 static struct _checker_rcb *checker_rcb = NULL;
1280 static void recycle_check(noit_check_t *checker) {
1281   struct _checker_rcb *n = malloc(sizeof(*n));
1282   n->checker = checker;
1283   n->next = checker_rcb;
1284   checker_rcb = n;
1285 }
1286 void
1287 noit_poller_free_check(noit_check_t *checker) {
1288   noit_module_t *mod;
1289
1290   if(checker->flags & NP_RUNNING) {
1291     recycle_check(checker);
1292     return;
1293   }
1294
1295   mod = noit_module_lookup(checker->module);
1296   if(mod && mod->cleanup) mod->cleanup(mod, checker);
1297   if(checker->fire_event) {
1298      eventer_remove(checker->fire_event);
1299      free(checker->fire_event->closure);
1300      eventer_free(checker->fire_event);
1301      checker->fire_event = NULL;
1302   }
1303   if(checker->closure) free(checker->closure);
1304   if(checker->target) free(checker->target);
1305   if(checker->module) free(checker->module);
1306   if(checker->name) free(checker->name);
1307   if(checker->config) {
1308     mtev_hash_destroy(checker->config, free, free);
1309     free(checker->config);
1310     checker->config = NULL;
1311   }
1312   if(checker->module_metadata) {
1313     int i;
1314     for(i=0; i<reg_module_id; i++) {
1315       struct vp_w_free *tuple;
1316       tuple = checker->module_metadata[i];
1317       if(tuple) {
1318         if(tuple->freefunc) tuple->freefunc(tuple->ptr);
1319         free(tuple);
1320       }
1321     }
1322     free(checker->module_metadata);
1323   }
1324   if(checker->module_configs) {
1325     int i;
1326     for(i=0; i<reg_module_id; i++) {
1327       if(checker->module_configs[i]) {
1328         mtev_hash_destroy(checker->module_configs[i], free, free);
1329         free(checker->module_configs[i]);
1330       }
1331     }
1332     free(checker->module_configs);
1333   }
1334   mtev_memory_safe_free(stats_inprogress(checker));
1335   mtev_memory_safe_free(stats_current(checker));
1336   mtev_memory_safe_free(stats_previous(checker));
1337   free(checker);
1338 }
1339 static int
1340 check_recycle_bin_processor(eventer_t e, int mask, void *closure,
1341                             struct timeval *now) {
1342   static struct timeval one_minute = { RECYCLE_INTERVAL, 0L };
1343   struct _checker_rcb *prev = NULL, *curr = checker_rcb;
1344   mtevL(noit_debug, "Scanning check recycle bin\n");
1345   while(curr) {
1346     if(!(curr->checker->flags & NP_RUNNING)) {
1347       mtevL(noit_debug, "Check is ready to free.\n");
1348       noit_poller_free_check(curr->checker);
1349       if(prev) prev->next = curr->next;
1350       else checker_rcb = curr->next;
1351       free(curr);
1352       curr = prev ? prev->next : checker_rcb;
1353     }
1354     else {
1355       prev = curr;
1356       curr = curr->next;
1357     }
1358   }
1359   add_timeval(*now, one_minute, &e->whence);
1360   return EVENTER_TIMER;
1361 }
1362
1363 int
1364 noit_poller_deschedule(uuid_t in) {
1365   void *vcheck;
1366   noit_check_t *checker;
1367   if(mtev_hash_retrieve(&polls,
1368                         (char *)in, UUID_SIZE,
1369                         &vcheck) == 0) {
1370     return -1;
1371   }
1372   checker = (noit_check_t *)vcheck;
1373   checker->flags |= (NP_DISABLED|NP_KILLED);
1374
1375   noit_check_log_delete(checker);
1376
1377   assert(mtev_skiplist_remove(&polls_by_name, checker, NULL));
1378   assert(mtev_hash_delete(&polls, (char *)in, UUID_SIZE, NULL, NULL));
1379
1380   noit_poller_free_check(checker);
1381   return 0;
1382 }
1383
1384 noit_check_t *
1385 noit_poller_lookup(uuid_t in) {
1386   noit_check_t *check;
1387   pthread_mutex_lock(&polls_lock);
1388   check = noit_poller_lookup__nolock(in);
1389   pthread_mutex_unlock(&polls_lock);
1390   return check;
1391 }
1392 noit_check_t *
1393 noit_poller_lookup_by_name(char *target, char *name) {
1394   noit_check_t *check;
1395   pthread_mutex_lock(&polls_lock);
1396   check = noit_poller_lookup_by_name__nolock(target,name);
1397   pthread_mutex_unlock(&polls_lock);
1398   return check;
1399 }
1400 int
1401 noit_poller_target_ip_do(const char *target_ip,
1402                          int (*f)(noit_check_t *, void *),
1403                          void *closure) {
1404   int i, count = 0, todo_count = 0;
1405   noit_check_t pivot;
1406   mtev_skiplist *tlist;
1407   mtev_skiplist_node *next;
1408   noit_check_t *todo_onstack[8192];
1409   noit_check_t **todo = todo_onstack;
1410
1411   tlist = mtev_skiplist_find(polls_by_name.index,
1412                              __check_target_ip_compare, NULL);
1413
1414   pthread_mutex_lock(&polls_lock);
1415   /* First pass to count */
1416   memset(&pivot, 0, sizeof(pivot));
1417   strlcpy(pivot.target_ip, (char*)target_ip, sizeof(pivot.target_ip));
1418   pivot.name = "";
1419   pivot.target = "";
1420   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1421   while(next && next->data) {
1422     noit_check_t *check = next->data;
1423     if(strcmp(check->target_ip, target_ip)) break;
1424     todo_count++;
1425     mtev_skiplist_next(tlist, &next);
1426   }
1427
1428   if(todo_count > 8192) todo = malloc(todo_count * sizeof(*todo));
1429
1430   memset(&pivot, 0, sizeof(pivot));
1431   strlcpy(pivot.target_ip, (char*)target_ip, sizeof(pivot.target_ip));
1432   pivot.name = "";
1433   pivot.target = "";
1434   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1435   while(next && next->data) {
1436     noit_check_t *check = next->data;
1437     if(strcmp(check->target_ip, target_ip)) break;
1438     if(count < todo_count) todo[count++] = check;
1439     mtev_skiplist_next(tlist, &next);
1440   }
1441   pthread_mutex_unlock(&polls_lock);
1442
1443   todo_count = count;
1444   count = 0;
1445   for(i=0;i<todo_count;i++)
1446     count += f(todo[i],closure);
1447
1448   if(todo != todo_onstack) free(todo);
1449   return count;
1450 }
1451 int
1452 noit_poller_target_do(const char *target, int (*f)(noit_check_t *, void *),
1453                       void *closure) {
1454   int i, todo_count = 0, count = 0;
1455   noit_check_t pivot;
1456   mtev_skiplist *tlist;
1457   mtev_skiplist_node *next;
1458   noit_check_t *todo_onstack[8192];
1459   noit_check_t **todo = todo_onstack;
1460
1461   tlist = mtev_skiplist_find(polls_by_name.index,
1462                              __check_target_compare, NULL);
1463
1464   pthread_mutex_lock(&polls_lock);
1465   memset(&pivot, 0, sizeof(pivot));
1466   pivot.name = "";
1467   pivot.target = (char *)target;
1468   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1469   while(next && next->data) {
1470     noit_check_t *check = next->data;
1471     if(strcmp(check->target, target)) break;
1472     todo_count++;
1473     mtev_skiplist_next(tlist, &next);
1474   }
1475
1476   if(todo_count > 8192) todo = malloc(todo_count * sizeof(*todo));
1477
1478   memset(&pivot, 0, sizeof(pivot));
1479   pivot.name = "";
1480   pivot.target = (char *)target;
1481   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1482   while(next && next->data) {
1483     noit_check_t *check = next->data;
1484     if(strcmp(check->target, target)) break;
1485     if(count < todo_count) todo[count++] = check;
1486     mtev_skiplist_next(tlist, &next);
1487   }
1488   pthread_mutex_unlock(&polls_lock);
1489
1490   todo_count = count;
1491   count = 0;
1492   for(i=0;i<todo_count;i++)
1493     count += f(todo[i],closure);
1494
1495   if(todo != todo_onstack) free(todo);
1496   return count;
1497 }
1498
1499 int
1500 noit_poller_do(int (*f)(noit_check_t *, void *),
1501                void *closure) {
1502   mtev_skiplist_node *iter;
1503   int i, count = 0, max_count = 0;
1504   noit_check_t **todo;
1505
1506   if(polls_by_name.size == 0) return 0;
1507
1508   max_count = polls_by_name.size;
1509   todo = malloc(max_count * sizeof(*todo));
1510
1511   pthread_mutex_lock(&polls_lock);
1512   for(iter = mtev_skiplist_getlist(&polls_by_name); iter;
1513       mtev_skiplist_next(&polls_by_name, &iter)) {
1514     if(count < max_count) todo[count++] = (noit_check_t *)iter->data;
1515   }
1516   pthread_mutex_unlock(&polls_lock);
1517
1518   max_count = count;
1519   count = 0;
1520   for(i=0;i<max_count;i++)
1521     count += f(todo[i], closure);
1522   free(todo);
1523   return count;
1524 }
1525
1526 struct ip_module_collector_crutch {
1527   noit_check_t **array;
1528   const char *module;
1529   int idx;
1530   int allocd;
1531 };
1532 static int ip_module_collector(noit_check_t *check, void *cl) {
1533   struct ip_module_collector_crutch *c = cl;
1534   if(c->idx >= c->allocd) return 0;
1535   if(strcmp(check->module, c->module)) return 0;
1536   c->array[c->idx++] = check;
1537   return 1;
1538 }
1539 int
1540 noit_poller_lookup_by_ip_module(const char *ip, const char *mod,
1541                                 noit_check_t **checks, int nchecks) {
1542   struct ip_module_collector_crutch crutch;
1543   crutch.array = checks;
1544   crutch.allocd = nchecks;
1545   crutch.idx = 0;
1546   crutch.module = mod;
1547   return noit_poller_target_ip_do(ip, ip_module_collector, &crutch);
1548 }
1549 int
1550 noit_poller_lookup_by_module(const char *ip, const char *mod,
1551                              noit_check_t **checks, int nchecks) {
1552   struct ip_module_collector_crutch crutch;
1553   crutch.array = checks;
1554   crutch.allocd = nchecks;
1555   crutch.idx = 0;
1556   crutch.module = mod;
1557   return noit_poller_target_do(ip, ip_module_collector, &crutch);
1558 }
1559
1560
1561 int
1562 noit_check_xpath(char *xpath, int len,
1563                  const char *base, const char *arg) {
1564   uuid_t checkid;
1565   int base_trailing_slash;
1566   char argcopy[1024], *target, *module, *name;
1567
1568   base_trailing_slash = (base[strlen(base)-1] == '/');
1569   xpath[0] = '\0';
1570   argcopy[0] = '\0';
1571   if(arg) strlcpy(argcopy, arg, sizeof(argcopy));
1572
1573   if(uuid_parse(argcopy, checkid) == 0) {
1574     /* If they kill by uuid, we'll seek and destroy -- find it anywhere */
1575     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
1576              base, base_trailing_slash ? "" : "/", argcopy);
1577   }
1578   else if((module = strchr(argcopy, '`')) != NULL) {
1579     noit_check_t *check;
1580     char uuid_str[37];
1581     target = argcopy;
1582     *module++ = '\0';
1583     if((name = strchr(module+1, '`')) == NULL)
1584       name = module;
1585     else
1586       name++;
1587     check = noit_poller_lookup_by_name(target, name);
1588     if(!check) {
1589       return -1;
1590     }
1591     uuid_unparse_lower(check->checkid, uuid_str);
1592     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
1593              base, base_trailing_slash ? "" : "/", uuid_str);
1594   }
1595   return strlen(xpath);
1596 }
1597
1598 static int
1599 bad_check_initiate(noit_module_t *self, noit_check_t *check,
1600                    int once, noit_check_t *cause) {
1601   /* self is likely null here -- why it is bad, in fact */
1602   /* this is only suitable to call in one-offs */
1603   struct timeval now;
1604   stats_t *inp;
1605   char buff[256];
1606   if(!once) return -1;
1607   if(!check) return -1;
1608   assert(!(check->flags & NP_RUNNING));
1609   check->flags |= NP_RUNNING;
1610   inp = noit_check_get_stats_inprogress(check);
1611   gettimeofday(&now, NULL);
1612   noit_check_stats_whence(inp, &now);
1613   snprintf(buff, sizeof(buff), "check[%s] implementation offline",
1614            check->module);
1615   noit_check_stats_status(inp, buff);
1616   noit_check_set_stats(check);
1617   check->flags &= ~NP_RUNNING;
1618   return 0;
1619 }
1620 void
1621 noit_check_stats_clear(noit_check_t *check, stats_t *s) {
1622   memset(s, 0, sizeof(*s));
1623   s->state = NP_UNKNOWN;
1624   s->available = NP_UNKNOWN;
1625 }
1626
1627 void
1628 __stats_add_metric(stats_t *newstate, metric_t *m) {
1629   mtev_hash_replace(&newstate->metrics, m->metric_name, strlen(m->metric_name),
1630                     m, NULL, (void (*)(void *))mtev_memory_safe_free);
1631 }
1632
1633 static size_t
1634 noit_metric_sizes(metric_type_t type, const void *value) {
1635   switch(type) {
1636     case METRIC_INT32:
1637     case METRIC_UINT32:
1638       return sizeof(int32_t);
1639     case METRIC_INT64:
1640     case METRIC_UINT64:
1641       return sizeof(int64_t);
1642     case METRIC_DOUBLE:
1643       return sizeof(double);
1644     case METRIC_STRING: {
1645       int len = strlen((char*)value) + 1;
1646       return ((len >= text_size_limit) ? text_size_limit+1 : len);
1647     }
1648     case METRIC_GUESS:
1649       break;
1650   }
1651   assert(type != type);
1652   return 0;
1653 }
1654 static metric_type_t
1655 noit_metric_guess_type(const char *s, void **replacement) {
1656   char *copy, *cp, *trailer, *rpl;
1657   int negative = 0;
1658   metric_type_t type = METRIC_STRING;
1659
1660   if(!s) return METRIC_GUESS;
1661   copy = cp = strdup(s);
1662
1663   /* TRIM the string */
1664   while(*cp && isspace(*cp)) cp++; /* ltrim */
1665   s = cp; /* found a good starting point */
1666   while(*cp) cp++; /* advance to \0 */
1667   cp--; /* back up one */
1668   while(cp > s && isspace(*cp)) *cp-- = '\0'; /* rtrim */
1669
1670   /* Find the first space */
1671   cp = (char *)s;
1672   while(*cp && !isspace(*cp)) cp++;
1673   trailer = cp;
1674   cp--; /* backup one */
1675   if(cp > s && *cp == '%') *cp-- = '\0'; /* chop a last % is there is one */
1676
1677   while(*trailer && isspace(*trailer)) *trailer++ = '\0'; /* rtrim */
1678
1679   /* string was       '  -1.23e-01%  inodes used  ' */
1680   /* copy is (~ = \0) '  -1.23e-01~  inodes used~~' */
1681   /*                     ^           ^              */
1682   /*                     s           trailer        */
1683
1684   /* So, the trailer must not contain numbers */
1685   while(*trailer) { if(isdigit(*trailer)) goto notanumber; trailer++; }
1686
1687   /* And the 's' must be of the form:
1688    *  0) may start with a sign [-+]?
1689    *  1) [1-9][0-9]*
1690    *  2) [0]?.[0-9]+
1691    *  3) 0
1692    *  4) [1-9][0-9]*.[0-9]+
1693    *  5) all of the above ending with e[+-][0-9]+
1694    */
1695    rpl = (char *)s;
1696    /* CASE 0 */
1697    if(s[0] == '-' || s[0] == '+') {
1698      if(s[0] == '-') negative = 1;
1699      s++;
1700    }
1701
1702    if(s[0] == '.') goto decimal; /* CASE 2 */
1703    if(s[0] == '0') { /* CASE 2 & 3 */
1704      s++;
1705      if(!s[0]) goto scanint; /* CASE 3 */
1706      if(s[0] == '.') goto decimal; /* CASE 2 */
1707      goto notanumber;
1708    }
1709    if(s[0] >= '1' && s[0] <= '9') { /* CASE 1 & 4 */
1710      s++;
1711      while(isdigit(s[0])) s++; /* CASE 1 & 4 */
1712      if(!s[0]) goto scanint; /* CASE 1 */
1713      if(s[0] == '.') goto decimal; /* CASE 4 */
1714      goto notanumber;
1715    }
1716    /* Not case 1,2,3,4 */
1717    goto notanumber;
1718
1719   decimal:
1720    s++;
1721    if(!isdigit(s[0])) goto notanumber;
1722    s++;
1723    while(isdigit(s[0])) s++;
1724    if(!s[0]) goto scandouble;
1725    if(s[0] == 'e' || s[0] == 'E') goto exponent; /* CASE 5 */
1726    goto notanumber;
1727
1728   exponent:
1729    s++;
1730    if(s[0] != '-' && s[0] != '+') goto notanumber;
1731    s++;
1732    if(!isdigit(s[0])) goto notanumber;
1733    s++;
1734    while(isdigit(s[0])) s++;
1735    if(!s[0]) goto scandouble;
1736    goto notanumber;
1737
1738  scanint:
1739    if(negative) {
1740      int64_t *v;
1741      v = malloc(sizeof(*v));
1742      *v = strtoll(rpl, NULL, 10);
1743      *replacement = v;
1744      type = METRIC_INT64;
1745      goto alldone;
1746    }
1747    else {
1748      u_int64_t *v;
1749      v = malloc(sizeof(*v));
1750      *v = strtoull(rpl, NULL, 10);
1751      *replacement = v;
1752      type = METRIC_UINT64;
1753      goto alldone;
1754    }
1755  scandouble:
1756    {
1757      double *v;
1758      v = malloc(sizeof(*v));
1759      *v = strtod(rpl, NULL);
1760      *replacement = v;
1761      type = METRIC_DOUBLE;
1762      goto alldone;
1763    }
1764
1765  alldone:
1766  notanumber:
1767   free(copy);
1768   return type;
1769 }
1770
1771 static void
1772 cleanse_metric_name(char *m) {
1773   char *cp;
1774   for(cp = m; *cp; cp++)
1775     if(!isprint(*cp)) *cp=' ';
1776   for(cp--; *cp == ' ' && cp > m; cp--) /* always leave first char */
1777     *cp = '\0';
1778 }
1779
1780 int
1781 noit_stats_populate_metric(metric_t *m, const char *name, metric_type_t type,
1782                            const void *value) {
1783   void *replacement = NULL;
1784
1785   m->metric_name = strdup(name);
1786   cleanse_metric_name(m->metric_name);
1787
1788   if(type == METRIC_GUESS)
1789     type = noit_metric_guess_type((char *)value, &replacement);
1790   if(type == METRIC_GUESS) return -1;
1791
1792   m->metric_type = type;
1793
1794   if(replacement)
1795     m->metric_value.vp = replacement;
1796   else if(value) {
1797     size_t len;
1798     len = noit_metric_sizes(type, value);
1799     m->metric_value.vp = malloc(len);
1800     memcpy(m->metric_value.vp, value, len);
1801     if (type == METRIC_STRING) {
1802       m->metric_value.s[len-1] = 0;
1803     }
1804   }
1805   else m->metric_value.vp = NULL;
1806   return 0;
1807 }
1808
1809 metric_t *
1810 noit_stats_get_metric(noit_check_t *check,
1811                       stats_t *newstate, const char *name) {
1812   void *v;
1813   if(newstate == NULL)
1814     newstate = stats_inprogress(check);
1815   if(mtev_hash_retrieve(&newstate->metrics, name, strlen(name), &v))
1816     return (metric_t *)v;
1817   return NULL;
1818 }
1819
1820 void
1821 noit_stats_set_metric(noit_check_t *check,
1822                       const char *name, metric_type_t type,
1823                       const void *value) {
1824   stats_t *c;
1825   metric_t *m = mtev_memory_safe_malloc_cleanup(sizeof(*m), noit_check_safe_free_metric);
1826   memset(m, 0, sizeof(*m));
1827   if(noit_stats_populate_metric(m, name, type, value)) {
1828     mtev_memory_safe_free(m);
1829     return;
1830   }
1831   noit_check_metric_count_add(1);
1832   c = noit_check_get_stats_inprogress(check);
1833   check_stats_set_metric_hook_invoke(check, c, m);
1834   __stats_add_metric(c, m);
1835 }
1836 void
1837 noit_stats_set_metric_coerce(noit_check_t *check,
1838                              const char *name, metric_type_t t,
1839                              const char *v) {
1840   char *endptr;
1841   stats_t *c;
1842   c = noit_check_get_stats_inprogress(check);
1843   if(v == NULL) {
1844    bogus:
1845     check_stats_set_metric_coerce_hook_invoke(check, c, name, t, v, mtev_false);
1846     noit_stats_set_metric(check, name, t, NULL);
1847     return;
1848   }
1849   switch(t) {
1850     case METRIC_STRING:
1851       noit_stats_set_metric(check, name, t, v);
1852       break;
1853     case METRIC_INT32:
1854     {
1855       int32_t val;
1856       val = strtol(v, &endptr, 10);
1857       if(endptr == v) goto bogus;
1858       noit_stats_set_metric(check, name, t, &val);
1859       break;
1860     }
1861     case METRIC_UINT32:
1862     {
1863       u_int32_t val;
1864       val = strtoul(v, &endptr, 10);
1865       if(endptr == v) goto bogus;
1866       noit_stats_set_metric(check, name, t, &val);
1867       break;
1868     }
1869     case METRIC_INT64:
1870     {
1871       int64_t val;
1872       val = strtoll(v, &endptr, 10);
1873       if(endptr == v) goto bogus;
1874       noit_stats_set_metric(check, name, t, &val);
1875       break;
1876     }
1877     case METRIC_UINT64:
1878     {
1879       u_int64_t val;
1880       val = strtoull(v, &endptr, 10);
1881       if(endptr == v) goto bogus;
1882       noit_stats_set_metric(check, name, t, &val);
1883       break;
1884     }
1885     case METRIC_DOUBLE:
1886     {
1887       double val;
1888       val = strtod(v, &endptr);
1889       if(endptr == v) goto bogus;
1890       noit_stats_set_metric(check, name, t, &val);
1891       break;
1892     }
1893     case METRIC_GUESS:
1894       noit_stats_set_metric(check, name, t, v);
1895       break;
1896   }
1897   check_stats_set_metric_coerce_hook_invoke(check, c, name, t, v, mtev_true);
1898 }
1899 void
1900 noit_stats_log_immediate_metric(noit_check_t *check,
1901                                 const char *name, metric_type_t type,
1902                                 void *value) {
1903   struct timeval now;
1904   metric_t *m = mtev_memory_safe_malloc_cleanup(sizeof(*m), noit_check_safe_free_metric);
1905   memset(m, 0, sizeof(*m));
1906   if(noit_stats_populate_metric(m, name, type, value)) {
1907     mtev_memory_safe_free(m);
1908     return;
1909   }
1910   gettimeofday(&now, NULL);
1911   noit_check_log_metric(check, &now, m);
1912   mtev_memory_safe_free(m);
1913 }
1914
1915 void
1916 noit_check_passive_set_stats(noit_check_t *check) {
1917   int i, nwatches = 0;
1918   mtev_skiplist_node *next;
1919   noit_check_t n;
1920   noit_check_t *watches[8192];
1921
1922   uuid_copy(n.checkid, check->checkid);
1923   n.period = 0;
1924
1925   noit_check_set_stats(check);
1926
1927   pthread_mutex_lock(&polls_lock);
1928   mtev_skiplist_find_neighbors(&watchlist, &n, NULL, NULL, &next);
1929   while(next && next->data && nwatches < 8192) {
1930     noit_check_t *wcheck = next->data;
1931     if(uuid_compare(n.checkid, wcheck->checkid)) break;
1932     watches[nwatches++] = wcheck;
1933     mtev_skiplist_next(&watchlist, &next);
1934   }
1935   pthread_mutex_unlock(&polls_lock);
1936
1937   for(i=0;i<nwatches;i++) {
1938     void *backup;
1939     noit_check_t *wcheck = watches[i];
1940     /* Swap the real check's stats into place */
1941     backup = wcheck->statistics;
1942     wcheck->statistics = check->statistics;
1943
1944     if(check_passive_log_stats_hook_invoke(check) == MTEV_HOOK_CONTINUE) {
1945       /* Write out our status */
1946       noit_check_log_status(wcheck);
1947       /* Write out all metrics */
1948       noit_check_log_metrics(wcheck);
1949     }
1950     /* Swap them back out */
1951     wcheck->statistics = backup;
1952   }
1953 }
1954 void
1955 noit_check_set_stats(noit_check_t *check) {
1956   int report_change = 0;
1957   char *cp;
1958   dep_list_t *dep;
1959   stats_t *old, *prev, *current;
1960   old = stats_previous(check);
1961   prev = stats_previous(check) = stats_current(check);
1962   current = stats_current(check) = stats_inprogress(check);
1963   stats_inprogress(check) = noit_check_stats_alloc();
1964  
1965   if(old) {
1966     mtev_memory_safe_free(old);
1967   }
1968
1969   if(current) {
1970     for(cp = current->status; cp && *cp; cp++)
1971       if(*cp == '\r' || *cp == '\n') *cp = ' ';
1972   }
1973
1974   /* check for state changes */
1975   if((!current || (current->available != NP_UNKNOWN)) &&
1976      (!prev || (prev->available != NP_UNKNOWN)) &&
1977      (!current || !prev || (current->available != prev->available)))
1978     report_change = 1;
1979   if((!current || (current->state != NP_UNKNOWN)) &&
1980      (!prev || (prev->state != NP_UNKNOWN)) &&
1981      (!current || !prev || (current->state != prev->state)))
1982     report_change = 1;
1983
1984   mtevL(noit_debug, "%s`%s <- [%s]\n", check->target, check->name,
1985         current ? current->status : "null");
1986   if(report_change) {
1987     mtevL(noit_debug, "%s`%s -> [%s:%s]\n",
1988           check->target, check->name,
1989           noit_check_available_string(current ? current->available : NP_UNKNOWN),
1990           noit_check_state_string(current ? current->state : NP_UNKNOWN));
1991   }
1992
1993   if(NOIT_CHECK_STATUS_ENABLED()) {
1994     char id[UUID_STR_LEN+1];
1995     uuid_unparse_lower(check->checkid, id);
1996     NOIT_CHECK_STATUS(id, check->module, check->name, check->target,
1997                       current ? current->available : NP_UNKNOWN,
1998                       current ? current->state : NP_UNKNOWN,
1999                       current ? current->status : "null");
2000   }
2001
2002   if(check_log_stats_hook_invoke(check) == MTEV_HOOK_CONTINUE) {
2003     /* Write out the bundled information */
2004     noit_check_log_bundle(check);
2005   }
2006   /* count the check as complete */
2007   check_completion_count++;
2008
2009   for(dep = check->causal_checks; dep; dep = dep->next) {
2010     noit_module_t *mod;
2011     mod = noit_module_lookup(dep->check->module);
2012     if(!mod) {
2013       bad_check_initiate(mod, dep->check, 1, check);
2014     }
2015     else {
2016       mtevL(noit_debug, "Firing %s`%s in response to %s`%s\n",
2017             dep->check->target, dep->check->name,
2018             check->target, check->name);
2019       if((dep->check->flags & NP_DISABLED) == 0)
2020         if(mod->initiate_check)
2021           mod->initiate_check(mod, dep->check, 1, check);
2022     }
2023   }
2024 }
2025
2026 static int
2027 noit_console_show_watchlist(mtev_console_closure_t ncct,
2028                             int argc, char **argv,
2029                             mtev_console_state_t *dstate,
2030                             void *closure) {
2031   mtev_skiplist_node *iter, *fiter;
2032   int nwatches = 0, i;
2033   noit_check_t *watches[8192];
2034
2035   nc_printf(ncct, "%d active watches.\n", watchlist.size);
2036   pthread_mutex_lock(&polls_lock);
2037   for(iter = mtev_skiplist_getlist(&watchlist); iter && nwatches < 8192;
2038       mtev_skiplist_next(&watchlist, &iter)) {
2039     noit_check_t *check = iter->data;
2040     watches[nwatches++] = check;
2041   }
2042   pthread_mutex_unlock(&polls_lock);
2043
2044   for(i=0;i<nwatches;i++) {
2045     noit_check_t *check = watches[i];
2046     char uuid_str[UUID_STR_LEN + 1];
2047
2048     uuid_unparse_lower(check->checkid, uuid_str);
2049     nc_printf(ncct, "%s:\n\t[%s`%s`%s]\n\tPeriod: %dms\n\tFeeds[%d]:\n",
2050               uuid_str, check->target, check->module, check->name,
2051               check->period, check->feeds ? check->feeds->size : 0);
2052     if(check->feeds && check->feeds->size) {
2053       for(fiter = mtev_skiplist_getlist(check->feeds); fiter;
2054           mtev_skiplist_next(check->feeds, &fiter)) {
2055         nc_printf(ncct, "\t\t%s\n", (const char *)fiter->data);
2056       }
2057     }
2058   }
2059   return 0;
2060 }
2061
2062 static void
2063 nc_printf_check_brief(mtev_console_closure_t ncct,
2064                       noit_check_t *check) {
2065   stats_t *current;
2066   char out[512];
2067   char uuid_str[37];
2068   snprintf(out, sizeof(out), "%s`%s (%s [%x])", check->target, check->name,
2069            check->target_ip, check->flags);
2070   uuid_unparse_lower(check->checkid, uuid_str);
2071   nc_printf(ncct, "%s %s\n", uuid_str, out);
2072   current = stats_current(check);
2073   if(current && current->status)
2074     nc_printf(ncct, "\t%s\n", current->status);
2075 }
2076
2077 char *
2078 noit_console_conf_check_opts(mtev_console_closure_t ncct,
2079                              mtev_console_state_stack_t *stack,
2080                              mtev_console_state_t *dstate,
2081                              int argc, char **argv, int idx) {
2082   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
2083   uuid_t key_id;
2084   int klen, i = 0;
2085   void *vcheck;
2086
2087   if(argc == 1) {
2088     if(!strncmp("new", argv[0], strlen(argv[0]))) {
2089       if(idx == i) return strdup("new");
2090       i++;
2091     }
2092     pthread_mutex_lock(&polls_lock);
2093     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
2094                          &vcheck)) {
2095       noit_check_t *check = (noit_check_t *)vcheck;
2096       char out[512];
2097       char uuid_str[37];
2098       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
2099       uuid_unparse_lower(check->checkid, uuid_str);
2100       if(!strncmp(out, argv[0], strlen(argv[0]))) {
2101         if(idx == i) {
2102           pthread_mutex_unlock(&polls_lock);
2103           return strdup(out);
2104         }
2105         i++;
2106       }
2107       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
2108         if(idx == i) {
2109           pthread_mutex_unlock(&polls_lock);
2110           return strdup(uuid_str);
2111         }
2112         i++;
2113       }
2114     }
2115     pthread_mutex_unlock(&polls_lock);
2116   }
2117   if(argc == 2) {
2118     cmd_info_t *cmd;
2119     if(!strcmp("new", argv[0])) return NULL;
2120     cmd = mtev_skiplist_find(&dstate->cmds, "attribute", NULL);
2121     if(!cmd) return NULL;
2122     return mtev_console_opt_delegate(ncct, stack, cmd->dstate, argc-1, argv+1, idx);
2123   }
2124   return NULL;
2125 }
2126
2127 char *
2128 noit_console_check_opts(mtev_console_closure_t ncct,
2129                         mtev_console_state_stack_t *stack,
2130                         mtev_console_state_t *dstate,
2131                         int argc, char **argv, int idx) {
2132   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
2133   uuid_t key_id;
2134   int klen, i = 0;
2135
2136   if(argc == 1) {
2137     void *vcheck;
2138     pthread_mutex_lock(&polls_lock);
2139     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
2140                          &vcheck)) {
2141       char out[512];
2142       char uuid_str[37];
2143       noit_check_t *check = (noit_check_t *)vcheck;
2144       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
2145       uuid_unparse_lower(check->checkid, uuid_str);
2146       if(!strncmp(out, argv[0], strlen(argv[0]))) {
2147         if(idx == i) {
2148           pthread_mutex_unlock(&polls_lock);
2149           return strdup(out);
2150         }
2151         i++;
2152       }
2153       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
2154         if(idx == i) {
2155           pthread_mutex_unlock(&polls_lock);
2156           return strdup(uuid_str);
2157         }
2158         i++;
2159       }
2160     }
2161     pthread_mutex_unlock(&polls_lock);
2162   }
2163   if(argc == 2) {
2164     return mtev_console_opt_delegate(ncct, stack, dstate, argc-1, argv+1, idx);
2165   }
2166   return NULL;
2167 }
2168
2169 static int
2170 noit_console_show_checks(mtev_console_closure_t ncct,
2171                          int argc, char **argv,
2172                          mtev_console_state_t *dstate,
2173                          void *closure) {
2174   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
2175   uuid_t key_id;
2176   int klen, i = 0, nchecks;
2177   void *vcheck;
2178   noit_check_t **checks;
2179
2180   nchecks = mtev_hash_size(&polls);
2181   if(nchecks == 0) return 0;
2182   checks = malloc(nchecks * sizeof(*checks));
2183
2184   pthread_mutex_lock(&polls_lock);
2185   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
2186                        &vcheck)) {
2187     if(i<nchecks) checks[i++] = vcheck;
2188   }
2189   pthread_mutex_unlock(&polls_lock);
2190
2191   nchecks = i;
2192   for(i=0;i<nchecks;i++)
2193     nc_printf_check_brief(ncct,checks[i]);
2194
2195   free(checks);
2196   return 0;
2197 }
2198
2199 static int
2200 noit_console_short_checks_sl(mtev_console_closure_t ncct,
2201                              mtev_skiplist *tlist) {
2202   int max_count, i = 0;
2203   noit_check_t **todo;
2204   mtev_skiplist_node *iter;
2205
2206   max_count = tlist->size;
2207   if(max_count == 0) return 0;
2208   todo = malloc(max_count * sizeof(*todo));
2209
2210   pthread_mutex_lock(&polls_lock);
2211   for(iter = mtev_skiplist_getlist(tlist); i < max_count && iter;
2212       mtev_skiplist_next(tlist, &iter)) {
2213     todo[i++] = iter->data;
2214   }
2215   pthread_mutex_unlock(&polls_lock);
2216
2217   max_count = i;
2218   for(i=0;i<max_count;i++)
2219     nc_printf_check_brief(ncct, todo[i]);
2220
2221   free(todo);
2222   return 0;
2223 }
2224 static int
2225 noit_console_show_checks_name(mtev_console_closure_t ncct,
2226                               int argc, char **argv,
2227                               mtev_console_state_t *dstate,
2228                               void *closure) {
2229   return noit_console_short_checks_sl(ncct, &polls_by_name);
2230 }
2231
2232 static int
2233 noit_console_show_checks_target(mtev_console_closure_t ncct,
2234                                    int argc, char **argv,
2235                                    mtev_console_state_t *dstate,
2236                                    void *closure) {
2237   return noit_console_short_checks_sl(ncct,
2238            mtev_skiplist_find(polls_by_name.index,
2239            __check_target_compare, NULL));
2240 }
2241
2242 static int
2243 noit_console_show_checks_target_ip(mtev_console_closure_t ncct,
2244                                    int argc, char **argv,
2245                                    mtev_console_state_t *dstate,
2246                                    void *closure) {
2247   return noit_console_short_checks_sl(ncct,
2248            mtev_skiplist_find(polls_by_name.index,
2249            __check_target_ip_compare, NULL));
2250 }
2251
2252 static void
2253 register_console_check_commands() {
2254   mtev_console_state_t *tl;
2255   cmd_info_t *showcmd;
2256
2257   tl = mtev_console_state_initial();
2258   showcmd = mtev_console_state_get_cmd(tl, "show");
2259   assert(showcmd && showcmd->dstate);
2260
2261   mtev_console_state_add_cmd(showcmd->dstate,
2262     NCSCMD("timing_slots", noit_console_show_timing_slots, NULL, NULL, NULL));
2263
2264   mtev_console_state_add_cmd(showcmd->dstate,
2265     NCSCMD("checks", noit_console_show_checks, NULL, NULL, NULL));
2266
2267   mtev_console_state_add_cmd(showcmd->dstate,
2268     NCSCMD("checks:name", noit_console_show_checks_name, NULL,
2269            NULL, NULL));
2270
2271   mtev_console_state_add_cmd(showcmd->dstate,
2272     NCSCMD("checks:target", noit_console_show_checks_target, NULL,
2273            NULL, NULL));
2274
2275   mtev_console_state_add_cmd(showcmd->dstate,
2276     NCSCMD("checks:target_ip", noit_console_show_checks_target_ip, NULL,
2277            NULL, NULL));
2278
2279   mtev_console_state_add_cmd(showcmd->dstate,
2280     NCSCMD("watches", noit_console_show_watchlist, NULL, NULL, NULL));
2281 }
2282
2283 int
2284 noit_check_register_module(const char *name) {
2285   int i;
2286   for(i=0; i<reg_module_id; i++)
2287     if(!strcmp(reg_module_names[i], name)) return i;
2288   if(reg_module_id >= MAX_MODULE_REGISTRATIONS) return -1;
2289   mtevL(noit_debug, "Registered module %s as %d\n", name, i);
2290   i = reg_module_id++;
2291   reg_module_names[i] = strdup(name);
2292   mtev_conf_set_namespace(reg_module_names[i]);
2293   return i;
2294 }
2295 int
2296 noit_check_registered_module_cnt() {
2297   return reg_module_id;
2298 }
2299 const char *
2300 noit_check_registered_module(int idx) {
2301   if(reg_module_used < 0) reg_module_used = reg_module_id;
2302   assert(reg_module_used == reg_module_id);
2303   if(idx >= reg_module_id || idx < 0) return NULL;
2304   return reg_module_names[idx];
2305 }
2306
2307 void
2308 noit_check_set_module_metadata(noit_check_t *c, int idx, void *md, void (*freefunc)(void *)) {
2309   struct vp_w_free *tuple;
2310   if(reg_module_used < 0) reg_module_used = reg_module_id;
2311   assert(reg_module_used == reg_module_id);
2312   if(idx >= reg_module_id || idx < 0) return;
2313   if(!c->module_metadata) c->module_metadata = calloc(reg_module_id, sizeof(void *));
2314   c->module_metadata[idx] = calloc(1, sizeof(struct vp_w_free));
2315   tuple = c->module_metadata[idx];
2316   tuple->ptr = md;
2317   tuple->freefunc = freefunc;
2318 }
2319 void
2320 noit_check_set_module_config(noit_check_t *c, int idx, mtev_hash_table *config) {
2321   if(reg_module_used < 0) reg_module_used = reg_module_id;
2322   assert(reg_module_used == reg_module_id);
2323   if(idx >= reg_module_id || idx < 0) return;
2324   if(!c->module_configs) c->module_configs = calloc(reg_module_id, sizeof(mtev_hash_table *));
2325   c->module_configs[idx] = config;
2326 }
2327 void *
2328 noit_check_get_module_metadata(noit_check_t *c, int idx) {
2329   struct vp_w_free *tuple;
2330   if(reg_module_used < 0) reg_module_used = reg_module_id;
2331   assert(reg_module_used == reg_module_id);
2332   if(idx >= reg_module_id || idx < 0 || !c->module_metadata) return NULL;
2333   tuple = c->module_metadata[idx];
2334   return tuple ? tuple->ptr : NULL;
2335 }
2336 mtev_hash_table *
2337 noit_check_get_module_config(noit_check_t *c, int idx) {
2338   if(reg_module_used < 0) reg_module_used = reg_module_id;
2339   assert(reg_module_used == reg_module_id);
2340   if(idx >= reg_module_id || idx < 0 || !c->module_configs) return NULL;
2341   return c->module_configs[idx];
2342 }
Note: See TracBrowser for help on using the browser.