root/src/noit_check.c

Revision e777c361cd646308f382dc6e94c33adc3d8efda0, 75.1 kB (checked in by GitHub <noreply@github.com>, 2 months ago)

Websockets and externalization of histogram code into libcircllhist (#278)

* WIP, livestreaming over websocket

* fix livestream mtev_log_stream_t leak

* move histogram impl out of reconnoiter tree into libcircllhist repo, tests for websockets

* whitespace cleanup

* fix comment

* output any websocket errors as json

* fix integer names

  • Property mode set to 100644
Line 
1 /*
2  * Copyright (c) 2007, OmniTI Computer Consulting, Inc.
3  * All rights reserved.
4  * Copyright (c) 2015, Circonus, Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are
8  * met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above
13  *       copyright notice, this list of conditions and the following
14  *       disclaimer in the documentation and/or other materials provided
15  *       with the distribution.
16  *     * Neither the name OmniTI Computer Consulting, Inc. nor the names
17  *       of its contributors may be used to endorse or promote products
18  *       derived from this software without specific prior written
19  *       permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include "noit_config.h"
35 #include <mtev_defines.h>
36
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <unistd.h>
40 #include <ctype.h>
41 #include <errno.h>
42 #include <netinet/in.h>
43 #include <arpa/inet.h>
44 #include <time.h>
45
46 #include <eventer/eventer.h>
47 #include <mtev_memory.h>
48 #include <mtev_log.h>
49 #include <mtev_hash.h>
50 #include <mtev_skiplist.h>
51 #include <mtev_watchdog.h>
52 #include <mtev_conf.h>
53 #include <mtev_console.h>
54 #include <mtev_cluster.h>
55
56 #include "noit_mtev_bridge.h"
57 #include "noit_dtrace_probes.h"
58 #include "noit_check.h"
59 #include "noit_module.h"
60 #include "noit_check_tools.h"
61 #include "noit_check_resolver.h"
62
63 #define DEFAULT_TEXT_METRIC_SIZE_LIMIT  512
64 #define RECYCLE_INTERVAL 60
65
66 MTEV_HOOK_IMPL(check_config_fixup,
67   (noit_check_t *check),
68   void *, closure,
69   (void *closure, noit_check_t *check),
70   (closure,check))
71
72 MTEV_HOOK_IMPL(check_stats_set_metric,
73   (noit_check_t *check, stats_t *stats, metric_t *m),
74   void *, closure,
75   (void *closure, noit_check_t *check, stats_t *stats, metric_t *m),
76   (closure,check,stats,m))
77
78 MTEV_HOOK_IMPL(check_stats_set_metric_coerce,
79   (noit_check_t *check, stats_t *stats, const char *name,
80    metric_type_t type, const char *v, mtev_boolean success),
81   void *, closure,
82   (void *closure, noit_check_t *check, stats_t *stats, const char *name,
83    metric_type_t type, const char *v, mtev_boolean success),
84   (closure,check,stats,name,type,v,success))
85
86 MTEV_HOOK_IMPL(check_passive_log_stats,
87   (noit_check_t *check),
88   void *, closure,
89   (void *closure, noit_check_t *check),
90   (closure,check))
91
92 MTEV_HOOK_IMPL(check_set_stats,
93   (noit_check_t *check),
94   void *, closure,
95   (void *closure, noit_check_t *check),
96   (closure,check))
97
98 MTEV_HOOK_IMPL(check_log_stats,
99   (noit_check_t *check),
100   void *, closure,
101   (void *closure, noit_check_t *check),
102   (closure,check))
103
104 #define STATS_INPROGRESS 0
105 #define STATS_CURRENT 1
106 #define STATS_PREVIOUS 2
107
108 void
109 free_metric(metric_t *m) {
110   if(m->metric_name) free(m->metric_name);
111   if(m->metric_value.i) free(m->metric_value.i);
112 }
113
114 #define stats_inprogress(c) ((stats_t **)(c->statistics))[STATS_INPROGRESS]
115 #define stats_current(c) ((stats_t **)(c->statistics))[STATS_CURRENT]
116 #define stats_previous(c) ((stats_t **)(c->statistics))[STATS_PREVIOUS]
117
118 stats_t *
119 noit_check_get_stats_inprogress(noit_check_t *c) {
120   return stats_inprogress(c);
121 }
122 stats_t *
123 noit_check_get_stats_current(noit_check_t *c) {
124   return stats_current(c);
125 }
126 stats_t *
127 noit_check_get_stats_previous(noit_check_t *c) {
128   return stats_previous(c);
129 }
130
131 struct stats_t {
132   struct timeval whence;
133   int8_t available;
134   int8_t state;
135   u_int32_t duration;
136   mtev_hash_table metrics;
137   char status[256];
138 };
139
140 struct timeval *
141 noit_check_stats_whence(stats_t *s, struct timeval *n) {
142   if(n) memcpy(&s->whence, n, sizeof(*n));
143   return &s->whence;
144 }
145 int8_t
146 noit_check_stats_available(stats_t *s, int8_t *n) {
147   if(n) s->available = *n;
148   return s->available;
149 }
150 int8_t
151 noit_check_stats_state(stats_t *s, int8_t *n) {
152   if(n) s->state = *n;
153   return s->state;
154 }
155 u_int32_t
156 noit_check_stats_duration(stats_t *s, u_int32_t *n) {
157   if(n) s->duration = *n;
158   return s->duration;
159 }
160 const char *
161 noit_check_stats_status(stats_t *s, const char *n) {
162   if(n) strlcpy(s->status, n, sizeof(s->status));
163   return s->status;
164 }
165 mtev_hash_table *
166 noit_check_stats_metrics(stats_t *s) {
167   return &s->metrics;
168 }
169 void
170 noit_stats_set_whence(noit_check_t *c, struct timeval *t) {
171   (void)noit_check_stats_whence(noit_check_get_stats_inprogress(c), t);
172 }
173 void
174 noit_stats_set_state(noit_check_t *c, int8_t t) {
175   (void)noit_check_stats_state(noit_check_get_stats_inprogress(c), &t);
176 }
177 void
178 noit_stats_set_duration(noit_check_t *c, u_int32_t t) {
179   (void)noit_check_stats_duration(noit_check_get_stats_inprogress(c), &t);
180 }
181 void
182 noit_stats_set_status(noit_check_t *c, const char *s) {
183   (void)noit_check_stats_status(noit_check_get_stats_inprogress(c), s);
184 }
185 void
186 noit_stats_set_available(noit_check_t *c, int8_t t) {
187   (void)noit_check_stats_available(noit_check_get_stats_inprogress(c), &t);
188 }
189 static void
190 noit_check_safe_free_metric(void *vs) {
191   metric_t *m = vs;
192   if (m) {
193     free_metric(m);
194   }
195 }
196 static void
197 noit_check_safe_free_stats(void *vs) {
198   stats_t *s = vs;
199   mtev_hash_destroy(&s->metrics, NULL, (void (*)(void *))mtev_memory_safe_free);
200 }
201 static stats_t *
202 noit_check_stats_alloc() {
203   stats_t *n;
204   n = mtev_memory_safe_malloc_cleanup(sizeof(*n), noit_check_safe_free_stats);
205   memset(n, 0, sizeof(*n));
206   mtev_hash_init_locks(&n->metrics, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX);
207   return n;
208 }
209 static void *
210 noit_check_stats_set_calloc() {
211   int i;
212   stats_t **s;
213   s = calloc(sizeof(stats_t *), 3);
214   for(i=0;i<3;i++) s[i] = noit_check_stats_alloc();
215   return s;
216 }
217
218 /* 20 ms slots over 60 second for distribution */
219 #define SCHEDULE_GRANULARITY 20
220 #define SLOTS_PER_SECOND (1000/SCHEDULE_GRANULARITY)
221 #define MAX_MODULE_REGISTRATIONS 64
222
223 /* used to manage per-check generic module metadata */
224 struct vp_w_free {
225   void *ptr;
226   void (*freefunc)(void *);
227 };
228
229 static mtev_boolean system_needs_causality = mtev_false;
230 static int text_size_limit = DEFAULT_TEXT_METRIC_SIZE_LIMIT;
231 static int reg_module_id = 0;
232 static char *reg_module_names[MAX_MODULE_REGISTRATIONS] = { NULL };
233 static int reg_module_used = -1;
234 static u_int64_t check_completion_count = 0ULL;
235 static u_int64_t check_metrics_seen = 0ULL;
236 static pthread_mutex_t polls_lock = PTHREAD_MUTEX_INITIALIZER;
237 static pthread_mutex_t recycling_lock = PTHREAD_MUTEX_INITIALIZER;
238 static mtev_hash_table polls;
239 static mtev_hash_table dns_ignore_list;
240 static mtev_skiplist watchlist = { 0 };
241 static mtev_skiplist polls_by_name = { 0 };
242 static u_int32_t __config_load_generation = 0;
243 static unsigned short check_slots_count[60000 / SCHEDULE_GRANULARITY] = { 0 },
244                       check_slots_seconds_count[60] = { 0 };
245 static mtev_boolean priority_scheduling = mtev_false;
246 static int priority_dead_zone_seconds = 3;
247
248 static noit_check_t *
249 noit_poller_lookup__nolock(uuid_t in) {
250   void *vcheck;
251   if(mtev_hash_retrieve(&polls, (char *)in, UUID_SIZE, &vcheck))
252     return (noit_check_t *)vcheck;
253   return NULL;
254 }
255 static noit_check_t *
256 noit_poller_lookup_by_name__nolock(char *target, char *name) {
257   noit_check_t tmp_check;
258   memset(&tmp_check, 0, sizeof(tmp_check));
259   tmp_check.target = target;
260   tmp_check.name = name;
261   return mtev_skiplist_find(&polls_by_name, &tmp_check, NULL);
262 }
263
264 static int
265 noit_console_show_timing_slots(mtev_console_closure_t ncct,
266                                int argc, char **argv,
267                                mtev_console_state_t *dstate,
268                                void *closure) {
269   int i, j;
270   const int upl = (60000 / SCHEDULE_GRANULARITY) / 60;
271   for(i=0;i<60;i++) {
272     nc_printf(ncct, "[%02d] %04d: ", i, check_slots_seconds_count[i]);
273     for(j=i*upl;j<(i+1)*upl;j++) {
274       char cp = '!';
275       if(check_slots_count[j] < 10) cp = '0' + check_slots_count[j];
276       else if(check_slots_count[j] < 36) cp = 'a' + (check_slots_count[j] - 10);
277       nc_printf(ncct, "%c", cp);
278     }
279     nc_printf(ncct, "\n");
280   }
281   return 0;
282 }
283 static int
284 noit_check_add_to_list(noit_check_t *new_check, const char *newname) {
285   char *oldname = NULL, *newnamecopy;
286   if(newname) {
287     /* track this stuff outside the lock to avoid allocs */
288     oldname = new_check->name;
289     newnamecopy = strdup(newname);
290   }
291   pthread_mutex_lock(&polls_lock);
292   if(!(new_check->flags & NP_TRANSIENT)) {
293     mtevAssert(new_check->name || newname);
294     /* This remove could fail -- no big deal */
295     if(new_check->name != NULL)
296       mtev_skiplist_remove(&polls_by_name, new_check, NULL);
297
298     /* optional update the name (at the critical point) */
299     if(newname) new_check->name = newnamecopy;
300
301     /* This insert could fail.. which means we have a conflict on
302      * target`name.  That should result in the check being disabled. */
303     if(!mtev_skiplist_insert(&polls_by_name, new_check)) {
304       mtevL(noit_error, "Check %s`%s disabled due to naming conflict\n",
305             new_check->target, new_check->name);
306       new_check->flags |= NP_DISABLED;
307     }
308     if(oldname) free(oldname);
309   }
310   pthread_mutex_unlock(&polls_lock);
311   return 1;
312 }
313
314 u_int64_t noit_check_metric_count() {
315   return check_metrics_seen;
316 }
317 void noit_check_metric_count_add(int add) {
318   mtev_atomic64_t *n = (mtev_atomic64_t *)&check_metrics_seen;
319   mtev_atomic64_t v = (mtev_atomic64_t)add;
320   mtev_atomic_add64(n, v);
321 }
322
323 u_int64_t noit_check_completion_count() {
324   return check_completion_count;
325 }
326 static void register_console_check_commands();
327 static int check_recycle_bin_processor(eventer_t, int, void *,
328                                        struct timeval *);
329
330 static int
331 check_slots_find_smallest(int sec, struct timeval* period, int timeout) {
332   int i, j, cyclic, random_offset, jbase = 0, mini = 0, minj = 0;
333   unsigned short min_running_i = 0xffff, min_running_j = 0xffff;
334   int period_seconds = period->tv_sec;
335
336   /* If we're greater than sixty seconds, we should do our
337    * initial scheduling as if the period was sixty seconds. */
338   if (period_seconds > 60) {
339     period_seconds = 60;
340   }
341
342   /* If a check is configured to run at times aligned with sixty seconds
343    * and we're configured to use priority scheduling, schedule so that
344    * we're guaranteed to finish before the timeout */
345   if ((priority_scheduling == mtev_true) &&
346       (((period->tv_sec % 60) == 0) && (period->tv_usec == 0))) {
347     /* Don't allow a ton of stuff to schedule in the first second in the case
348      * of very long timeouts - use the first 10 seconds in this case */
349     int allowable_time = MAX(60 - (timeout/1000) - 1, 10);
350     int max_seconds = MIN(60-priority_dead_zone_seconds, allowable_time);
351     for(i=0;i<max_seconds;i++) {
352       int adj_i = (i + sec) % max_seconds;
353       if(check_slots_seconds_count[adj_i] < min_running_i) {
354         min_running_i = check_slots_seconds_count[adj_i];
355         mini = adj_i;
356       }
357     }
358   }
359   else {
360     /* Just schedule normally*/
361     for(i=0;i<period_seconds;i++) {
362       int adj_i = (i + sec) % 60;
363       if(check_slots_seconds_count[adj_i] < min_running_i) {
364         min_running_i = check_slots_seconds_count[adj_i];
365         mini = adj_i;
366       }
367     }
368   }
369   jbase = mini * (1000/SCHEDULE_GRANULARITY);
370   random_offset = drand48() * SLOTS_PER_SECOND;
371   for(cyclic=0;cyclic<SLOTS_PER_SECOND;cyclic++) {
372     j = jbase + ((random_offset + cyclic) % SLOTS_PER_SECOND);
373     if(check_slots_count[j] < min_running_j) {
374       min_running_j = check_slots_count[j];
375       minj = j;
376     }
377   }
378   return (minj * SCHEDULE_GRANULARITY) + drand48() * SCHEDULE_GRANULARITY;
379 }
380 static void
381 check_slots_adjust_tv(struct timeval *tv, short adj) {
382   int offset_ms, idx;
383   offset_ms = (tv->tv_sec % 60) * 1000 + (tv->tv_usec / 1000);
384   idx = offset_ms / SCHEDULE_GRANULARITY;
385   check_slots_count[idx] += adj;
386   check_slots_seconds_count[offset_ms / 1000] += adj;
387 }
388 void check_slots_inc_tv(struct timeval *tv) {
389   check_slots_adjust_tv(tv, 1);
390 }
391 void check_slots_dec_tv(struct timeval *tv) {
392   check_slots_adjust_tv(tv, -1);
393 }
394 static int
395 noit_check_generic_safe_string(const char *p) {
396   if(!p) return 0;
397   for(;*p;p++) {
398     if(!isprint(*p)) return 0;
399   }
400   return 1;
401 }
402 int
403 noit_check_validate_target(const char *p) {
404   if(!noit_check_generic_safe_string(p)) return 0;
405   return 1;
406 }
407 int
408 noit_check_validate_name(const char *p) {
409   if(!noit_check_generic_safe_string(p)) return 0;
410   return 1;
411 }
412 const char *
413 noit_check_available_string(int16_t available) {
414   switch(available) {
415     case NP_AVAILABLE:    return "available";
416     case NP_UNAVAILABLE:  return "unavailable";
417     case NP_UNKNOWN:      return "unknown";
418   }
419   return NULL;
420 }
421 const char *
422 noit_check_state_string(int16_t state) {
423   switch(state) {
424     case NP_GOOD:         return "good";
425     case NP_BAD:          return "bad";
426     case NP_UNKNOWN:      return "unknown";
427   }
428   return NULL;
429 }
430 static int __check_name_compare(const void *a, const void *b) {
431   const noit_check_t *ac = a;
432   const noit_check_t *bc = b;
433   int rv;
434   if((rv = strcmp(ac->target, bc->target)) != 0) return rv;
435   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
436   return 0;
437 }
438 static int __watchlist_compare(const void *a, const void *b) {
439   const noit_check_t *ac = a;
440   const noit_check_t *bc = b;
441   int rv;
442   if((rv = memcmp(ac->checkid, bc->checkid, sizeof(ac->checkid))) != 0) return rv;
443   if(ac->period < bc->period) return -1;
444   if(ac->period == bc->period) return 0;
445   return 1;
446 }
447 static int __check_target_ip_compare(const void *a, const void *b) {
448   const noit_check_t *ac = a;
449   const noit_check_t *bc = b;
450   int rv;
451   if((rv = strcmp(ac->target_ip, bc->target_ip)) != 0) return rv;
452   if (ac->name == NULL) return 1;
453   if (bc->name == NULL) return -1;
454   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
455   return 1;
456 }
457 static int __check_target_compare(const void *a, const void *b) {
458   const noit_check_t *ac = a;
459   const noit_check_t *bc = b;
460   int rv;
461   if (ac->target == NULL) return 1;
462   if (bc->target == NULL) return -1;
463   if((rv = strcmp(ac->target, bc->target)) != 0) return rv;
464   if (ac->name == NULL) return 1;
465   if (bc->name == NULL) return -1;
466   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
467   return 1;
468 }
469 int
470 noit_calc_rtype_flag(char *resolve_rtype) {
471   int flags = 0;
472   if(resolve_rtype) {
473     flags |= strcmp(resolve_rtype, PREFER_IPV6) == 0 ||
474              strcmp(resolve_rtype, FORCE_IPV6) == 0 ? NP_PREFER_IPV6 : 0;
475     flags |= strcmp(resolve_rtype, FORCE_IPV4) == 0 ||
476              strcmp(resolve_rtype, FORCE_IPV6) == 0 ? NP_SINGLE_RESOLVE : 0;
477   }
478   return flags;
479 }
480 void
481 noit_check_fake_last_check(noit_check_t *check,
482                            struct timeval *lc, struct timeval *_now) {
483   struct timeval now, period, lc_copy;
484   int balance_ms;
485
486   if(!_now) {
487     gettimeofday(&now, NULL);
488     _now = &now;
489   }
490   period.tv_sec = check->period / 1000;
491   period.tv_usec = (check->period % 1000) * 1000;
492   sub_timeval(*_now, period, lc);
493
494   /* We need to set the last check value based on the period, but
495    * we also need to store a value that is based around the one-minute
496    * time to properly increment the slots; otherwise, the slots will
497    * get all messed up */
498   if(!(check->flags & NP_TRANSIENT) && check->period) {
499     balance_ms = check_slots_find_smallest(_now->tv_sec+1, &period, check->timeout);
500     lc->tv_sec = (lc->tv_sec / 60) * 60 + balance_ms / 1000;
501     lc->tv_usec = (balance_ms % 1000) * 1000;
502     memcpy(&lc_copy, lc, sizeof(lc_copy));
503     if(compare_timeval(*_now, *lc) < 0) {
504       do {
505         sub_timeval(*lc, period, lc);
506       } while(compare_timeval(*_now, *lc) < 0);
507     }
508     else {
509       struct timeval test;
510       while(1) {
511         add_timeval(*lc, period, &test);
512         if(compare_timeval(*_now, test) < 0) break;
513         memcpy(lc, &test, sizeof(test));
514       }
515     }
516   }
517   else {
518     memcpy(&lc_copy, lc, sizeof(lc_copy));
519   }
520  
521   /* now, we're going to do an even distribution using the slots */
522   if(!(check->flags & NP_TRANSIENT)) check_slots_inc_tv(&lc_copy);
523 }
524 void
525 noit_poller_process_checks(const char *xpath) {
526   int i, flags, cnt = 0, found;
527   mtev_conf_section_t *sec;
528   __config_load_generation++;
529   sec = mtev_conf_get_sections(NULL, xpath, &cnt);
530   for(i=0; i<cnt; i++) {
531     void *vcheck;
532     char uuid_str[37];
533     char target[256] = "";
534     char module[256] = "";
535     char name[256] = "";
536     char filterset[256] = "";
537     char oncheck[1024] = "";
538     char resolve_rtype[16] = "";
539     int ridx;
540     int no_period = 0;
541     int no_oncheck = 0;
542     int period = 0, timeout = 0;
543     mtev_boolean disabled = mtev_false, busted = mtev_false;
544     uuid_t uuid, out_uuid;
545     int64_t config_seq = 0;
546     mtev_hash_table *options;
547     mtev_hash_table **moptions = NULL;
548     mtev_boolean moptions_used = mtev_false, backdated = mtev_false;
549
550     /* We want to heartbeat here... otherwise, if a lot of checks are
551      * configured or if we're running on a slower system, we could
552      * end up getting watchdog killed before we get a chance to run
553      * any checks */
554     mtev_watchdog_child_heartbeat();
555
556     if(reg_module_id > 0) {
557       moptions = alloca(reg_module_id * sizeof(mtev_hash_table *));
558       memset(moptions, 0, reg_module_id * sizeof(mtev_hash_table *));
559       moptions_used = mtev_true;
560     }
561
562 #define NEXT(...) mtevL(noit_stderr, __VA_ARGS__); continue
563 #define MYATTR(type,a,...) mtev_conf_get_##type(sec[i], "@" #a, __VA_ARGS__)
564 #define INHERIT(type,a,...) \
565   mtev_conf_get_##type(sec[i], "ancestor-or-self::node()/@" #a, __VA_ARGS__)
566
567     if(!MYATTR(stringbuf, uuid, uuid_str, sizeof(uuid_str))) {
568       mtevL(noit_stderr, "check %d has no uuid\n", i+1);
569       continue;
570     }
571
572     MYATTR(int64, seq, &config_seq);
573
574     if(uuid_parse(uuid_str, uuid)) {
575       mtevL(noit_stderr, "check uuid: '%s' is invalid\n", uuid_str);
576       continue;
577     }
578
579     if(!INHERIT(stringbuf, target, target, sizeof(target))) {
580       mtevL(noit_stderr, "check uuid: '%s' has no target\n", uuid_str);
581       busted = mtev_true;
582     }
583     if(!noit_check_validate_target(target)) {
584       mtevL(noit_stderr, "check uuid: '%s' has malformed target\n", uuid_str);
585       busted = mtev_true;
586     }
587     if(!INHERIT(stringbuf, module, module, sizeof(module))) {
588       mtevL(noit_stderr, "check uuid: '%s' has no module\n", uuid_str);
589       busted = mtev_true;
590     }
591
592     if(!INHERIT(stringbuf, filterset, filterset, sizeof(filterset)))
593       filterset[0] = '\0';
594    
595     if (!INHERIT(stringbuf, resolve_rtype, resolve_rtype, sizeof(resolve_rtype)))
596       strlcpy(resolve_rtype, PREFER_IPV4, sizeof(resolve_rtype));
597
598     if(!MYATTR(stringbuf, name, name, sizeof(name)))
599       strlcpy(name, module, sizeof(name));
600
601     if(!noit_check_validate_name(name)) {
602       mtevL(noit_stderr, "check uuid: '%s' has malformed name\n", uuid_str);
603       busted = mtev_true;
604     }
605
606     if(!INHERIT(int, period, &period) || period == 0)
607       no_period = 1;
608
609     if(!INHERIT(stringbuf, oncheck, oncheck, sizeof(oncheck)) || !oncheck[0])
610       no_oncheck = 1;
611
612     if(no_period && no_oncheck) {
613       mtevL(noit_stderr, "check uuid: '%s' has neither period nor oncheck\n",
614             uuid_str);
615       busted = mtev_true;
616     }
617     if(!(no_period || no_oncheck)) {
618       mtevL(noit_stderr, "check uuid: '%s' has oncheck and period.\n",
619             uuid_str);
620       busted = mtev_true;
621     }
622     if(!INHERIT(int, timeout, &timeout)) {
623       mtevL(noit_stderr, "check uuid: '%s' has no timeout\n", uuid_str);
624       busted = mtev_true;
625     }
626     if(!no_period && timeout >= period) {
627       mtevL(noit_stderr, "check uuid: '%s' timeout > period\n", uuid_str);
628       timeout = period/2;
629     }
630     options = mtev_conf_get_hash(sec[i], "config");
631     for(ridx=0; ridx<reg_module_id; ridx++) {
632       moptions[ridx] = mtev_conf_get_namespaced_hash(sec[i], "config",
633                                                      reg_module_names[ridx]);
634     }
635
636     INHERIT(boolean, disable, &disabled);
637     flags = 0;
638     if(busted) flags |= (NP_UNCONFIG|NP_DISABLED);
639     else if(disabled) flags |= NP_DISABLED;
640
641     flags |= noit_calc_rtype_flag(resolve_rtype);
642
643     pthread_mutex_lock(&polls_lock);
644     found = mtev_hash_retrieve(&polls, (char *)uuid, UUID_SIZE, &vcheck);
645     if(found) {
646       noit_check_t *check = (noit_check_t *)vcheck;
647       /* Possibly reset the seq */
648       if(config_seq < 0) check->config_seq = 0;
649
650       /* Otherwise note a non-increasing sequence */
651       if(check->config_seq > config_seq) backdated = mtev_true;
652     }
653     pthread_mutex_unlock(&polls_lock);
654     if(found)
655       noit_poller_deschedule(uuid, mtev_false);
656     if(backdated) {
657       mtevL(noit_error, "Check config seq backwards, ignored\n");
658       if(found) noit_check_log_delete((noit_check_t *)vcheck);
659     }
660     else {
661       noit_poller_schedule(target, module, name, filterset, options,
662                            moptions_used ? moptions : NULL,
663                            period, timeout, oncheck[0] ? oncheck : NULL,
664                            config_seq, flags, uuid, out_uuid);
665       mtevL(noit_debug, "loaded uuid: %s\n", uuid_str);
666     }
667     for(ridx=0; ridx<reg_module_id; ridx++) {
668       if(moptions[ridx]) {
669         mtev_hash_destroy(moptions[ridx], free, free);
670         free(moptions[ridx]);
671       }
672     }
673     mtev_hash_destroy(options, free, free);
674     free(options);
675   }
676   if(sec) free(sec);
677 }
678
679 int
680 noit_check_activate(noit_check_t *check) {
681   noit_module_t *mod;
682   if(NOIT_CHECK_LIVE(check)) return 0;
683   mod = noit_module_lookup(check->module);
684   if(mod && mod->initiate_check) {
685     if((check->flags & NP_DISABLED) == 0) {
686       mod->initiate_check(mod, check, 0, NULL);
687       return 1;
688     }
689     else
690       mtevL(noit_debug, "Skipping %s`%s, disabled.\n",
691             check->target, check->name);
692   }
693   else {
694     if(!mod) {
695       mtevL(noit_stderr, "Cannot find module '%s'\n", check->module);
696       check->flags |= NP_DISABLED;
697     }
698   }
699   return 0;
700 }
701
702 void
703 noit_poller_initiate() {
704   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
705   uuid_t key_id;
706   int klen;
707   void *vcheck;
708   /* This is only ever called in the beginning, no lock needed */
709   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
710                        &vcheck)) {
711     noit_check_activate((noit_check_t *)vcheck);
712     mtev_watchdog_child_heartbeat();
713   }
714 }
715
716 void
717 noit_poller_flush_epoch(int oldest_allowed) {
718   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
719   uuid_t key_id;
720   int klen, i;
721   void *vcheck;
722 #define TOFREE_PER_ITER 1024
723   noit_check_t *tofree[TOFREE_PER_ITER];
724
725   /* Cleanup any previous causal map */
726   while(1) {
727     i = 0;
728     pthread_mutex_lock(&polls_lock);
729     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
730                          &vcheck) && i < TOFREE_PER_ITER) {
731       noit_check_t *check = (noit_check_t *)vcheck;
732       if(check->generation < oldest_allowed) {
733         tofree[i++] = check;
734       }
735     }
736     pthread_mutex_unlock(&polls_lock);
737     if(i==0) break;
738     while(i>0) noit_poller_deschedule(tofree[--i]->checkid, mtev_true);
739   }
740 #undef TOFREE_PER_ITER
741 }
742
743 void
744 noit_poller_make_causal_map() {
745   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
746   uuid_t key_id;
747   int klen;
748   void *vcheck;
749
750   if(!system_needs_causality) return;
751
752   /* set it to false, we'll set it to true during the scan if we
753    * find anything causal.  */
754   system_needs_causality = mtev_false;
755
756   /* Cleanup any previous causal map */
757   pthread_mutex_lock(&polls_lock);
758   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
759                        &vcheck)) {
760     noit_check_t *check = (noit_check_t *)vcheck;
761     dep_list_t *dep;
762     while((dep = check->causal_checks) != NULL) {
763       check->causal_checks = dep->next;
764       free(dep);
765     }
766   }
767
768   memset(&iter, 0, sizeof(iter));
769   /* Walk all checks and add check dependencies to their parents */
770   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
771                        &vcheck)) {
772     noit_check_t *check = (noit_check_t *)vcheck, *parent;
773     if(check->oncheck) {
774       /* This service is causally triggered by another service */
775       uuid_t id;
776       char fullcheck[1024];
777       char *name = check->oncheck;
778       char *target = NULL;
779
780       system_needs_causality = mtev_true;
781       mtevL(noit_debug, "Searching for upstream trigger on %s\n", name);
782       parent = NULL;
783       if(uuid_parse(check->oncheck, id) == 0) {
784         target = "";
785         parent = noit_poller_lookup__nolock(id);
786       }
787       else if((target = strchr(check->oncheck, '`')) != NULL) {
788         strlcpy(fullcheck, check->oncheck, target + 1 - check->oncheck);
789         name = target + 1;
790         target = fullcheck;
791         parent = noit_poller_lookup_by_name__nolock(target, name);
792       }
793       else {
794         target = check->target;
795         parent = noit_poller_lookup_by_name__nolock(target, name);
796       }
797
798       if(!parent) {
799         check->flags |= NP_DISABLED;
800         mtevL(noit_stderr, "Disabling check %s`%s, can't find oncheck %s`%s\n",
801               check->target, check->name, target, name);
802       }
803       else {
804         dep_list_t *dep;
805         dep = malloc(sizeof(*dep));
806         dep->check = check;
807         dep->next = parent->causal_checks;
808         parent->causal_checks = dep;
809         mtevL(noit_debug, "Causal map %s`%s --> %s`%s\n",
810               parent->target, parent->name, check->target, check->name);
811       }
812     }
813   }
814   pthread_mutex_unlock(&polls_lock);
815   /* We found some causal checks, so we might need to activate stuff */
816   if(system_needs_causality) noit_poller_initiate();
817 }
818 void
819 noit_poller_reload(const char *xpath)
820 {
821   noit_poller_process_checks(xpath ? xpath : "/noit/checks//check");
822   if(!xpath) {
823     /* Full reload, we need to wipe old checks */
824     noit_poller_flush_epoch(__config_load_generation);
825   }
826   noit_poller_make_causal_map();
827 }
828 void
829 noit_check_dns_ignore_tld(const char* extension, const char* ignore) {
830   mtev_hash_replace(&dns_ignore_list, strdup(extension), strlen(extension), strdup(ignore), NULL, NULL);
831 }
832 static void
833 noit_check_dns_ignore_list_init() {
834   mtev_conf_section_t* dns;
835   int cnt;
836
837   dns = mtev_conf_get_sections(NULL, "/noit/dns/extension", &cnt);
838   if(dns) {
839     int i = 0;
840     for (i = 0; i < cnt; i++) {
841       char* extension;
842       char* ignore;
843       if(!mtev_conf_get_string(dns[i], "self::node()/@value", &extension)) {
844         continue;
845       }
846       if(!mtev_conf_get_string(dns[i], "self::node()/@ignore", &ignore)) {
847         continue;
848       }
849       noit_check_dns_ignore_tld(extension, ignore);
850     }
851   }
852 }
853 static void
854 noit_check_poller_scheduling_init() {
855   mtev_conf_get_boolean(NULL, "//checks/@priority_scheduling", &priority_scheduling);
856 }
857 void
858 noit_poller_init() {
859   srand48((getpid() << 16) ^ time(NULL));
860   noit_check_poller_scheduling_init();
861   noit_check_resolver_init();
862   noit_check_tools_init();
863   mtev_skiplist_init(&polls_by_name);
864   mtev_skiplist_set_compare(&polls_by_name, __check_name_compare,
865                             __check_name_compare);
866   mtev_skiplist_add_index(&polls_by_name, __check_target_ip_compare,
867                             __check_target_ip_compare);
868   mtev_skiplist_add_index(&polls_by_name, __check_target_compare,
869                             __check_target_compare);
870   mtev_skiplist_init(&watchlist);
871   mtev_skiplist_set_compare(&watchlist, __watchlist_compare,
872                             __watchlist_compare);
873   register_console_check_commands();
874   eventer_name_callback("check_recycle_bin_processor",
875                         check_recycle_bin_processor);
876   eventer_add_in_s_us(check_recycle_bin_processor, NULL, RECYCLE_INTERVAL, 0);
877   mtev_conf_get_int(NULL, "noit/@text_size_limit", &text_size_limit);
878   if (text_size_limit <= 0) {
879     text_size_limit = DEFAULT_TEXT_METRIC_SIZE_LIMIT;
880   }
881   noit_check_dns_ignore_list_init();
882   noit_poller_reload(NULL);
883 }
884
885 int
886 noit_poller_check_count() {
887   return polls_by_name.size;
888 }
889
890 int
891 noit_poller_transient_check_count() {
892   return watchlist.size;
893 }
894
895 noit_check_t *
896 noit_check_clone(uuid_t in) {
897   int i;
898   noit_check_t *checker, *new_check;
899   void *vcheck;
900   if(mtev_hash_retrieve(&polls,
901                         (char *)in, UUID_SIZE,
902                         &vcheck) == 0) {
903     return NULL;
904   }
905   checker = (noit_check_t *)vcheck;
906   if(checker->oncheck) {
907     return NULL;
908   }
909   new_check = mtev_memory_safe_calloc(1, sizeof(*new_check));
910   mtevAssert(new_check != NULL);
911   memcpy(new_check, checker, sizeof(*new_check));
912   new_check->target = strdup(new_check->target);
913   new_check->module = strdup(new_check->module);
914   new_check->name = strdup(new_check->name);
915   new_check->filterset = strdup(new_check->filterset);
916   new_check->flags = 0;
917   new_check->fire_event = NULL;
918   memset(&new_check->last_fire_time, 0, sizeof(new_check->last_fire_time));
919   new_check->statistics = noit_check_stats_set_calloc();
920   new_check->closure = NULL;
921   new_check->config = calloc(1, sizeof(*new_check->config));
922   mtev_hash_init_locks(new_check->config, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX);
923   mtev_hash_merge_as_dict(new_check->config, checker->config);
924   new_check->module_configs = NULL;
925   new_check->module_metadata = NULL;
926
927   for(i=0; i<reg_module_id; i++) {
928     void *src_metadata;
929     mtev_hash_table *src_mconfig;
930     src_mconfig = noit_check_get_module_config(checker, i);
931     if(src_mconfig) {
932       mtev_hash_table *t = calloc(1, sizeof(*new_check->config));
933       mtev_hash_init_locks(t, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX);
934       mtev_hash_merge_as_dict(t, src_mconfig);
935       noit_check_set_module_config(new_check, i, t);
936     }
937     if(checker->flags & NP_PASSIVE_COLLECTION)
938       if(NULL != (src_metadata = noit_check_get_module_metadata(new_check, i)))
939         noit_check_set_module_metadata(new_check, i, src_metadata, NULL);
940   }
941   return new_check;
942 }
943
944 noit_check_t *
945 noit_check_watch(uuid_t in, int period) {
946   /* First look for a copy that is being watched */
947   int minimum_pi = 1000, granularity_pi = 500;
948   mtev_conf_section_t check_node;
949   char uuid_str[UUID_STR_LEN + 1];
950   char xpath[1024];
951   noit_check_t n, *f;
952
953   uuid_unparse_lower(in, uuid_str);
954
955   mtevL(noit_debug, "noit_check_watch(%s,%d)\n", uuid_str, period);
956   if(period == 0) {
957     return noit_poller_lookup(in);
958   }
959
960   /* Find the check */
961   snprintf(xpath, sizeof(xpath), "//checks//check[@uuid=\"%s\"]", uuid_str);
962   check_node = mtev_conf_get_section(NULL, xpath);
963   mtev_conf_get_int(NULL, "//checks/@transient_min_period", &minimum_pi);
964   mtev_conf_get_int(NULL, "//checks/@transient_period_granularity", &granularity_pi);
965   if(check_node) {
966     mtev_conf_get_int(check_node,
967                       "ancestor-or-self::node()/@transient_min_period",
968                       &minimum_pi);
969     mtev_conf_get_int(check_node,
970                       "ancestor-or-self::node()/@transient_period_granularity",
971                       &granularity_pi);
972   }
973
974   /* apply the bounds */
975   period /= granularity_pi;
976   period *= granularity_pi;
977   period = MAX(period, minimum_pi);
978
979   uuid_copy(n.checkid, in);
980   n.period = period;
981
982   f = mtev_skiplist_find(&watchlist, &n, NULL);
983   if(f) return f;
984   f = noit_check_clone(in);
985   if(!f) return NULL;
986   f->period = period;
987   f->timeout = period - 10;
988   f->flags |= NP_TRANSIENT;
989   mtevL(noit_debug, "Watching %s@%d\n", uuid_str, period);
990   mtev_skiplist_insert(&watchlist, f);
991   return f;
992 }
993
994 noit_check_t *
995 noit_check_get_watch(uuid_t in, int period) {
996   noit_check_t n, *f;
997
998   uuid_copy(n.checkid, in);
999   n.period = period;
1000
1001   f = mtev_skiplist_find(&watchlist, &n, NULL);
1002   return f;
1003 }
1004
1005 void
1006 noit_check_transient_add_feed(noit_check_t *check, const char *feed) {
1007   char *feedcopy;
1008   if(!check->feeds) {
1009     check->feeds = calloc(1, sizeof(*check->feeds));
1010     mtev_skiplist_init(check->feeds);
1011     mtev_skiplist_set_compare(check->feeds,
1012                               (mtev_skiplist_comparator_t)strcmp,
1013                               (mtev_skiplist_comparator_t)strcmp);
1014   }
1015   feedcopy = strdup(feed);
1016   /* No error on failure -- it's already there */
1017   if(mtev_skiplist_insert(check->feeds, feedcopy) == NULL) free(feedcopy);
1018   mtevL(noit_debug, "check %s`%s @ %dms has %d feed(s): %s.\n",
1019         check->target, check->name, check->period, check->feeds->size, feed);
1020 }
1021 void
1022 noit_check_transient_remove_feed(noit_check_t *check, const char *feed) {
1023   if(!check->feeds) return;
1024   if(feed) {
1025     mtevL(noit_debug, "check %s`%s @ %dms removing 1 of %d feeds: %s.\n",
1026           check->target, check->name, check->period, check->feeds->size, feed);
1027     mtev_skiplist_remove(check->feeds, feed, free);
1028   }
1029   if(check->feeds->size == 0) {
1030     char uuid_str[UUID_STR_LEN + 1];
1031     uuid_unparse_lower(check->checkid, uuid_str);
1032     mtevL(noit_debug, "Unwatching %s@%d\n", uuid_str, check->period);
1033     mtev_skiplist_remove(&watchlist, check, NULL);
1034     mtev_skiplist_destroy(check->feeds, free);
1035     free(check->feeds);
1036     check->feeds = NULL;
1037     if(check->flags & NP_TRANSIENT) {
1038       mtevL(noit_debug, "check %s`%s @ %dms has no more listeners.\n",
1039             check->target, check->name, check->period);
1040       check->flags |= NP_KILLED;
1041     }
1042     noit_poller_free_check(check);
1043   }
1044 }
1045
1046 mtev_boolean
1047 noit_check_is_valid_target(const char *target) {
1048   int8_t family;
1049   int rv;
1050   union {
1051     struct in_addr addr4;
1052     struct in6_addr addr6;
1053   } a;
1054
1055   family = AF_INET;
1056   rv = inet_pton(family, target, &a);
1057   if(rv != 1) {
1058     family = AF_INET6;
1059     rv = inet_pton(family, target, &a);
1060     if(rv != 1) {
1061       return mtev_false;
1062     }
1063   }
1064   return mtev_true;
1065 }
1066 int
1067 noit_check_set_ip(noit_check_t *new_check,
1068                   const char *ip_str, const char *newname) {
1069   int8_t family;
1070   int rv, failed = 0;
1071   char old_target_ip[INET6_ADDRSTRLEN];
1072   union {
1073     struct in_addr addr4;
1074     struct in6_addr addr6;
1075   } a;
1076
1077   memset(old_target_ip, 0, INET6_ADDRSTRLEN);
1078   strlcpy(old_target_ip, new_check->target_ip, sizeof(old_target_ip));
1079
1080   family = NOIT_CHECK_PREFER_V6(new_check) ? AF_INET6 : AF_INET;
1081   rv = inet_pton(family, ip_str, &a);
1082   if(rv != 1) {
1083     if (!NOIT_CHECK_SINGLE_RESOLVE(new_check)) {
1084       family = family == AF_INET ? AF_INET6 : AF_INET;
1085       rv = inet_pton(family, ip_str, &a);
1086       if(rv != 1) {
1087         family = AF_INET;
1088         memset(&a, 0, sizeof(a));
1089         failed = -1;
1090       }
1091     } else {
1092       failed = -1;
1093     }
1094   }
1095
1096   new_check->target_family = family;
1097   memcpy(&new_check->target_addr, &a, sizeof(a));
1098   new_check->target_ip[0] = '\0';
1099   if(failed == 0)
1100     if(inet_ntop(new_check->target_family,
1101                  &new_check->target_addr,
1102                  new_check->target_ip,
1103                  sizeof(new_check->target_ip)) == NULL) {
1104       mtevL(noit_error, "inet_ntop failed [%s] -> %d\n", ip_str, errno);
1105     }
1106   /*
1107    * new_check->name could be null if this check is being set for the
1108    * first time.  add_to_list will set it.
1109    */
1110   if (new_check->name == NULL ||
1111       strcmp(old_target_ip, new_check->target_ip) != 0) {
1112     noit_check_add_to_list(new_check, newname);
1113   }
1114
1115   if(new_check->name == NULL && newname != NULL) {
1116     mtevAssert(new_check->flags & NP_TRANSIENT);
1117     new_check->name = strdup(newname);
1118   }
1119
1120   return failed;
1121 }
1122 int
1123 noit_check_resolve(noit_check_t *check) {
1124   uint8_t family_pref = NOIT_CHECK_PREFER_V6(check) ? AF_INET6 : AF_INET;
1125   char ipaddr[INET6_ADDRSTRLEN];
1126   if(!NOIT_CHECK_SHOULD_RESOLVE(check)) return 1; /* success, not required */
1127   noit_check_resolver_remind(check->target);
1128   if(noit_check_resolver_fetch(check->target, ipaddr, sizeof(ipaddr),
1129                                family_pref) >= 0) {
1130     check->flags |= NP_RESOLVED;
1131     noit_check_set_ip(check, ipaddr, NULL);
1132     return 0;
1133   }
1134   check->flags &= ~NP_RESOLVED;
1135   return -1;
1136 }
1137 int
1138 noit_check_update(noit_check_t *new_check,
1139                   const char *target,
1140                   const char *name,
1141                   const char *filterset,
1142                   mtev_hash_table *config,
1143                   mtev_hash_table **mconfigs,
1144                   u_int32_t period,
1145                   u_int32_t timeout,
1146                   const char *oncheck,
1147                   int64_t seq,
1148                   int flags) {
1149   char uuid_str[37];
1150   int mask = NP_DISABLED | NP_UNCONFIG;
1151
1152   mtevAssert(name);
1153   uuid_unparse_lower(new_check->checkid, uuid_str);
1154   if(!new_check->statistics) new_check->statistics = noit_check_stats_set_calloc();
1155   if(seq < 0) new_check->config_seq = seq = 0;
1156   if(new_check->config_seq > seq) {
1157     mtevL(mtev_error, "noit_check_update[%s] skipped: seq backwards\n", uuid_str);
1158     return -1;
1159   }
1160
1161   /* selfcheck will identify this node in a cluster */
1162   if(mtev_cluster_enabled() && !strcmp(new_check->module, "selfcheck")) {
1163     uuid_t cluster_id;
1164     mtev_cluster_get_self(cluster_id);
1165     if(uuid_compare(cluster_id, new_check->checkid)) {
1166       mtevL(mtev_error, "Setting global cluster identity to '%s'\n", uuid_str);
1167       mtev_cluster_set_self(new_check->checkid);
1168     }
1169   }
1170
1171   if(NOIT_CHECK_RUNNING(new_check)) {
1172     char module[256];
1173     uuid_t id, dummy;
1174     uuid_copy(id, new_check->checkid);
1175     strlcpy(module, new_check->module, sizeof(module));
1176     noit_poller_deschedule(id, mtev_false);
1177     return noit_poller_schedule(target, module, name, filterset,
1178                                 config, mconfigs, period, timeout, oncheck,
1179                                 seq, flags, id, dummy);
1180   }
1181
1182   new_check->generation = __config_load_generation;
1183   if(new_check->target) free(new_check->target);
1184   new_check->target = strdup(target);
1185
1186   // apply resolution flags to check.
1187   if (flags & NP_PREFER_IPV6)
1188     new_check->flags |= NP_PREFER_IPV6;
1189   else
1190     new_check->flags &= ~NP_PREFER_IPV6;
1191   if (flags & NP_SINGLE_RESOLVE)
1192     new_check->flags |= NP_SINGLE_RESOLVE;
1193   else
1194     new_check->flags &= ~NP_SINGLE_RESOLVE;
1195   if (flags & NP_RESOLVE)
1196     new_check->flags |= NP_RESOLVE;
1197   else
1198     new_check->flags &= ~NP_RESOLVE;
1199
1200   /* This sets both the name and the target_addr */
1201   if(noit_check_set_ip(new_check, target, name)) {
1202     mtev_boolean should_resolve;
1203     mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1204     const char *key, *value;
1205     int klen;
1206     char* extension = strrchr(target, '.');
1207     new_check->flags |= NP_RESOLVE;
1208     new_check->flags &= ~NP_RESOLVED;
1209     /* If we match any of the extensions we're supposed to ignore,
1210      * don't resolve */
1211     if (extension && (strlen(extension) > 1)) {
1212       while(mtev_hash_next(&dns_ignore_list, &iter, &key, &klen, (void**)&value)) {
1213         if ((!strcmp("true", value)) && (!strcmp(extension+1, key))) {
1214             new_check->flags &= ~NP_RESOLVE;
1215             break;
1216         }
1217       }
1218     }
1219     if(noit_check_should_resolve_targets(&should_resolve) && !should_resolve)
1220       flags |= NP_DISABLED | NP_UNCONFIG;
1221     noit_check_resolve(new_check);
1222   }
1223
1224   if(new_check->filterset) free(new_check->filterset);
1225   new_check->filterset = filterset ? strdup(filterset): NULL;
1226
1227   if(config != NULL) {
1228     mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1229     const char *k;
1230     int klen;
1231     void *data;
1232     if(new_check->config) mtev_hash_delete_all(new_check->config, free, free);
1233     else {
1234       new_check->config = calloc(1, sizeof(*new_check->config));
1235       mtev_hash_init_locks(new_check->config, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX);
1236     }
1237     while(mtev_hash_next(config, &iter, &k, &klen, &data)) {
1238       mtev_hash_store(new_check->config, strdup(k), klen, strdup((char *)data));
1239     }
1240   }
1241   if(mconfigs != NULL) {
1242     int i;
1243     for(i=0; i<reg_module_id; i++) {
1244       mtev_hash_table *t;
1245       if(NULL != (t = noit_check_get_module_config(new_check, i))) {
1246         noit_check_set_module_config(new_check, i, NULL);
1247         mtev_hash_destroy(t, free, free);
1248         free(t);
1249       }
1250       if(mconfigs[i]) {
1251         mtev_hash_table *t = calloc(1, sizeof(*new_check->config));
1252         mtev_hash_init_locks(t, MTEV_HASH_DEFAULT_SIZE, MTEV_HASH_LOCK_MODE_MUTEX);
1253         mtev_hash_merge_as_dict(t, mconfigs[i]);
1254         noit_check_set_module_config(new_check, i, t);
1255       }
1256     }
1257   }
1258   if(new_check->oncheck) free(new_check->oncheck);
1259   new_check->oncheck = oncheck ? strdup(oncheck) : NULL;
1260   if(new_check->oncheck) system_needs_causality = mtev_true;
1261   new_check->period = period;
1262   new_check->timeout = timeout;
1263   new_check->config_seq = seq;
1264
1265   /* Unset what could be set.. then set what should be set */
1266   new_check->flags = (new_check->flags & ~mask) | flags;
1267
1268   check_config_fixup_hook_invoke(new_check);
1269
1270   if((new_check->flags & NP_TRANSIENT) == 0)
1271     noit_check_activate(new_check);
1272
1273   noit_check_add_to_list(new_check, NULL);
1274   return 0;
1275 }
1276 int
1277 noit_poller_schedule(const char *target,
1278                      const char *module,
1279                      const char *name,
1280                      const char *filterset,
1281                      mtev_hash_table *config,
1282                      mtev_hash_table **mconfigs,
1283                      u_int32_t period,
1284                      u_int32_t timeout,
1285                      const char *oncheck,
1286                      int64_t seq,
1287                      int flags,
1288                      uuid_t in,
1289                      uuid_t out) {
1290   noit_check_t *new_check;
1291   new_check = mtev_memory_safe_calloc(1, sizeof(*new_check));
1292   mtevAssert(new_check != NULL);
1293
1294   /* The module and the UUID can never be changed */
1295   new_check->module = strdup(module);
1296   if(uuid_is_null(in))
1297     uuid_generate(new_check->checkid);
1298   else
1299     uuid_copy(new_check->checkid, in);
1300
1301   new_check->statistics = noit_check_stats_set_calloc();
1302   noit_check_update(new_check, target, name, filterset, config, mconfigs,
1303                     period, timeout, oncheck, seq, flags);
1304   mtevAssert(mtev_hash_store(&polls,
1305                          (char *)new_check->checkid, UUID_SIZE,
1306                          new_check));
1307   uuid_copy(out, new_check->checkid);
1308   noit_check_log_check(new_check);
1309
1310   return 0;
1311 }
1312
1313 /* A quick little list of recycleable checks.  This list never really
1314  * grows large, so no sense in thinking too hard about the algorithmic
1315  * complexity.
1316  */
1317 struct _checker_rcb {
1318   noit_check_t *checker;
1319   struct _checker_rcb *next;
1320 };
1321 static struct _checker_rcb *checker_rcb = NULL;
1322 static void recycle_check(noit_check_t *checker, mtev_boolean has_lock) {
1323   struct _checker_rcb *n = malloc(sizeof(*n));
1324   if(!has_lock) pthread_mutex_lock(&recycling_lock);
1325   n->checker = checker;
1326   n->next = checker_rcb;
1327   checker_rcb = n;
1328   if(!has_lock) pthread_mutex_unlock(&recycling_lock);
1329 }
1330 void
1331 noit_poller_free_check_internal(noit_check_t *checker, mtev_boolean has_lock) {
1332   noit_module_t *mod;
1333
1334   if (checker->flags & NP_PASSIVE_COLLECTION) {
1335     struct timeval current_time;
1336     gettimeofday(&current_time, NULL);
1337     if (checker->last_fire_time.tv_sec == 0) {
1338       memcpy(&checker->last_fire_time, &current_time, sizeof(struct timeval));
1339     }
1340     /* If NP_RUNNING is set for some reason or we've fired recently, recycle
1341      * the check.... we don't want to free it */
1342     if ((checker->flags & NP_RUNNING) ||
1343         (sub_timeval_ms(current_time,checker->last_fire_time) < (checker->period*2))) {
1344       recycle_check(checker, has_lock);
1345       return;
1346     }
1347   }
1348   else if(checker->flags & NP_RUNNING) {
1349     /* If the check is running, don't free it - will clean it up later */
1350     recycle_check(checker, has_lock);
1351     return;
1352   }
1353
1354   mod = noit_module_lookup(checker->module);
1355   if(mod && mod->cleanup) mod->cleanup(mod, checker);
1356   if(checker->fire_event) {
1357      eventer_remove(checker->fire_event);
1358      free(checker->fire_event->closure);
1359      eventer_free(checker->fire_event);
1360      checker->fire_event = NULL;
1361   }
1362   if(checker->closure) free(checker->closure);
1363   if(checker->target) free(checker->target);
1364   if(checker->module) free(checker->module);
1365   if(checker->name) free(checker->name);
1366   if(checker->filterset) free(checker->filterset);
1367   if(checker->config) {
1368     mtev_hash_destroy(checker->config, free, free);
1369     free(checker->config);
1370     checker->config = NULL;
1371   }
1372   if(checker->module_metadata) {
1373     int i;
1374     for(i=0; i<reg_module_id; i++) {
1375       struct vp_w_free *tuple;
1376       tuple = checker->module_metadata[i];
1377       if(tuple) {
1378         if(tuple->freefunc) tuple->freefunc(tuple->ptr);
1379         free(tuple);
1380       }
1381     }
1382     free(checker->module_metadata);
1383   }
1384   if(checker->module_configs) {
1385     int i;
1386     for(i=0; i<reg_module_id; i++) {
1387       if(checker->module_configs[i]) {
1388         mtev_hash_destroy(checker->module_configs[i], free, free);
1389         free(checker->module_configs[i]);
1390       }
1391     }
1392     free(checker->module_configs);
1393   }
1394
1395   mtev_memory_safe_free(stats_inprogress(checker));
1396   mtev_memory_safe_free(stats_current(checker));
1397   mtev_memory_safe_free(stats_previous(checker));
1398
1399   free(checker->statistics);
1400
1401   mtev_memory_safe_free(checker);
1402 }
1403 void
1404 noit_poller_free_check(noit_check_t *checker) {
1405   noit_poller_free_check_internal(checker, mtev_false);
1406 }
1407 static int
1408 check_recycle_bin_processor(eventer_t e, int mask, void *closure,
1409                             struct timeval *now) {
1410   static struct timeval one_minute = { RECYCLE_INTERVAL, 0L };
1411   struct _checker_rcb *prev = NULL, *curr = NULL;
1412   mtevL(noit_debug, "Scanning check recycle bin\n");
1413   pthread_mutex_lock(&recycling_lock);
1414   curr = checker_rcb;
1415   while(curr) {
1416     noit_check_t *check = curr->checker;
1417     mtev_boolean free_check = mtev_false;
1418     if (check->flags & NP_PASSIVE_COLLECTION) {
1419       struct timeval current_time;
1420       gettimeofday(&current_time, NULL);
1421       if ((!(check->flags & NP_RUNNING)) &&
1422           (sub_timeval_ms(current_time,check->last_fire_time) >= (check->period*2))) {
1423         free_check = mtev_true;
1424       }
1425     }
1426     else if(!(curr->checker->flags & NP_RUNNING)) {
1427       free_check = mtev_true;
1428     }
1429
1430     if (free_check == mtev_true) {
1431       mtevL(noit_debug, "0x%p: Check is ready to free.\n", check);
1432       noit_poller_free_check_internal(curr->checker, mtev_true);
1433       if(prev) prev->next = curr->next;
1434       else checker_rcb = curr->next;
1435       free(curr);
1436       curr = prev ? prev->next : checker_rcb;
1437     }
1438     else {
1439       prev = curr;
1440       curr = curr->next;
1441     }
1442   }
1443   pthread_mutex_unlock(&recycling_lock);
1444   add_timeval(*now, one_minute, &e->whence);
1445   return EVENTER_TIMER;
1446 }
1447
1448 int
1449 noit_poller_deschedule(uuid_t in, mtev_boolean log) {
1450   void *vcheck;
1451   noit_check_t *checker;
1452   if(mtev_hash_retrieve(&polls,
1453                         (char *)in, UUID_SIZE,
1454                         &vcheck) == 0) {
1455     return -1;
1456   }
1457   checker = (noit_check_t *)vcheck;
1458   checker->flags |= (NP_DISABLED|NP_KILLED);
1459
1460   if(log) noit_check_log_delete(checker);
1461
1462   mtevAssert(mtev_skiplist_remove(&polls_by_name, checker, NULL));
1463   mtevAssert(mtev_hash_delete(&polls, (char *)in, UUID_SIZE, NULL, NULL));
1464
1465   noit_poller_free_check(checker);
1466   return 0;
1467 }
1468
1469 noit_check_t *
1470 noit_poller_lookup(uuid_t in) {
1471   noit_check_t *check;
1472   pthread_mutex_lock(&polls_lock);
1473   check = noit_poller_lookup__nolock(in);
1474   pthread_mutex_unlock(&polls_lock);
1475   return check;
1476 }
1477 noit_check_t *
1478 noit_poller_lookup_by_name(char *target, char *name) {
1479   noit_check_t *check;
1480   pthread_mutex_lock(&polls_lock);
1481   check = noit_poller_lookup_by_name__nolock(target,name);
1482   pthread_mutex_unlock(&polls_lock);
1483   return check;
1484 }
1485 int
1486 noit_poller_target_ip_do(const char *target_ip,
1487                          int (*f)(noit_check_t *, void *),
1488                          void *closure) {
1489   int i, count = 0, todo_count = 0;
1490   noit_check_t pivot;
1491   mtev_skiplist *tlist;
1492   mtev_skiplist_node *next;
1493   noit_check_t *todo_onstack[8192];
1494   noit_check_t **todo = todo_onstack;
1495
1496   tlist = mtev_skiplist_find(polls_by_name.index,
1497                              __check_target_ip_compare, NULL);
1498
1499   pthread_mutex_lock(&polls_lock);
1500   /* First pass to count */
1501   memset(&pivot, 0, sizeof(pivot));
1502   strlcpy(pivot.target_ip, (char*)target_ip, sizeof(pivot.target_ip));
1503   pivot.name = "";
1504   pivot.target = "";
1505   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1506   while(next && next->data) {
1507     noit_check_t *check = next->data;
1508     if(strcmp(check->target_ip, target_ip)) break;
1509     todo_count++;
1510     mtev_skiplist_next(tlist, &next);
1511   }
1512
1513   if(todo_count > 8192) todo = malloc(todo_count * sizeof(*todo));
1514
1515   memset(&pivot, 0, sizeof(pivot));
1516   strlcpy(pivot.target_ip, (char*)target_ip, sizeof(pivot.target_ip));
1517   pivot.name = "";
1518   pivot.target = "";
1519   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1520   while(next && next->data) {
1521     noit_check_t *check = next->data;
1522     if(strcmp(check->target_ip, target_ip)) break;
1523     if(count < todo_count) todo[count++] = check;
1524     mtev_skiplist_next(tlist, &next);
1525   }
1526   pthread_mutex_unlock(&polls_lock);
1527
1528   todo_count = count;
1529   count = 0;
1530   for(i=0;i<todo_count;i++)
1531     count += f(todo[i],closure);
1532
1533   if(todo != todo_onstack) free(todo);
1534   return count;
1535 }
1536 int
1537 noit_poller_target_do(const char *target, int (*f)(noit_check_t *, void *),
1538                       void *closure) {
1539   int i, todo_count = 0, count = 0;
1540   noit_check_t pivot;
1541   mtev_skiplist *tlist;
1542   mtev_skiplist_node *next;
1543   noit_check_t *todo_onstack[8192];
1544   noit_check_t **todo = todo_onstack;
1545
1546   tlist = mtev_skiplist_find(polls_by_name.index,
1547                              __check_target_compare, NULL);
1548
1549   pthread_mutex_lock(&polls_lock);
1550   memset(&pivot, 0, sizeof(pivot));
1551   pivot.name = "";
1552   pivot.target = (char *)target;
1553   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1554   while(next && next->data) {
1555     noit_check_t *check = next->data;
1556     if(strcmp(check->target, target)) break;
1557     todo_count++;
1558     mtev_skiplist_next(tlist, &next);
1559   }
1560
1561   if(todo_count > 8192) todo = malloc(todo_count * sizeof(*todo));
1562
1563   memset(&pivot, 0, sizeof(pivot));
1564   pivot.name = "";
1565   pivot.target = (char *)target;
1566   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1567   while(next && next->data) {
1568     noit_check_t *check = next->data;
1569     if(strcmp(check->target, target)) break;
1570     if(count < todo_count) todo[count++] = check;
1571     mtev_skiplist_next(tlist, &next);
1572   }
1573   pthread_mutex_unlock(&polls_lock);
1574
1575   todo_count = count;
1576   count = 0;
1577   for(i=0;i<todo_count;i++)
1578     count += f(todo[i],closure);
1579
1580   if(todo != todo_onstack) free(todo);
1581   return count;
1582 }
1583
1584 int
1585 noit_poller_do(int (*f)(noit_check_t *, void *),
1586                void *closure) {
1587   mtev_skiplist_node *iter;
1588   int i, count = 0, max_count = 0;
1589   noit_check_t **todo;
1590
1591   if(polls_by_name.size == 0) return 0;
1592
1593   max_count = polls_by_name.size;
1594   todo = malloc(max_count * sizeof(*todo));
1595
1596   pthread_mutex_lock(&polls_lock);
1597   for(iter = mtev_skiplist_getlist(&polls_by_name); iter;
1598       mtev_skiplist_next(&polls_by_name, &iter)) {
1599     if(count < max_count) todo[count++] = (noit_check_t *)iter->data;
1600   }
1601   pthread_mutex_unlock(&polls_lock);
1602
1603   max_count = count;
1604   count = 0;
1605   for(i=0;i<max_count;i++)
1606     count += f(todo[i], closure);
1607   free(todo);
1608   return count;
1609 }
1610
1611 struct ip_module_collector_crutch {
1612   noit_check_t **array;
1613   const char *module;
1614   int idx;
1615   int allocd;
1616 };
1617 static int ip_module_collector(noit_check_t *check, void *cl) {
1618   struct ip_module_collector_crutch *c = cl;
1619   if(c->idx >= c->allocd) return 0;
1620   if(strcmp(check->module, c->module)) return 0;
1621   c->array[c->idx++] = check;
1622   return 1;
1623 }
1624 int
1625 noit_poller_lookup_by_ip_module(const char *ip, const char *mod,
1626                                 noit_check_t **checks, int nchecks) {
1627   struct ip_module_collector_crutch crutch;
1628   crutch.array = checks;
1629   crutch.allocd = nchecks;
1630   crutch.idx = 0;
1631   crutch.module = mod;
1632   return noit_poller_target_ip_do(ip, ip_module_collector, &crutch);
1633 }
1634 int
1635 noit_poller_lookup_by_module(const char *ip, const char *mod,
1636                              noit_check_t **checks, int nchecks) {
1637   struct ip_module_collector_crutch crutch;
1638   crutch.array = checks;
1639   crutch.allocd = nchecks;
1640   crutch.idx = 0;
1641   crutch.module = mod;
1642   return noit_poller_target_do(ip, ip_module_collector, &crutch);
1643 }
1644
1645
1646 int
1647 noit_check_xpath(char *xpath, int len,
1648                  const char *base, const char *arg) {
1649   uuid_t checkid;
1650   int base_trailing_slash;
1651   char argcopy[1024], *target, *module, *name;
1652
1653   base_trailing_slash = (base[strlen(base)-1] == '/');
1654   xpath[0] = '\0';
1655   argcopy[0] = '\0';
1656   if(arg) strlcpy(argcopy, arg, sizeof(argcopy));
1657
1658   if(uuid_parse(argcopy, checkid) == 0) {
1659     /* If they kill by uuid, we'll seek and destroy -- find it anywhere */
1660     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
1661              base, base_trailing_slash ? "" : "/", argcopy);
1662   }
1663   else if((module = strchr(argcopy, '`')) != NULL) {
1664     noit_check_t *check;
1665     char uuid_str[37];
1666     target = argcopy;
1667     *module++ = '\0';
1668     if((name = strchr(module+1, '`')) == NULL)
1669       name = module;
1670     else
1671       name++;
1672     check = noit_poller_lookup_by_name(target, name);
1673     if(!check) {
1674       return -1;
1675     }
1676     uuid_unparse_lower(check->checkid, uuid_str);
1677     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
1678              base, base_trailing_slash ? "" : "/", uuid_str);
1679   }
1680   return strlen(xpath);
1681 }
1682
1683 static int
1684 bad_check_initiate(noit_module_t *self, noit_check_t *check,
1685                    int once, noit_check_t *cause) {
1686   /* self is likely null here -- why it is bad, in fact */
1687   /* this is only suitable to call in one-offs */
1688   struct timeval now;
1689   stats_t *inp;
1690   char buff[256];
1691   if(!once) return -1;
1692   if(!check) return -1;
1693   mtevAssert(!(check->flags & NP_RUNNING));
1694   check->flags |= NP_RUNNING;
1695   inp = noit_check_get_stats_inprogress(check);
1696   gettimeofday(&now, NULL);
1697   noit_check_stats_whence(inp, &now);
1698   snprintf(buff, sizeof(buff), "check[%s] implementation offline",
1699            check->module);
1700   noit_check_stats_status(inp, buff);
1701   noit_check_set_stats(check);
1702   check->flags &= ~NP_RUNNING;
1703   return 0;
1704 }
1705 void
1706 noit_check_stats_clear(noit_check_t *check, stats_t *s) {
1707   memset(s, 0, sizeof(*s));
1708   s->state = NP_UNKNOWN;
1709   s->available = NP_UNKNOWN;
1710 }
1711
1712 static void
1713 __stats_add_metric(stats_t *newstate, metric_t *m) {
1714   mtev_hash_replace(&newstate->metrics, m->metric_name, strlen(m->metric_name),
1715                     m, NULL, (void (*)(void *))mtev_memory_safe_free);
1716 }
1717
1718 static void
1719 __mark_metric_logged(stats_t *newstate, const char *metric_name) {
1720   void *vm;
1721   if(mtev_hash_retrieve(&newstate->metrics,
1722                         metric_name, strlen(metric_name), &vm)) {
1723     ((metric_t *)vm)->logged = mtev_true;
1724   }
1725 }
1726
1727 static size_t
1728 noit_metric_sizes(metric_type_t type, const void *value) {
1729   switch(type) {
1730     case METRIC_INT32:
1731     case METRIC_UINT32:
1732       return sizeof(int32_t);
1733     case METRIC_INT64:
1734     case METRIC_UINT64:
1735       return sizeof(int64_t);
1736     case METRIC_DOUBLE:
1737       return sizeof(double);
1738     case METRIC_STRING: {
1739       int len = strlen((char*)value) + 1;
1740       return ((len >= text_size_limit) ? text_size_limit+1 : len);
1741     }
1742     case METRIC_ABSENT:
1743     case METRIC_NULL:
1744     case METRIC_GUESS:
1745       break;
1746   }
1747   mtevAssert(type != type);
1748   return 0;
1749 }
1750 static metric_type_t
1751 noit_metric_guess_type(const char *s, void **replacement) {
1752   char *copy, *cp, *trailer, *rpl;
1753   int negative = 0;
1754   metric_type_t type = METRIC_STRING;
1755
1756   if(!s) return METRIC_GUESS;
1757   copy = cp = strdup(s);
1758
1759   /* TRIM the string */
1760   while(*cp && isspace(*cp)) cp++; /* ltrim */
1761   s = cp; /* found a good starting point */
1762   while(*cp) cp++; /* advance to \0 */
1763   cp--; /* back up one */
1764   while(cp > s && isspace(*cp)) *cp-- = '\0'; /* rtrim */
1765
1766   /* Find the first space */
1767   cp = (char *)s;
1768   while(*cp && !isspace(*cp)) cp++;
1769   trailer = cp;
1770   cp--; /* backup one */
1771   if(cp > s && *cp == '%') *cp-- = '\0'; /* chop a last % is there is one */
1772
1773   while(*trailer && isspace(*trailer)) *trailer++ = '\0'; /* rtrim */
1774
1775   /* string was       '  -1.23e-01%  inodes used  ' */
1776   /* copy is (~ = \0) '  -1.23e-01~  inodes used~~' */
1777   /*                     ^           ^              */
1778   /*                     s           trailer        */
1779
1780   /* So, the trailer must not contain numbers */
1781   while(*trailer) { if(isdigit(*trailer)) goto notanumber; trailer++; }
1782
1783   /* And the 's' must be of the form:
1784    *  0) may start with a sign [-+]?
1785    *  1) [1-9][0-9]*
1786    *  2) [0]?.[0-9]+
1787    *  3) 0
1788    *  4) [1-9][0-9]*.[0-9]+
1789    *  5) all of the above ending with e[+-][0-9]+
1790    */
1791    rpl = (char *)s;
1792    /* CASE 0 */
1793    if(s[0] == '-' || s[0] == '+') {
1794      if(s[0] == '-') negative = 1;
1795      s++;
1796    }
1797
1798    if(s[0] == '.') goto decimal; /* CASE 2 */
1799    if(s[0] == '0') { /* CASE 2 & 3 */
1800      s++;
1801      if(!s[0]) goto scanint; /* CASE 3 */
1802      if(s[0] == '.') goto decimal; /* CASE 2 */
1803      goto notanumber;
1804    }
1805    if(s[0] >= '1' && s[0] <= '9') { /* CASE 1 & 4 */
1806      s++;
1807      while(isdigit(s[0])) s++; /* CASE 1 & 4 */
1808      if(!s[0]) goto scanint; /* CASE 1 */
1809      if(s[0] == '.') goto decimal; /* CASE 4 */
1810      goto notanumber;
1811    }
1812    /* Not case 1,2,3,4 */
1813    goto notanumber;
1814
1815   decimal:
1816    s++;
1817    if(!isdigit(s[0])) goto notanumber;
1818    s++;
1819    while(isdigit(s[0])) s++;
1820    if(!s[0]) goto scandouble;
1821    if(s[0] == 'e' || s[0] == 'E') goto exponent; /* CASE 5 */
1822    goto notanumber;
1823
1824   exponent:
1825    s++;
1826    if(s[0] != '-' && s[0] != '+') goto notanumber;
1827    s++;
1828    if(!isdigit(s[0])) goto notanumber;
1829    s++;
1830    while(isdigit(s[0])) s++;
1831    if(!s[0]) goto scandouble;
1832    goto notanumber;
1833
1834  scanint:
1835    if(negative) {
1836      int64_t *v;
1837      v = malloc(sizeof(*v));
1838      *v = strtoll(rpl, NULL, 10);
1839      *replacement = v;
1840      type = METRIC_INT64;
1841      goto alldone;
1842    }
1843    else {
1844      u_int64_t *v;
1845      v = malloc(sizeof(*v));
1846      *v = strtoull(rpl, NULL, 10);
1847      *replacement = v;
1848      type = METRIC_UINT64;
1849      goto alldone;
1850    }
1851  scandouble:
1852    {
1853      double *v;
1854      v = malloc(sizeof(*v));
1855      *v = strtod(rpl, NULL);
1856      *replacement = v;
1857      type = METRIC_DOUBLE;
1858      goto alldone;
1859    }
1860
1861  alldone:
1862  notanumber:
1863   free(copy);
1864   return type;
1865 }
1866
1867 static void
1868 cleanse_metric_name(char *m) {
1869   char *cp;
1870   for(cp = m; *cp; cp++)
1871     if(!isprint(*cp)) *cp=' ';
1872   for(cp--; *cp == ' ' && cp > m; cp--) /* always leave first char */
1873     *cp = '\0';
1874 }
1875
1876 int
1877 noit_stats_populate_metric(metric_t *m, const char *name, metric_type_t type,
1878                            const void *value) {
1879   void *replacement = NULL;
1880
1881   /* If we are passed a null name, we want to quit populating the metric...
1882    * no reason we should ever have a null metric name */
1883   if (!name) {
1884     return -1;
1885   }
1886
1887   m->metric_name = strdup(name);
1888   cleanse_metric_name(m->metric_name);
1889
1890   if(type == METRIC_GUESS)
1891     type = noit_metric_guess_type((char *)value, &replacement);
1892   if(type == METRIC_GUESS) return -1;
1893
1894   m->metric_type = type;
1895
1896   if(replacement)
1897     m->metric_value.vp = replacement;
1898   else if(value) {
1899     size_t len;
1900     len = noit_metric_sizes(type, value);
1901     m->metric_value.vp = malloc(len);
1902     memcpy(m->metric_value.vp, value, len);
1903     if (type == METRIC_STRING) {
1904       m->metric_value.s[len-1] = 0;
1905     }
1906   }
1907   else m->metric_value.vp = NULL;
1908   return 0;
1909 }
1910
1911 metric_t *
1912 noit_stats_get_metric(noit_check_t *check,
1913                       stats_t *newstate, const char *name) {
1914   void *v;
1915   if(newstate == NULL)
1916     newstate = stats_inprogress(check);
1917   if(mtev_hash_retrieve(&newstate->metrics, name, strlen(name), &v))
1918     return (metric_t *)v;
1919   return NULL;
1920 }
1921
1922 void
1923 noit_stats_set_metric(noit_check_t *check,
1924                       const char *name, metric_type_t type,
1925                       const void *value) {
1926   stats_t *c;
1927   metric_t *m = mtev_memory_safe_malloc_cleanup(sizeof(*m), noit_check_safe_free_metric);
1928   memset(m, 0, sizeof(*m));
1929   if(noit_stats_populate_metric(m, name, type, value)) {
1930     mtev_memory_safe_free(m);
1931     return;
1932   }
1933   noit_check_metric_count_add(1);
1934   c = noit_check_get_stats_inprogress(check);
1935   check_stats_set_metric_hook_invoke(check, c, m);
1936   __stats_add_metric(c, m);
1937 }
1938 void
1939 noit_stats_set_metric_coerce(noit_check_t *check,
1940                              const char *name, metric_type_t t,
1941                              const char *v) {
1942   char *endptr;
1943   stats_t *c;
1944   c = noit_check_get_stats_inprogress(check);
1945   if(v == NULL) {
1946    bogus:
1947     check_stats_set_metric_coerce_hook_invoke(check, c, name, t, v, mtev_false);
1948     noit_stats_set_metric(check, name, t, NULL);
1949     return;
1950   }
1951   switch(t) {
1952     case METRIC_STRING:
1953       noit_stats_set_metric(check, name, t, v);
1954       break;
1955     case METRIC_INT32:
1956     {
1957       int32_t val;
1958       val = strtol(v, &endptr, 10);
1959       if(endptr == v) goto bogus;
1960       noit_stats_set_metric(check, name, t, &val);
1961       break;
1962     }
1963     case METRIC_UINT32:
1964     {
1965       u_int32_t val;
1966       val = strtoul(v, &endptr, 10);
1967       if(endptr == v) goto bogus;
1968       noit_stats_set_metric(check, name, t, &val);
1969       break;
1970     }
1971     case METRIC_INT64:
1972     {
1973       int64_t val;
1974       val = strtoll(v, &endptr, 10);
1975       if(endptr == v) goto bogus;
1976       noit_stats_set_metric(check, name, t, &val);
1977       break;
1978     }
1979     case METRIC_UINT64:
1980     {
1981       u_int64_t val;
1982       val = strtoull(v, &endptr, 10);
1983       if(endptr == v) goto bogus;
1984       noit_stats_set_metric(check, name, t, &val);
1985       break;
1986     }
1987     case METRIC_DOUBLE:
1988     {
1989       double val;
1990       val = strtod(v, &endptr);
1991       if(endptr == v) goto bogus;
1992       noit_stats_set_metric(check, name, t, &val);
1993       break;
1994     }
1995     case METRIC_GUESS:
1996       noit_stats_set_metric(check, name, t, v);
1997       break;
1998     case METRIC_ABSENT:
1999     case METRIC_NULL:
2000       mtevAssert(0 && "ABSENT and NULL metrics may not be passed to noit_stats_set_metric_coerce");
2001   }
2002   check_stats_set_metric_coerce_hook_invoke(check, c, name, t, v, mtev_true);
2003 }
2004 void
2005 noit_stats_log_immediate_metric(noit_check_t *check,
2006                                 const char *name, metric_type_t type,
2007                                 const void *value) {
2008   struct timeval now;
2009   stats_t *c;
2010   metric_t *m = mtev_memory_safe_malloc_cleanup(sizeof(*m), noit_check_safe_free_metric);
2011   memset(m, 0, sizeof(*m));
2012   if(noit_stats_populate_metric(m, name, type, value)) {
2013     mtev_memory_safe_free(m);
2014     return;
2015   }
2016   gettimeofday(&now, NULL);
2017   noit_check_log_metric(check, &now, m);
2018   mtev_memory_safe_free(m);
2019   c = noit_check_get_stats_inprogress(check);
2020   __mark_metric_logged(c, name);
2021 }
2022
2023 void
2024 noit_check_passive_set_stats(noit_check_t *check) {
2025   int i, nwatches = 0;
2026   mtev_skiplist_node *next;
2027   noit_check_t n;
2028   noit_check_t *watches[8192];
2029
2030   uuid_copy(n.checkid, check->checkid);
2031   n.period = 0;
2032
2033   noit_check_set_stats(check);
2034
2035   pthread_mutex_lock(&polls_lock);
2036   mtev_skiplist_find_neighbors(&watchlist, &n, NULL, NULL, &next);
2037   while(next && next->data && nwatches < 8192) {
2038     noit_check_t *wcheck = next->data;
2039     if(uuid_compare(n.checkid, wcheck->checkid)) break;
2040     watches[nwatches++] = wcheck;
2041     mtev_skiplist_next(&watchlist, &next);
2042   }
2043   pthread_mutex_unlock(&polls_lock);
2044
2045   for(i=0;i<nwatches;i++) {
2046     void *backup;
2047     noit_check_t *wcheck = watches[i];
2048     /* Swap the real check's stats into place */
2049     backup = wcheck->statistics;
2050     wcheck->statistics = check->statistics;
2051
2052     if(check_passive_log_stats_hook_invoke(check) == MTEV_HOOK_CONTINUE) {
2053       /* Write out our status */
2054       noit_check_log_status(wcheck);
2055       /* Write out all metrics */
2056       noit_check_log_metrics(wcheck);
2057     }
2058     /* Swap them back out */
2059     wcheck->statistics = backup;
2060   }
2061 }
2062 void
2063 noit_check_set_stats(noit_check_t *check) {
2064   int report_change = 0;
2065   char *cp;
2066   dep_list_t *dep;
2067   stats_t *old, *prev, *current;
2068
2069   if(check_set_stats_hook_invoke(check) == MTEV_HOOK_ABORT) return;
2070
2071   old = stats_previous(check);
2072   prev = stats_previous(check) = stats_current(check);
2073   current = stats_current(check) = stats_inprogress(check);
2074   stats_inprogress(check) = noit_check_stats_alloc();
2075  
2076   if(old) {
2077     mtev_memory_safe_free(old);
2078   }
2079
2080   if(current) {
2081     for(cp = current->status; cp && *cp; cp++)
2082       if(*cp == '\r' || *cp == '\n') *cp = ' ';
2083   }
2084
2085   /* check for state changes */
2086   if((!current || (current->available != NP_UNKNOWN)) &&
2087      (!prev || (prev->available != NP_UNKNOWN)) &&
2088      (!current || !prev || (current->available != prev->available)))
2089     report_change = 1;
2090   if((!current || (current->state != NP_UNKNOWN)) &&
2091      (!prev || (prev->state != NP_UNKNOWN)) &&
2092      (!current || !prev || (current->state != prev->state)))
2093     report_change = 1;
2094
2095   mtevL(noit_debug, "%s`%s <- [%s]\n", check->target, check->name,
2096         current ? current->status : "null");
2097   if(report_change) {
2098     mtevL(noit_debug, "%s`%s -> [%s:%s]\n",
2099           check->target, check->name,
2100           noit_check_available_string(current ? current->available : NP_UNKNOWN),
2101           noit_check_state_string(current ? current->state : NP_UNKNOWN));
2102   }
2103
2104   if(NOIT_CHECK_STATUS_ENABLED()) {
2105     char id[UUID_STR_LEN+1];
2106     uuid_unparse_lower(check->checkid, id);
2107     NOIT_CHECK_STATUS(id, check->module, check->name, check->target,
2108                       current ? current->available : NP_UNKNOWN,
2109                       current ? current->state : NP_UNKNOWN,
2110                       current ? current->status : "null");
2111   }
2112
2113   if(check_log_stats_hook_invoke(check) == MTEV_HOOK_CONTINUE) {
2114     /* Write out the bundled information */
2115     noit_check_log_bundle(check);
2116   }
2117   /* count the check as complete */
2118   check_completion_count++;
2119
2120   for(dep = check->causal_checks; dep; dep = dep->next) {
2121     noit_module_t *mod;
2122     mod = noit_module_lookup(dep->check->module);
2123     if(!mod) {
2124       bad_check_initiate(mod, dep->check, 1, check);
2125     }
2126     else {
2127       mtevL(noit_debug, "Firing %s`%s in response to %s`%s\n",
2128             dep->check->target, dep->check->name,
2129             check->target, check->name);
2130       if((dep->check->flags & NP_DISABLED) == 0)
2131         if(mod->initiate_check)
2132           mod->initiate_check(mod, dep->check, 1, check);
2133     }
2134   }
2135 }
2136
2137 static int
2138 noit_console_show_watchlist(mtev_console_closure_t ncct,
2139                             int argc, char **argv,
2140                             mtev_console_state_t *dstate,
2141                             void *closure) {
2142   mtev_skiplist_node *iter, *fiter;
2143   int nwatches = 0, i;
2144   noit_check_t *watches[8192];
2145
2146   nc_printf(ncct, "%d active watches.\n", watchlist.size);
2147   pthread_mutex_lock(&polls_lock);
2148   for(iter = mtev_skiplist_getlist(&watchlist); iter && nwatches < 8192;
2149       mtev_skiplist_next(&watchlist, &iter)) {
2150     noit_check_t *check = iter->data;
2151     watches[nwatches++] = check;
2152   }
2153   pthread_mutex_unlock(&polls_lock);
2154
2155   for(i=0;i<nwatches;i++) {
2156     noit_check_t *check = watches[i];
2157     char uuid_str[UUID_STR_LEN + 1];
2158
2159     uuid_unparse_lower(check->checkid, uuid_str);
2160     nc_printf(ncct, "%s:\n\t[%s`%s`%s]\n\tPeriod: %dms\n\tFeeds[%d]:\n",
2161               uuid_str, check->target, check->module, check->name,
2162               check->period, check->feeds ? check->feeds->size : 0);
2163     if(check->feeds && check->feeds->size) {
2164       for(fiter = mtev_skiplist_getlist(check->feeds); fiter;
2165           mtev_skiplist_next(check->feeds, &fiter)) {
2166         nc_printf(ncct, "\t\t%s\n", (const char *)fiter->data);
2167       }
2168     }
2169   }
2170   return 0;
2171 }
2172
2173 static void
2174 nc_printf_check_brief(mtev_console_closure_t ncct,
2175                       noit_check_t *check) {
2176   stats_t *current;
2177   char out[512];
2178   char uuid_str[37];
2179   snprintf(out, sizeof(out), "%s`%s (%s [%x])", check->target, check->name,
2180            check->target_ip, check->flags);
2181   uuid_unparse_lower(check->checkid, uuid_str);
2182   nc_printf(ncct, "%s %s\n", uuid_str, out);
2183   current = stats_current(check);
2184   if(current)
2185     nc_printf(ncct, "\t%s\n", current->status);
2186 }
2187
2188 char *
2189 noit_console_conf_check_opts(mtev_console_closure_t ncct,
2190                              mtev_console_state_stack_t *stack,
2191                              mtev_console_state_t *dstate,
2192                              int argc, char **argv, int idx) {
2193   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
2194   uuid_t key_id;
2195   int klen, i = 0;
2196   void *vcheck;
2197
2198   if(argc == 1) {
2199     if(!strncmp("new", argv[0], strlen(argv[0]))) {
2200       if(idx == i) return strdup("new");
2201       i++;
2202     }
2203     pthread_mutex_lock(&polls_lock);
2204     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
2205                          &vcheck)) {
2206       noit_check_t *check = (noit_check_t *)vcheck;
2207       char out[512];
2208       char uuid_str[37];
2209       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
2210       uuid_unparse_lower(check->checkid, uuid_str);
2211       if(!strncmp(out, argv[0], strlen(argv[0]))) {
2212         if(idx == i) {
2213           pthread_mutex_unlock(&polls_lock);
2214           return strdup(out);
2215         }
2216         i++;
2217       }
2218       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
2219         if(idx == i) {
2220           pthread_mutex_unlock(&polls_lock);
2221           return strdup(uuid_str);
2222         }
2223         i++;
2224       }
2225     }
2226     pthread_mutex_unlock(&polls_lock);
2227   }
2228   if(argc == 2) {
2229     cmd_info_t *cmd;
2230     if(!strcmp("new", argv[0])) return NULL;
2231     cmd = mtev_skiplist_find(&dstate->cmds, "attribute", NULL);
2232     if(!cmd) return NULL;
2233     return mtev_console_opt_delegate(ncct, stack, cmd->dstate, argc-1, argv+1, idx);
2234   }
2235   return NULL;
2236 }
2237
2238 char *
2239 noit_console_check_opts(mtev_console_closure_t ncct,
2240                         mtev_console_state_stack_t *stack,
2241                         mtev_console_state_t *dstate,
2242                         int argc, char **argv, int idx) {
2243   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
2244   uuid_t key_id;
2245   int klen, i = 0;
2246
2247   if(argc == 1) {
2248     void *vcheck;
2249     pthread_mutex_lock(&polls_lock);
2250     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
2251                          &vcheck)) {
2252       char out[512];
2253       char uuid_str[37];
2254       noit_check_t *check = (noit_check_t *)vcheck;
2255       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
2256       uuid_unparse_lower(check->checkid, uuid_str);
2257       if(!strncmp(out, argv[0], strlen(argv[0]))) {
2258         if(idx == i) {
2259           pthread_mutex_unlock(&polls_lock);
2260           return strdup(out);
2261         }
2262         i++;
2263       }
2264       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
2265         if(idx == i) {
2266           pthread_mutex_unlock(&polls_lock);
2267           return strdup(uuid_str);
2268         }
2269         i++;
2270       }
2271     }
2272     pthread_mutex_unlock(&polls_lock);
2273   }
2274   if(argc == 2) {
2275     return mtev_console_opt_delegate(ncct, stack, dstate, argc-1, argv+1, idx);
2276   }
2277   return NULL;
2278 }
2279
2280 static int
2281 noit_console_show_checks(mtev_console_closure_t ncct,
2282                          int argc, char **argv,
2283                          mtev_console_state_t *dstate,
2284                          void *closure) {
2285   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
2286   uuid_t key_id;
2287   int klen, i = 0, nchecks;
2288   void *vcheck;
2289   noit_check_t **checks;
2290
2291   nchecks = mtev_hash_size(&polls);
2292   if(nchecks == 0) return 0;
2293   checks = malloc(nchecks * sizeof(*checks));
2294
2295   pthread_mutex_lock(&polls_lock);
2296   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
2297                        &vcheck)) {
2298     if(i<nchecks) checks[i++] = vcheck;
2299   }
2300   pthread_mutex_unlock(&polls_lock);
2301
2302   nchecks = i;
2303   for(i=0;i<nchecks;i++)
2304     nc_printf_check_brief(ncct,checks[i]);
2305
2306   free(checks);
2307   return 0;
2308 }
2309
2310 static int
2311 noit_console_short_checks_sl(mtev_console_closure_t ncct,
2312                              mtev_skiplist *tlist) {
2313   int max_count, i = 0;
2314   noit_check_t **todo;
2315   mtev_skiplist_node *iter;
2316
2317   max_count = tlist->size;
2318   if(max_count == 0) return 0;
2319   todo = malloc(max_count * sizeof(*todo));
2320
2321   pthread_mutex_lock(&polls_lock);
2322   for(iter = mtev_skiplist_getlist(tlist); i < max_count && iter;
2323       mtev_skiplist_next(tlist, &iter)) {
2324     todo[i++] = iter->data;
2325   }
2326   pthread_mutex_unlock(&polls_lock);
2327
2328   max_count = i;
2329   for(i=0;i<max_count;i++)
2330     nc_printf_check_brief(ncct, todo[i]);
2331
2332   free(todo);
2333   return 0;
2334 }
2335 static int
2336 noit_console_show_checks_name(mtev_console_closure_t ncct,
2337                               int argc, char **argv,
2338                               mtev_console_state_t *dstate,
2339                               void *closure) {
2340   return noit_console_short_checks_sl(ncct, &polls_by_name);
2341 }
2342
2343 static int
2344 noit_console_show_checks_target(mtev_console_closure_t ncct,
2345                                    int argc, char **argv,
2346                                    mtev_console_state_t *dstate,
2347                                    void *closure) {
2348   return noit_console_short_checks_sl(ncct,
2349            mtev_skiplist_find(polls_by_name.index,
2350            __check_target_compare, NULL));
2351 }
2352
2353 static int
2354 noit_console_show_checks_target_ip(mtev_console_closure_t ncct,
2355                                    int argc, char **argv,
2356                                    mtev_console_state_t *dstate,
2357                                    void *closure) {
2358   return noit_console_short_checks_sl(ncct,
2359            mtev_skiplist_find(polls_by_name.index,
2360            __check_target_ip_compare, NULL));
2361 }
2362
2363 static void
2364 register_console_check_commands() {
2365   mtev_console_state_t *tl;
2366   cmd_info_t *showcmd;
2367
2368   tl = mtev_console_state_initial();
2369   showcmd = mtev_console_state_get_cmd(tl, "show");
2370   mtevAssert(showcmd && showcmd->dstate);
2371
2372   mtev_console_state_add_cmd(showcmd->dstate,
2373     NCSCMD("timing_slots", noit_console_show_timing_slots, NULL, NULL, NULL));
2374
2375   mtev_console_state_add_cmd(showcmd->dstate,
2376     NCSCMD("checks", noit_console_show_checks, NULL, NULL, NULL));
2377
2378   mtev_console_state_add_cmd(showcmd->dstate,
2379     NCSCMD("checks:name", noit_console_show_checks_name, NULL,
2380            NULL, NULL));
2381
2382   mtev_console_state_add_cmd(showcmd->dstate,
2383     NCSCMD("checks:target", noit_console_show_checks_target, NULL,
2384            NULL, NULL));
2385
2386   mtev_console_state_add_cmd(showcmd->dstate,
2387     NCSCMD("checks:target_ip", noit_console_show_checks_target_ip, NULL,
2388            NULL, NULL));
2389
2390   mtev_console_state_add_cmd(showcmd->dstate,
2391     NCSCMD("watches", noit_console_show_watchlist, NULL, NULL, NULL));
2392 }
2393
2394 int
2395 noit_check_register_module(const char *name) {
2396   int i;
2397   for(i=0; i<reg_module_id; i++)
2398     if(!strcmp(reg_module_names[i], name)) return i;
2399   if(reg_module_id >= MAX_MODULE_REGISTRATIONS) return -1;
2400   mtevL(noit_debug, "Registered module %s as %d\n", name, i);
2401   i = reg_module_id++;
2402   reg_module_names[i] = strdup(name);
2403   mtev_conf_set_namespace(reg_module_names[i]);
2404   return i;
2405 }
2406 int
2407 noit_check_registered_module_by_name(const char *name) {
2408   int i;
2409   for(i=0; i<reg_module_id; i++)
2410     if(!strcmp(reg_module_names[i], name)) return i;
2411   return -1;
2412 }
2413 int
2414 noit_check_registered_module_cnt() {
2415   return reg_module_id;
2416 }
2417 const char *
2418 noit_check_registered_module(int idx) {
2419   if(reg_module_used < 0) reg_module_used = reg_module_id;
2420   mtevAssert(reg_module_used == reg_module_id);
2421   if(idx >= reg_module_id || idx < 0) return NULL;
2422   return reg_module_names[idx];
2423 }
2424
2425 void
2426 noit_check_set_module_metadata(noit_check_t *c, int idx, void *md, void (*freefunc)(void *)) {
2427   struct vp_w_free *tuple;
2428   if(reg_module_used < 0) reg_module_used = reg_module_id;
2429   mtevAssert(reg_module_used == reg_module_id);
2430   if(idx >= reg_module_id || idx < 0) return;
2431   if(!c->module_metadata) c->module_metadata = calloc(reg_module_id, sizeof(void *));
2432   c->module_metadata[idx] = calloc(1, sizeof(struct vp_w_free));
2433   tuple = c->module_metadata[idx];
2434   tuple->ptr = md;
2435   tuple->freefunc = freefunc;
2436 }
2437 void
2438 noit_check_set_module_config(noit_check_t *c, int idx, mtev_hash_table *config) {
2439   if(reg_module_used < 0) reg_module_used = reg_module_id;
2440   mtevAssert(reg_module_used == reg_module_id);
2441   if(idx >= reg_module_id || idx < 0) return;
2442   if(!c->module_configs) c->module_configs = calloc(reg_module_id, sizeof(mtev_hash_table *));
2443   c->module_configs[idx] = config;
2444 }
2445 void *
2446 noit_check_get_module_metadata(noit_check_t *c, int idx) {
2447   struct vp_w_free *tuple;
2448   if(reg_module_used < 0) reg_module_used = reg_module_id;
2449   mtevAssert(reg_module_used == reg_module_id);
2450   if(idx >= reg_module_id || idx < 0 || !c->module_metadata) return NULL;
2451   tuple = c->module_metadata[idx];
2452   return tuple ? tuple->ptr : NULL;
2453 }
2454 mtev_hash_table *
2455 noit_check_get_module_config(noit_check_t *c, int idx) {
2456   if(reg_module_used < 0) reg_module_used = reg_module_id;
2457   mtevAssert(reg_module_used == reg_module_id);
2458   if(idx >= reg_module_id || idx < 0 || !c->module_configs) return NULL;
2459   return c->module_configs[idx];
2460 }
2461 void
2462 noit_check_init_globals(void) {
2463   mtev_hash_init(&polls);
2464   mtev_hash_init(&dns_ignore_list);
2465 }
2466
Note: See TracBrowser for help on using the browser.