root/src/noit_check.c

Revision 304ec80b8cf842fc0abe5f9029790908b6455957, 67.0 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 1 week ago)

Convert to libmtev.

  • Property mode set to 100644
Line 
1 /*
2  * Copyright (c) 2007, OmniTI Computer Consulting, Inc.
3  * All rights reserved.
4  * Copyright (c) 2015, Circonus, Inc. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are
8  * met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above
13  *       copyright notice, this list of conditions and the following
14  *       disclaimer in the documentation and/or other materials provided
15  *       with the distribution.
16  *     * Neither the name OmniTI Computer Consulting, Inc. nor the names
17  *       of its contributors may be used to endorse or promote products
18  *       derived from this software without specific prior written
19  *       permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include "noit_config.h"
35 #include <mtev_defines.h>
36
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <unistd.h>
40 #include <ctype.h>
41 #include <assert.h>
42 #include <errno.h>
43 #include <netinet/in.h>
44 #include <arpa/inet.h>
45 #include <time.h>
46
47 #include <eventer/eventer.h>
48 #include <mtev_memory.h>
49 #include <mtev_log.h>
50 #include <mtev_hash.h>
51 #include <mtev_skiplist.h>
52 #include <mtev_watchdog.h>
53 #include <mtev_conf.h>
54 #include <mtev_console.h>
55
56 #include "noit_mtev_bridge.h"
57 #include "noit_dtrace_probes.h"
58 #include "noit_check.h"
59 #include "noit_module.h"
60 #include "noit_check_tools.h"
61 #include "noit_check_resolver.h"
62
63 #define DEFAULT_TEXT_METRIC_SIZE_LIMIT  512
64 #define RECYCLE_INTERVAL 60
65
66 MTEV_HOOK_IMPL(check_config_fixup,
67   (noit_check_t *check),
68   void *, closure,
69   (void *closure, noit_check_t *check),
70   (closure,check))
71
72 MTEV_HOOK_IMPL(check_stats_set_metric,
73   (noit_check_t *check, stats_t *stats, metric_t *m),
74   void *, closure,
75   (void *closure, noit_check_t *check, stats_t *stats, metric_t *m),
76   (closure,check,stats,m))
77
78 MTEV_HOOK_IMPL(check_stats_set_metric_coerce,
79   (noit_check_t *check, stats_t *stats, const char *name,
80    metric_type_t type, const char *v, mtev_boolean success),
81   void *, closure,
82   (void *closure, noit_check_t *check, stats_t *stats, const char *name,
83    metric_type_t type, const char *v, mtev_boolean success),
84   (closure,check,stats,name,type,v,success))
85
86 MTEV_HOOK_IMPL(check_passive_log_stats,
87   (noit_check_t *check),
88   void *, closure,
89   (void *closure, noit_check_t *check),
90   (closure,check))
91
92 MTEV_HOOK_IMPL(check_log_stats,
93   (noit_check_t *check),
94   void *, closure,
95   (void *closure, noit_check_t *check),
96   (closure,check))
97
98 /* 20 ms slots over 60 second for distribution */
99 #define SCHEDULE_GRANULARITY 20
100 #define SLOTS_PER_SECOND (1000/SCHEDULE_GRANULARITY)
101 #define MAX_MODULE_REGISTRATIONS 64
102
103 /* used to manage per-check generic module metadata */
104 struct vp_w_free {
105   void *ptr;
106   void (*freefunc)(void *);
107 };
108
109 static mtev_boolean system_needs_causality = mtev_false;
110 static int text_size_limit = DEFAULT_TEXT_METRIC_SIZE_LIMIT;
111 static int reg_module_id = 0;
112 static char *reg_module_names[MAX_MODULE_REGISTRATIONS] = { NULL };
113 static int reg_module_used = -1;
114 static u_int64_t check_completion_count = 0ULL;
115 static u_int64_t check_metrics_seen = 0ULL;
116 static pthread_mutex_t polls_lock = PTHREAD_MUTEX_INITIALIZER;
117 static mtev_hash_table polls = MTEV_HASH_EMPTY;
118 static mtev_hash_table dns_ignore_list = MTEV_HASH_EMPTY;
119 static mtev_skiplist watchlist = { 0 };
120 static mtev_skiplist polls_by_name = { 0 };
121 static u_int32_t __config_load_generation = 0;
122 static unsigned short check_slots_count[60000 / SCHEDULE_GRANULARITY] = { 0 },
123                       check_slots_seconds_count[60] = { 0 };
124
125 static noit_check_t *
126 noit_poller_lookup__nolock(uuid_t in) {
127   void *vcheck;
128   if(mtev_hash_retrieve(&polls, (char *)in, UUID_SIZE, &vcheck))
129     return (noit_check_t *)vcheck;
130   return NULL;
131 }
132 static noit_check_t *
133 noit_poller_lookup_by_name__nolock(char *target, char *name) {
134   noit_check_t tmp_check;
135   memset(&tmp_check, 0, sizeof(tmp_check));
136   tmp_check.target = target;
137   tmp_check.name = name;
138   return mtev_skiplist_find(&polls_by_name, &tmp_check, NULL);
139 }
140
141 static int
142 noit_console_show_timing_slots(mtev_console_closure_t ncct,
143                                int argc, char **argv,
144                                mtev_console_state_t *dstate,
145                                void *closure) {
146   int i, j;
147   const int upl = (60000 / SCHEDULE_GRANULARITY) / 60;
148   for(i=0;i<60;i++) {
149     nc_printf(ncct, "[%02d] %04d: ", i, check_slots_seconds_count[i]);
150     for(j=i*upl;j<(i+1)*upl;j++) {
151       char cp = '!';
152       if(check_slots_count[j] < 10) cp = '0' + check_slots_count[j];
153       else if(check_slots_count[j] < 36) cp = 'a' + (check_slots_count[j] - 10);
154       nc_printf(ncct, "%c", cp);
155     }
156     nc_printf(ncct, "\n");
157   }
158   return 0;
159 }
160 static int
161 noit_check_add_to_list(noit_check_t *new_check, const char *newname) {
162   char *oldname = NULL, *newnamecopy;
163   if(newname) {
164     /* track this stuff outside the lock to avoid allocs */
165     oldname = new_check->name;
166     newnamecopy = strdup(newname);
167   }
168   pthread_mutex_lock(&polls_lock);
169   if(!(new_check->flags & NP_TRANSIENT)) {
170     assert(new_check->name || newname);
171     /* This remove could fail -- no big deal */
172     if(new_check->name != NULL)
173       mtev_skiplist_remove(&polls_by_name, new_check, NULL);
174
175     /* optional update the name (at the critical point) */
176     if(newname) new_check->name = newnamecopy;
177
178     /* This insert could fail.. which means we have a conflict on
179      * target`name.  That should result in the check being disabled. */
180     if(!mtev_skiplist_insert(&polls_by_name, new_check)) {
181       mtevL(noit_error, "Check %s`%s disabled due to naming conflict\n",
182             new_check->target, new_check->name);
183       new_check->flags |= NP_DISABLED;
184     }
185     if(oldname) free(oldname);
186   }
187   pthread_mutex_unlock(&polls_lock);
188   return 1;
189 }
190
191 u_int64_t noit_check_metric_count() {
192   return check_metrics_seen;
193 }
194 void noit_check_metric_count_add(int add) {
195   mtev_atomic64_t *n = (mtev_atomic64_t *)&check_metrics_seen;
196   mtev_atomic64_t v = (mtev_atomic64_t)add;
197   mtev_atomic_add64(n, v);
198 }
199
200 u_int64_t noit_check_completion_count() {
201   return check_completion_count;
202 }
203 static void register_console_check_commands();
204 static int check_recycle_bin_processor(eventer_t, int, void *,
205                                        struct timeval *);
206
207 static int
208 check_slots_find_smallest(int sec, struct timeval* period) {
209   int i, j, cyclic, random_offset, jbase = 0, mini = 0, minj = 0;
210   unsigned short min_running_i = 0xffff, min_running_j = 0xffff;
211   int period_seconds = period->tv_sec;
212
213   /* If we're greater than sixty seconds, we should do our
214    * initial scheduling as if the period was sixty seconds. */
215   if (period_seconds > 60)
216     period_seconds = 60;
217
218   for(i=0;i<period_seconds;i++) {
219     int adj_i = (i + sec) % 60;
220     if(check_slots_seconds_count[adj_i] < min_running_i) {
221       min_running_i = check_slots_seconds_count[adj_i];
222       mini = adj_i;
223     }
224   }
225   jbase = mini * (1000/SCHEDULE_GRANULARITY);
226   random_offset = drand48() * SLOTS_PER_SECOND;
227   for(cyclic=0;cyclic<SLOTS_PER_SECOND;cyclic++) {
228     j = jbase + ((random_offset + cyclic) % SLOTS_PER_SECOND);
229     if(check_slots_count[j] < min_running_j) {
230       min_running_j = check_slots_count[j];
231       minj = j;
232     }
233   }
234   return (minj * SCHEDULE_GRANULARITY) + drand48() * SCHEDULE_GRANULARITY;
235 }
236 static void
237 check_slots_adjust_tv(struct timeval *tv, short adj) {
238   int offset_ms, idx;
239   offset_ms = (tv->tv_sec % 60) * 1000 + (tv->tv_usec / 1000);
240   idx = offset_ms / SCHEDULE_GRANULARITY;
241   check_slots_count[idx] += adj;
242   check_slots_seconds_count[offset_ms / 1000] += adj;
243 }
244 void check_slots_inc_tv(struct timeval *tv) {
245   check_slots_adjust_tv(tv, 1);
246 }
247 void check_slots_dec_tv(struct timeval *tv) {
248   check_slots_adjust_tv(tv, -1);
249 }
250 static int
251 noit_check_generic_safe_string(const char *p) {
252   if(!p) return 0;
253   for(;*p;p++) {
254     if(!isprint(*p)) return 0;
255   }
256   return 1;
257 }
258 int
259 noit_check_validate_target(const char *p) {
260   if(!noit_check_generic_safe_string(p)) return 0;
261   return 1;
262 }
263 int
264 noit_check_validate_name(const char *p) {
265   if(!noit_check_generic_safe_string(p)) return 0;
266   return 1;
267 }
268 const char *
269 noit_check_available_string(int16_t available) {
270   switch(available) {
271     case NP_AVAILABLE:    return "available";
272     case NP_UNAVAILABLE:  return "unavailable";
273     case NP_UNKNOWN:      return "unknown";
274   }
275   return NULL;
276 }
277 const char *
278 noit_check_state_string(int16_t state) {
279   switch(state) {
280     case NP_GOOD:         return "good";
281     case NP_BAD:          return "bad";
282     case NP_UNKNOWN:      return "unknown";
283   }
284   return NULL;
285 }
286 static int __check_name_compare(const void *a, const void *b) {
287   const noit_check_t *ac = a;
288   const noit_check_t *bc = b;
289   int rv;
290   if((rv = strcmp(ac->target, bc->target)) != 0) return rv;
291   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
292   return 0;
293 }
294 static int __watchlist_compare(const void *a, const void *b) {
295   const noit_check_t *ac = a;
296   const noit_check_t *bc = b;
297   int rv;
298   if((rv = memcmp(ac->checkid, bc->checkid, sizeof(ac->checkid))) != 0) return rv;
299   if(ac->period < bc->period) return -1;
300   if(ac->period == bc->period) return 0;
301   return 1;
302 }
303 static int __check_target_ip_compare(const void *a, const void *b) {
304   const noit_check_t *ac = a;
305   const noit_check_t *bc = b;
306   int rv;
307   if((rv = strcmp(ac->target_ip, bc->target_ip)) != 0) return rv;
308   if (ac->name == NULL) return 1;
309   if (bc->name == NULL) return -1;
310   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
311   return 1;
312 }
313 static int __check_target_compare(const void *a, const void *b) {
314   const noit_check_t *ac = a;
315   const noit_check_t *bc = b;
316   int rv;
317   if (ac->target == NULL) return 1;
318   if (bc->target == NULL) return -1;
319   if((rv = strcmp(ac->target, bc->target)) != 0) return rv;
320   if (ac->name == NULL) return 1;
321   if (bc->name == NULL) return -1;
322   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
323   return 1;
324 }
325 int
326 noit_calc_rtype_flag(char *resolve_rtype) {
327   int flags = 0;
328   if(resolve_rtype) {
329     flags |= strcmp(resolve_rtype, PREFER_IPV6) == 0 ||
330              strcmp(resolve_rtype, FORCE_IPV6) == 0 ? NP_PREFER_IPV6 : 0;
331     flags |= strcmp(resolve_rtype, FORCE_IPV4) == 0 ||
332              strcmp(resolve_rtype, FORCE_IPV6) == 0 ? NP_SINGLE_RESOLVE : 0;
333   }
334   return flags;
335 }
336 void
337 noit_check_fake_last_check(noit_check_t *check,
338                            struct timeval *lc, struct timeval *_now) {
339   struct timeval now, period, lc_copy;
340   int balance_ms;
341
342   if(!_now) {
343     gettimeofday(&now, NULL);
344     _now = &now;
345   }
346   period.tv_sec = check->period / 1000;
347   period.tv_usec = (check->period % 1000) * 1000;
348   sub_timeval(*_now, period, lc);
349
350   /* We need to set the last check value based on the period, but
351    * we also need to store a value that is based around the one-minute
352    * time to properly increment the slots; otherwise, the slots will
353    * get all messed up */
354   if(!(check->flags & NP_TRANSIENT) && check->period) {
355     balance_ms = check_slots_find_smallest(_now->tv_sec+1, &period);
356     lc->tv_sec = (lc->tv_sec / 60) * 60 + balance_ms / 1000;
357     lc->tv_usec = (balance_ms % 1000) * 1000;
358     memcpy(&lc_copy, lc, sizeof(lc_copy));
359     if(compare_timeval(*_now, *lc) < 0) {
360       do {
361         sub_timeval(*lc, period, lc);
362       } while(compare_timeval(*_now, *lc) < 0);
363     }
364     else {
365       struct timeval test;
366       while(1) {
367         add_timeval(*lc, period, &test);
368         if(compare_timeval(*_now, test) < 0) break;
369         memcpy(lc, &test, sizeof(test));
370       }
371     }
372   }
373   else {
374     memcpy(&lc_copy, lc, sizeof(lc_copy));
375   }
376  
377   /* now, we're going to do an even distribution using the slots */
378   if(!(check->flags & NP_TRANSIENT)) check_slots_inc_tv(&lc_copy);
379 }
380 void
381 noit_poller_process_checks(const char *xpath) {
382   int i, flags, cnt = 0, found;
383   mtev_conf_section_t *sec;
384   __config_load_generation++;
385   sec = mtev_conf_get_sections(NULL, xpath, &cnt);
386   for(i=0; i<cnt; i++) {
387     void *vcheck;
388     char uuid_str[37];
389     char target[256] = "";
390     char module[256] = "";
391     char name[256] = "";
392     char filterset[256] = "";
393     char oncheck[1024] = "";
394     char resolve_rtype[16] = "";
395     int ridx;
396     int no_period = 0;
397     int no_oncheck = 0;
398     int period = 0, timeout = 0;
399     mtev_boolean disabled = mtev_false, busted = mtev_false;
400     uuid_t uuid, out_uuid;
401     mtev_hash_table *options;
402     mtev_hash_table **moptions = NULL;
403     mtev_boolean moptions_used = mtev_false;
404
405     /* We want to heartbeat here... otherwise, if a lot of checks are
406      * configured or if we're running on a slower system, we could
407      * end up getting watchdog killed before we get a chance to run
408      * any checks */
409     mtev_watchdog_child_heartbeat();
410
411     if(reg_module_id > 0) {
412       moptions = alloca(reg_module_id * sizeof(mtev_hash_table *));
413       memset(moptions, 0, reg_module_id * sizeof(mtev_hash_table *));
414       moptions_used = mtev_true;
415     }
416
417 #define NEXT(...) mtevL(noit_stderr, __VA_ARGS__); continue
418 #define MYATTR(type,a,...) mtev_conf_get_##type(sec[i], "@" #a, __VA_ARGS__)
419 #define INHERIT(type,a,...) \
420   mtev_conf_get_##type(sec[i], "ancestor-or-self::node()/@" #a, __VA_ARGS__)
421
422     if(!MYATTR(stringbuf, uuid, uuid_str, sizeof(uuid_str))) {
423       mtevL(noit_stderr, "check %d has no uuid\n", i+1);
424       continue;
425     }
426
427     if(uuid_parse(uuid_str, uuid)) {
428       mtevL(noit_stderr, "check uuid: '%s' is invalid\n", uuid_str);
429       continue;
430     }
431
432     if(!INHERIT(stringbuf, target, target, sizeof(target))) {
433       mtevL(noit_stderr, "check uuid: '%s' has no target\n", uuid_str);
434       busted = mtev_true;
435     }
436     if(!noit_check_validate_target(target)) {
437       mtevL(noit_stderr, "check uuid: '%s' has malformed target\n", uuid_str);
438       busted = mtev_true;
439     }
440     if(!INHERIT(stringbuf, module, module, sizeof(module))) {
441       mtevL(noit_stderr, "check uuid: '%s' has no module\n", uuid_str);
442       busted = mtev_true;
443     }
444
445     if(!INHERIT(stringbuf, filterset, filterset, sizeof(filterset)))
446       filterset[0] = '\0';
447    
448     if (!INHERIT(stringbuf, resolve_rtype, resolve_rtype, sizeof(resolve_rtype)))
449       strlcpy(resolve_rtype, PREFER_IPV4, sizeof(resolve_rtype));
450
451     if(!MYATTR(stringbuf, name, name, sizeof(name)))
452       strlcpy(name, module, sizeof(name));
453
454     if(!noit_check_validate_name(name)) {
455       mtevL(noit_stderr, "check uuid: '%s' has malformed name\n", uuid_str);
456       busted = mtev_true;
457     }
458
459     if(!INHERIT(int, period, &period) || period == 0)
460       no_period = 1;
461
462     if(!INHERIT(stringbuf, oncheck, oncheck, sizeof(oncheck)) || !oncheck[0])
463       no_oncheck = 1;
464
465     if(no_period && no_oncheck) {
466       mtevL(noit_stderr, "check uuid: '%s' has neither period nor oncheck\n",
467             uuid_str);
468       busted = mtev_true;
469     }
470     if(!(no_period || no_oncheck)) {
471       mtevL(noit_stderr, "check uuid: '%s' has oncheck and period.\n",
472             uuid_str);
473       busted = mtev_true;
474     }
475     if(!INHERIT(int, timeout, &timeout)) {
476       mtevL(noit_stderr, "check uuid: '%s' has no timeout\n", uuid_str);
477       busted = mtev_true;
478     }
479     if(!no_period && timeout >= period) {
480       mtevL(noit_stderr, "check uuid: '%s' timeout > period\n", uuid_str);
481       timeout = period/2;
482     }
483     options = mtev_conf_get_hash(sec[i], "config");
484     for(ridx=0; ridx<reg_module_id; ridx++) {
485       moptions[ridx] = mtev_conf_get_namespaced_hash(sec[i], "config",
486                                                      reg_module_names[ridx]);
487     }
488
489     INHERIT(boolean, disable, &disabled);
490     flags = 0;
491     if(busted) flags |= (NP_UNCONFIG|NP_DISABLED);
492     else if(disabled) flags |= NP_DISABLED;
493
494     flags |= noit_calc_rtype_flag(resolve_rtype);
495
496     pthread_mutex_lock(&polls_lock);
497     found = mtev_hash_retrieve(&polls, (char *)uuid, UUID_SIZE, &vcheck);
498     pthread_mutex_unlock(&polls_lock);
499     if(found)
500       noit_poller_deschedule(uuid);
501     noit_poller_schedule(target, module, name, filterset, options,
502                          moptions_used ? moptions : NULL,
503                          period, timeout, oncheck[0] ? oncheck : NULL,
504                          flags, uuid, out_uuid);
505     mtevL(noit_debug, "loaded uuid: %s\n", uuid_str);
506
507     for(ridx=0; ridx<reg_module_id; ridx++) {
508       if(moptions[ridx]) {
509         mtev_hash_destroy(moptions[ridx], free, free);
510         free(moptions[ridx]);
511       }
512     }
513     mtev_hash_destroy(options, free, free);
514     free(options);
515   }
516   if(sec) free(sec);
517 }
518
519 int
520 noit_check_activate(noit_check_t *check) {
521   noit_module_t *mod;
522   if(NOIT_CHECK_LIVE(check)) return 0;
523   mod = noit_module_lookup(check->module);
524   if(mod && mod->initiate_check) {
525     if((check->flags & NP_DISABLED) == 0) {
526       mod->initiate_check(mod, check, 0, NULL);
527       return 1;
528     }
529     else
530       mtevL(noit_debug, "Skipping %s`%s, disabled.\n",
531             check->target, check->name);
532   }
533   else {
534     if(!mod) {
535       mtevL(noit_stderr, "Cannot find module '%s'\n", check->module);
536       check->flags |= NP_DISABLED;
537     }
538   }
539   return 0;
540 }
541
542 void
543 noit_poller_initiate() {
544   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
545   uuid_t key_id;
546   int klen;
547   void *vcheck;
548   /* This is only ever called in the beginning, no lock needed */
549   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
550                        &vcheck)) {
551     noit_check_activate((noit_check_t *)vcheck);
552     mtev_watchdog_child_heartbeat();
553   }
554 }
555
556 void
557 noit_poller_flush_epoch(int oldest_allowed) {
558   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
559   uuid_t key_id;
560   int klen, i;
561   void *vcheck;
562 #define TOFREE_PER_ITER 1024
563   noit_check_t *tofree[TOFREE_PER_ITER];
564
565   /* Cleanup any previous causal map */
566   while(1) {
567     i = 0;
568     pthread_mutex_lock(&polls_lock);
569     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
570                          &vcheck) && i < TOFREE_PER_ITER) {
571       noit_check_t *check = (noit_check_t *)vcheck;
572       if(check->generation < oldest_allowed) {
573         tofree[i++] = check;
574       }
575     }
576     pthread_mutex_unlock(&polls_lock);
577     if(i==0) break;
578     while(i>0) noit_poller_deschedule(tofree[--i]->checkid);
579   }
580 #undef TOFREE_PER_ITER
581 }
582
583 void
584 noit_poller_make_causal_map() {
585   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
586   uuid_t key_id;
587   int klen;
588   void *vcheck;
589
590   if(!system_needs_causality) return;
591
592   /* set it to false, we'll set it to true during the scan if we
593    * find anything causal.  */
594   system_needs_causality = mtev_false;
595
596   /* Cleanup any previous causal map */
597   pthread_mutex_lock(&polls_lock);
598   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
599                        &vcheck)) {
600     noit_check_t *check = (noit_check_t *)vcheck;
601     dep_list_t *dep;
602     while((dep = check->causal_checks) != NULL) {
603       check->causal_checks = dep->next;
604       free(dep);
605     }
606   }
607
608   memset(&iter, 0, sizeof(iter));
609   /* Walk all checks and add check dependencies to their parents */
610   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
611                        &vcheck)) {
612     noit_check_t *check = (noit_check_t *)vcheck, *parent;
613     if(check->oncheck) {
614       /* This service is causally triggered by another service */
615       uuid_t id;
616       char fullcheck[1024];
617       char *name = check->oncheck;
618       char *target = NULL;
619
620       system_needs_causality = mtev_true;
621       mtevL(noit_debug, "Searching for upstream trigger on %s\n", name);
622       parent = NULL;
623       if(uuid_parse(check->oncheck, id) == 0) {
624         target = "";
625         parent = noit_poller_lookup__nolock(id);
626       }
627       else if((target = strchr(check->oncheck, '`')) != NULL) {
628         strlcpy(fullcheck, check->oncheck, target + 1 - check->oncheck);
629         name = target + 1;
630         target = fullcheck;
631         parent = noit_poller_lookup_by_name__nolock(target, name);
632       }
633       else {
634         target = check->target;
635         parent = noit_poller_lookup_by_name__nolock(target, name);
636       }
637
638       if(!parent) {
639         check->flags |= NP_DISABLED;
640         mtevL(noit_stderr, "Disabling check %s`%s, can't find oncheck %s`%s\n",
641               check->target, check->name, target, name);
642       }
643       else {
644         dep_list_t *dep;
645         dep = malloc(sizeof(*dep));
646         dep->check = check;
647         dep->next = parent->causal_checks;
648         parent->causal_checks = dep;
649         mtevL(noit_debug, "Causal map %s`%s --> %s`%s\n",
650               parent->target, parent->name, check->target, check->name);
651       }
652     }
653   }
654   pthread_mutex_unlock(&polls_lock);
655   /* We found some causal checks, so we might need to activate stuff */
656   if(system_needs_causality) noit_poller_initiate();
657 }
658 void
659 noit_poller_reload(const char *xpath)
660 {
661   noit_poller_process_checks(xpath ? xpath : "/noit/checks//check");
662   if(!xpath) {
663     /* Full reload, we need to wipe old checks */
664     noit_poller_flush_epoch(__config_load_generation);
665   }
666   noit_poller_make_causal_map();
667 }
668 void
669 noit_check_dns_ignore_tld(const char* extension, const char* ignore) {
670   mtev_hash_replace(&dns_ignore_list, strdup(extension), strlen(extension), strdup(ignore), NULL, NULL);
671 }
672 static void
673 noit_check_dns_ignore_list_init() {
674   mtev_conf_section_t* dns;
675   int cnt;
676
677   dns = mtev_conf_get_sections(NULL, "/noit/dns/extension", &cnt);
678   if(dns) {
679     int i = 0;
680     for (i = 0; i < cnt; i++) {
681       char* extension;
682       char* ignore;
683       if(!mtev_conf_get_string(dns[i], "self::node()/@value", &extension)) {
684         continue;
685       }
686       if(!mtev_conf_get_string(dns[i], "self::node()/@ignore", &ignore)) {
687         continue;
688       }
689       noit_check_dns_ignore_tld(extension, ignore);
690     }
691   }
692 }
693 void
694 noit_poller_init() {
695   srand48((getpid() << 16) ^ time(NULL));
696   noit_check_resolver_init();
697   noit_check_tools_init();
698   mtev_skiplist_init(&polls_by_name);
699   mtev_skiplist_set_compare(&polls_by_name, __check_name_compare,
700                             __check_name_compare);
701   mtev_skiplist_add_index(&polls_by_name, __check_target_ip_compare,
702                             __check_target_ip_compare);
703   mtev_skiplist_add_index(&polls_by_name, __check_target_compare,
704                             __check_target_compare);
705   mtev_skiplist_init(&watchlist);
706   mtev_skiplist_set_compare(&watchlist, __watchlist_compare,
707                             __watchlist_compare);
708   register_console_check_commands();
709   eventer_name_callback("check_recycle_bin_processor",
710                         check_recycle_bin_processor);
711   eventer_add_in_s_us(check_recycle_bin_processor, NULL, RECYCLE_INTERVAL, 0);
712   mtev_conf_get_int(NULL, "noit/@text_size_limit", &text_size_limit);
713   if (text_size_limit <= 0) {
714     text_size_limit = DEFAULT_TEXT_METRIC_SIZE_LIMIT;
715   }
716   noit_check_dns_ignore_list_init();
717   noit_poller_reload(NULL);
718 }
719
720 int
721 noit_poller_check_count() {
722   return polls_by_name.size;
723 }
724
725 int
726 noit_poller_transient_check_count() {
727   return watchlist.size;
728 }
729
730 noit_check_t *
731 noit_check_clone(uuid_t in) {
732   int i;
733   noit_check_t *checker, *new_check;
734   void *vcheck;
735   if(mtev_hash_retrieve(&polls,
736                         (char *)in, UUID_SIZE,
737                         &vcheck) == 0) {
738     return NULL;
739   }
740   checker = (noit_check_t *)vcheck;
741   if(checker->oncheck) {
742     return NULL;
743   }
744   new_check = calloc(1, sizeof(*new_check));
745   memcpy(new_check, checker, sizeof(*new_check));
746   new_check->target = strdup(new_check->target);
747   new_check->module = strdup(new_check->module);
748   new_check->name = strdup(new_check->name);
749   new_check->filterset = strdup(new_check->filterset);
750   new_check->flags = 0;
751   new_check->fire_event = NULL;
752   memset(&new_check->last_fire_time, 0, sizeof(new_check->last_fire_time));
753   memset(&new_check->stats, 0, sizeof(new_check->stats));
754   new_check->closure = NULL;
755   new_check->config = calloc(1, sizeof(*new_check->config));
756   mtev_hash_merge_as_dict(new_check->config, checker->config);
757   new_check->module_configs = NULL;
758   new_check->module_metadata = NULL;
759
760   for(i=0; i<reg_module_id; i++) {
761     void *src_metadata;
762     mtev_hash_table *src_mconfig;
763     src_mconfig = noit_check_get_module_config(checker, i);
764     if(src_mconfig) {
765       mtev_hash_table *t = calloc(1, sizeof(*new_check->config));
766       mtev_hash_merge_as_dict(t, src_mconfig);
767       noit_check_set_module_config(new_check, i, t);
768     }
769     if(checker->flags & NP_PASSIVE_COLLECTION)
770       if(NULL != (src_metadata = noit_check_get_module_metadata(new_check, i)))
771         noit_check_set_module_metadata(new_check, i, src_metadata, NULL);
772   }
773   return new_check;
774 }
775
776 noit_check_t *
777 noit_check_watch(uuid_t in, int period) {
778   /* First look for a copy that is being watched */
779   int minimum_pi = 1000, granularity_pi = 500;
780   mtev_conf_section_t check_node;
781   char uuid_str[UUID_STR_LEN + 1];
782   char xpath[1024];
783   noit_check_t n, *f;
784
785   uuid_unparse_lower(in, uuid_str);
786
787   mtevL(noit_debug, "noit_check_watch(%s,%d)\n", uuid_str, period);
788   if(period == 0) {
789     return noit_poller_lookup(in);
790   }
791
792   /* Find the check */
793   snprintf(xpath, sizeof(xpath), "//checks//check[@uuid=\"%s\"]", uuid_str);
794   check_node = mtev_conf_get_section(NULL, xpath);
795   mtev_conf_get_int(NULL, "//checks/@transient_min_period", &minimum_pi);
796   mtev_conf_get_int(NULL, "//checks/@transient_period_granularity", &granularity_pi);
797   if(check_node) {
798     mtev_conf_get_int(check_node,
799                       "ancestor-or-self::node()/@transient_min_period",
800                       &minimum_pi);
801     mtev_conf_get_int(check_node,
802                       "ancestor-or-self::node()/@transient_period_granularity",
803                       &granularity_pi);
804   }
805
806   /* apply the bounds */
807   period /= granularity_pi;
808   period *= granularity_pi;
809   period = MAX(period, minimum_pi);
810
811   uuid_copy(n.checkid, in);
812   n.period = period;
813
814   f = mtev_skiplist_find(&watchlist, &n, NULL);
815   if(f) return f;
816   f = noit_check_clone(in);
817   if(!f) return NULL;
818   f->period = period;
819   f->timeout = period - 10;
820   f->flags |= NP_TRANSIENT;
821   mtevL(noit_debug, "Watching %s@%d\n", uuid_str, period);
822   mtev_skiplist_insert(&watchlist, f);
823   return f;
824 }
825
826 noit_check_t *
827 noit_check_get_watch(uuid_t in, int period) {
828   noit_check_t n, *f;
829
830   uuid_copy(n.checkid, in);
831   n.period = period;
832
833   f = mtev_skiplist_find(&watchlist, &n, NULL);
834   return f;
835 }
836
837 void
838 noit_check_transient_add_feed(noit_check_t *check, const char *feed) {
839   char *feedcopy;
840   if(!check->feeds) {
841     check->feeds = calloc(1, sizeof(*check->feeds));
842     mtev_skiplist_init(check->feeds);
843     mtev_skiplist_set_compare(check->feeds,
844                               (mtev_skiplist_comparator_t)strcmp,
845                               (mtev_skiplist_comparator_t)strcmp);
846   }
847   feedcopy = strdup(feed);
848   /* No error on failure -- it's already there */
849   if(mtev_skiplist_insert(check->feeds, feedcopy) == NULL) free(feedcopy);
850   mtevL(noit_debug, "check %s`%s @ %dms has %d feed(s): %s.\n",
851         check->target, check->name, check->period, check->feeds->size, feed);
852 }
853 void
854 noit_check_transient_remove_feed(noit_check_t *check, const char *feed) {
855   if(!check->feeds) return;
856   if(feed) {
857     mtevL(noit_debug, "check %s`%s @ %dms removing 1 of %d feeds: %s.\n",
858           check->target, check->name, check->period, check->feeds->size, feed);
859     mtev_skiplist_remove(check->feeds, feed, free);
860   }
861   if(check->feeds->size == 0) {
862     char uuid_str[UUID_STR_LEN + 1];
863     uuid_unparse_lower(check->checkid, uuid_str);
864     mtevL(noit_debug, "Unwatching %s@%d\n", uuid_str, check->period);
865     mtev_skiplist_remove(&watchlist, check, NULL);
866     mtev_skiplist_destroy(check->feeds, free);
867     free(check->feeds);
868     check->feeds = NULL;
869     if(check->flags & NP_TRANSIENT) {
870       mtevL(noit_debug, "check %s`%s @ %dms has no more listeners.\n",
871             check->target, check->name, check->period);
872       check->flags |= NP_KILLED;
873     }
874     noit_poller_free_check(check);
875   }
876 }
877
878 mtev_boolean
879 noit_check_is_valid_target(const char *target) {
880   int8_t family;
881   int rv;
882   union {
883     struct in_addr addr4;
884     struct in6_addr addr6;
885   } a;
886
887   family = AF_INET;
888   rv = inet_pton(family, target, &a);
889   if(rv != 1) {
890     family = AF_INET6;
891     rv = inet_pton(family, target, &a);
892     if(rv != 1) {
893       return mtev_false;
894     }
895   }
896   return mtev_true;
897 }
898 int
899 noit_check_set_ip(noit_check_t *new_check,
900                   const char *ip_str, const char *newname) {
901   int8_t family;
902   int rv, failed = 0;
903   char old_target_ip[INET6_ADDRSTRLEN];
904   union {
905     struct in_addr addr4;
906     struct in6_addr addr6;
907   } a;
908
909   memset(old_target_ip, 0, INET6_ADDRSTRLEN);
910   strlcpy(old_target_ip, new_check->target_ip, sizeof(old_target_ip));
911
912   family = NOIT_CHECK_PREFER_V6(new_check) ? AF_INET6 : AF_INET;
913   rv = inet_pton(family, ip_str, &a);
914   if(rv != 1) {
915     if (!NOIT_CHECK_SINGLE_RESOLVE(new_check)) {
916       family = family == AF_INET ? AF_INET6 : AF_INET;
917       rv = inet_pton(family, ip_str, &a);
918       if(rv != 1) {
919         family = AF_INET;
920         memset(&a, 0, sizeof(a));
921         failed = -1;
922       }
923     } else {
924       failed = -1;
925     }
926   }
927
928   new_check->target_family = family;
929   memcpy(&new_check->target_addr, &a, sizeof(a));
930   new_check->target_ip[0] = '\0';
931   if(failed == 0)
932     if(inet_ntop(new_check->target_family,
933                  &new_check->target_addr,
934                  new_check->target_ip,
935                  sizeof(new_check->target_ip)) == NULL) {
936       mtevL(noit_error, "inet_ntop failed [%s] -> %d\n", ip_str, errno);
937     }
938   /*
939    * new_check->name could be null if this check is being set for the
940    * first time.  add_to_list will set it.
941    */
942   if (new_check->name == NULL ||
943       strcmp(old_target_ip, new_check->target_ip) != 0) {
944     noit_check_add_to_list(new_check, newname);
945   }
946
947   if(new_check->name == NULL && newname != NULL) {
948     assert(new_check->flags & NP_TRANSIENT);
949     new_check->name = strdup(newname);
950   }
951
952   return failed;
953 }
954 int
955 noit_check_resolve(noit_check_t *check) {
956   uint8_t family_pref = NOIT_CHECK_PREFER_V6(check) ? AF_INET6 : AF_INET;
957   char ipaddr[INET6_ADDRSTRLEN];
958   if(!NOIT_CHECK_SHOULD_RESOLVE(check)) return 1; /* success, not required */
959   noit_check_resolver_remind(check->target);
960   if(noit_check_resolver_fetch(check->target, ipaddr, sizeof(ipaddr),
961                                family_pref) >= 0) {
962     check->flags |= NP_RESOLVED;
963     noit_check_set_ip(check, ipaddr, NULL);
964     return 0;
965   }
966   check->flags &= ~NP_RESOLVED;
967   return -1;
968 }
969 int
970 noit_check_update(noit_check_t *new_check,
971                   const char *target,
972                   const char *name,
973                   const char *filterset,
974                   mtev_hash_table *config,
975                   mtev_hash_table **mconfigs,
976                   u_int32_t period,
977                   u_int32_t timeout,
978                   const char *oncheck,
979                   int flags) {
980   int mask = NP_DISABLED | NP_UNCONFIG;
981
982   assert(name);
983
984   if(NOIT_CHECK_RUNNING(new_check)) {
985     char module[256];
986     uuid_t id, dummy;
987     uuid_copy(id, new_check->checkid);
988     strlcpy(module, new_check->module, sizeof(module));
989     noit_poller_deschedule(id);
990     return noit_poller_schedule(target, module, name, filterset,
991                                 config, mconfigs, period, timeout, oncheck,
992                                 flags, id, dummy);
993   }
994
995   new_check->generation = __config_load_generation;
996   if(new_check->target) free(new_check->target);
997   new_check->target = strdup(target);
998
999   // apply resolution flags to check.
1000   if (flags & NP_PREFER_IPV6)
1001     new_check->flags |= NP_PREFER_IPV6;
1002   else
1003     new_check->flags &= ~NP_PREFER_IPV6;
1004   if (flags & NP_SINGLE_RESOLVE)
1005     new_check->flags |= NP_SINGLE_RESOLVE;
1006   else
1007     new_check->flags &= ~NP_SINGLE_RESOLVE;
1008   if (flags & NP_RESOLVE)
1009     new_check->flags |= NP_RESOLVE;
1010   else
1011     new_check->flags &= ~NP_RESOLVE;
1012
1013   /* This sets both the name and the target_addr */
1014   if(noit_check_set_ip(new_check, target, name)) {
1015     mtev_boolean should_resolve;
1016     mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1017     const char *key, *value;
1018     int klen;
1019     char* extension = strrchr(target, '.');
1020     new_check->flags |= NP_RESOLVE;
1021     new_check->flags &= ~NP_RESOLVED;
1022     /* If we match any of the extensions we're supposed to ignore,
1023      * don't resolve */
1024     if (extension && (strlen(extension) > 1)) {
1025       while(mtev_hash_next(&dns_ignore_list, &iter, &key, &klen, (void**)&value)) {
1026         if ((!strcmp("true", value)) && (!strcmp(extension+1, key))) {
1027             new_check->flags &= ~NP_RESOLVE;
1028             break;
1029         }
1030       }
1031     }
1032     if(noit_check_should_resolve_targets(&should_resolve) && !should_resolve)
1033       flags |= NP_DISABLED | NP_UNCONFIG;
1034     noit_check_resolve(new_check);
1035   }
1036
1037   if(new_check->filterset) free(new_check->filterset);
1038   new_check->filterset = filterset ? strdup(filterset): NULL;
1039
1040   if(config != NULL) {
1041     mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1042     const char *k;
1043     int klen;
1044     void *data;
1045     if(new_check->config) mtev_hash_delete_all(new_check->config, free, free);
1046     else new_check->config = calloc(1, sizeof(*new_check->config));
1047     while(mtev_hash_next(config, &iter, &k, &klen, &data)) {
1048       mtev_hash_store(new_check->config, strdup(k), klen, strdup((char *)data));
1049     }
1050   }
1051   if(mconfigs != NULL) {
1052     int i;
1053     for(i=0; i<reg_module_id; i++) {
1054       mtev_hash_table *t;
1055       if(NULL != (t = noit_check_get_module_config(new_check, i))) {
1056         noit_check_set_module_config(new_check, i, NULL);
1057         mtev_hash_destroy(t, free, free);
1058         free(t);
1059       }
1060       if(mconfigs[i]) {
1061         mtev_hash_table *t = calloc(1, sizeof(*new_check->config));
1062         mtev_hash_merge_as_dict(t, mconfigs[i]);
1063         noit_check_set_module_config(new_check, i, t);
1064       }
1065     }
1066   }
1067   if(new_check->oncheck) free(new_check->oncheck);
1068   new_check->oncheck = oncheck ? strdup(oncheck) : NULL;
1069   if(new_check->oncheck) system_needs_causality = mtev_true;
1070   new_check->period = period;
1071   new_check->timeout = timeout;
1072
1073   /* Unset what could be set.. then set what should be set */
1074   new_check->flags = (new_check->flags & ~mask) | flags;
1075
1076   check_config_fixup_hook_invoke(new_check);
1077
1078   if((new_check->flags & NP_TRANSIENT) == 0)
1079     noit_check_activate(new_check);
1080
1081   noit_check_add_to_list(new_check, NULL);
1082   noit_check_log_check(new_check);
1083   return 0;
1084 }
1085 int
1086 noit_poller_schedule(const char *target,
1087                      const char *module,
1088                      const char *name,
1089                      const char *filterset,
1090                      mtev_hash_table *config,
1091                      mtev_hash_table **mconfigs,
1092                      u_int32_t period,
1093                      u_int32_t timeout,
1094                      const char *oncheck,
1095                      int flags,
1096                      uuid_t in,
1097                      uuid_t out) {
1098   noit_check_t *new_check;
1099   new_check = calloc(1, sizeof(*new_check));
1100   if(!new_check) return -1;
1101
1102   /* The module and the UUID can never be changed */
1103   new_check->module = strdup(module);
1104   if(uuid_is_null(in))
1105     uuid_generate(new_check->checkid);
1106   else
1107     uuid_copy(new_check->checkid, in);
1108
1109   noit_check_update(new_check, target, name, filterset, config, mconfigs,
1110                     period, timeout, oncheck, flags);
1111   assert(mtev_hash_store(&polls,
1112                          (char *)new_check->checkid, UUID_SIZE,
1113                          new_check));
1114   uuid_copy(out, new_check->checkid);
1115
1116   return 0;
1117 }
1118
1119 /* A quick little list of recycleable checks.  This list never really
1120  * grows large, so no sense in thinking too hard about the algorithmic
1121  * complexity.
1122  */
1123 struct _checker_rcb {
1124   noit_check_t *checker;
1125   struct _checker_rcb *next;
1126 };
1127 static struct _checker_rcb *checker_rcb = NULL;
1128 static void recycle_check(noit_check_t *checker) {
1129   struct _checker_rcb *n = malloc(sizeof(*n));
1130   n->checker = checker;
1131   n->next = checker_rcb;
1132   checker_rcb = n;
1133 }
1134 void
1135 free_metric(metric_t *m) {
1136   if(!m) return;
1137   if(m->metric_name) mtev_memory_safe_free(m->metric_name);
1138   if(m->metric_value.i) mtev_memory_safe_free(m->metric_value.i);
1139   mtev_memory_safe_free(m);
1140 }
1141 void
1142 noit_poller_free_check(noit_check_t *checker) {
1143   noit_module_t *mod;
1144
1145   if(checker->flags & NP_RUNNING) {
1146     recycle_check(checker);
1147     return;
1148   }
1149
1150   mod = noit_module_lookup(checker->module);
1151   if(mod && mod->cleanup) mod->cleanup(mod, checker);
1152   if(checker->fire_event) {
1153      eventer_remove(checker->fire_event);
1154      free(checker->fire_event->closure);
1155      eventer_free(checker->fire_event);
1156      checker->fire_event = NULL;
1157   }
1158   if(checker->closure) free(checker->closure);
1159   if(checker->target) free(checker->target);
1160   if(checker->module) free(checker->module);
1161   if(checker->name) free(checker->name);
1162   if(checker->config) {
1163     mtev_hash_destroy(checker->config, free, free);
1164     free(checker->config);
1165     checker->config = NULL;
1166   }
1167   if(checker->module_metadata) {
1168     int i;
1169     for(i=0; i<reg_module_id; i++) {
1170       struct vp_w_free *tuple;
1171       tuple = checker->module_metadata[i];
1172       if(tuple) {
1173         if(tuple->freefunc) tuple->freefunc(tuple->ptr);
1174         free(tuple);
1175       }
1176     }
1177     free(checker->module_metadata);
1178   }
1179   if(checker->module_configs) {
1180     int i;
1181     for(i=0; i<reg_module_id; i++) {
1182       if(checker->module_configs[i]) {
1183         mtev_hash_destroy(checker->module_configs[i], free, free);
1184         free(checker->module_configs[i]);
1185       }
1186     }
1187     free(checker->module_configs);
1188   }
1189   if(checker->stats.inprogress.status) free(checker->stats.inprogress.status);
1190   mtev_hash_destroy(&checker->stats.inprogress.metrics, NULL,
1191                     (void (*)(void *))free_metric);
1192   if(checker->stats.current.status) free(checker->stats.current.status);
1193   mtev_hash_destroy(&checker->stats.current.metrics, NULL,
1194                     (void (*)(void *))free_metric);
1195   if(checker->stats.previous.status) free(checker->stats.previous.status);
1196   mtev_hash_destroy(&checker->stats.previous.metrics, NULL,
1197                     (void (*)(void *))free_metric);
1198   free(checker);
1199 }
1200 static int
1201 check_recycle_bin_processor(eventer_t e, int mask, void *closure,
1202                             struct timeval *now) {
1203   static struct timeval one_minute = { RECYCLE_INTERVAL, 0L };
1204   struct _checker_rcb *prev = NULL, *curr = checker_rcb;
1205   mtevL(noit_debug, "Scanning check recycle bin\n");
1206   while(curr) {
1207     if(!(curr->checker->flags & NP_RUNNING)) {
1208       mtevL(noit_debug, "Check is ready to free.\n");
1209       noit_poller_free_check(curr->checker);
1210       if(prev) prev->next = curr->next;
1211       else checker_rcb = curr->next;
1212       free(curr);
1213       curr = prev ? prev->next : checker_rcb;
1214     }
1215     else {
1216       prev = curr;
1217       curr = curr->next;
1218     }
1219   }
1220   add_timeval(*now, one_minute, &e->whence);
1221   return EVENTER_TIMER;
1222 }
1223
1224 int
1225 noit_poller_deschedule(uuid_t in) {
1226   void *vcheck;
1227   noit_check_t *checker;
1228   if(mtev_hash_retrieve(&polls,
1229                         (char *)in, UUID_SIZE,
1230                         &vcheck) == 0) {
1231     return -1;
1232   }
1233   checker = (noit_check_t *)vcheck;
1234   checker->flags |= (NP_DISABLED|NP_KILLED);
1235
1236   noit_check_log_delete(checker);
1237
1238   assert(mtev_skiplist_remove(&polls_by_name, checker, NULL));
1239   assert(mtev_hash_delete(&polls, (char *)in, UUID_SIZE, NULL, NULL));
1240
1241   noit_poller_free_check(checker);
1242   return 0;
1243 }
1244
1245 noit_check_t *
1246 noit_poller_lookup(uuid_t in) {
1247   noit_check_t *check;
1248   pthread_mutex_lock(&polls_lock);
1249   check = noit_poller_lookup__nolock(in);
1250   pthread_mutex_unlock(&polls_lock);
1251   return check;
1252 }
1253 noit_check_t *
1254 noit_poller_lookup_by_name(char *target, char *name) {
1255   noit_check_t *check;
1256   pthread_mutex_lock(&polls_lock);
1257   check = noit_poller_lookup_by_name__nolock(target,name);
1258   pthread_mutex_unlock(&polls_lock);
1259   return check;
1260 }
1261 int
1262 noit_poller_target_ip_do(const char *target_ip,
1263                          int (*f)(noit_check_t *, void *),
1264                          void *closure) {
1265   int i, count = 0, todo_count = 0;
1266   noit_check_t pivot;
1267   mtev_skiplist *tlist;
1268   mtev_skiplist_node *next;
1269   noit_check_t *todo_onstack[8192];
1270   noit_check_t **todo = todo_onstack;
1271
1272   tlist = mtev_skiplist_find(polls_by_name.index,
1273                              __check_target_ip_compare, NULL);
1274
1275   pthread_mutex_lock(&polls_lock);
1276   /* First pass to count */
1277   memset(&pivot, 0, sizeof(pivot));
1278   strlcpy(pivot.target_ip, (char*)target_ip, sizeof(pivot.target_ip));
1279   pivot.name = "";
1280   pivot.target = "";
1281   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1282   while(next && next->data) {
1283     noit_check_t *check = next->data;
1284     if(strcmp(check->target_ip, target_ip)) break;
1285     todo_count++;
1286     mtev_skiplist_next(tlist, &next);
1287   }
1288
1289   if(todo_count > 8192) todo = malloc(todo_count * sizeof(*todo));
1290
1291   memset(&pivot, 0, sizeof(pivot));
1292   strlcpy(pivot.target_ip, (char*)target_ip, sizeof(pivot.target_ip));
1293   pivot.name = "";
1294   pivot.target = "";
1295   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1296   while(next && next->data) {
1297     noit_check_t *check = next->data;
1298     if(strcmp(check->target_ip, target_ip)) break;
1299     if(count < todo_count) todo[count++] = check;
1300     mtev_skiplist_next(tlist, &next);
1301   }
1302   pthread_mutex_unlock(&polls_lock);
1303
1304   todo_count = count;
1305   count = 0;
1306   for(i=0;i<todo_count;i++)
1307     count += f(todo[i],closure);
1308
1309   if(todo != todo_onstack) free(todo);
1310   return count;
1311 }
1312 int
1313 noit_poller_target_do(const char *target, int (*f)(noit_check_t *, void *),
1314                       void *closure) {
1315   int i, todo_count = 0, count = 0;
1316   noit_check_t pivot;
1317   mtev_skiplist *tlist;
1318   mtev_skiplist_node *next;
1319   noit_check_t *todo_onstack[8192];
1320   noit_check_t **todo = todo_onstack;
1321
1322   tlist = mtev_skiplist_find(polls_by_name.index,
1323                              __check_target_compare, NULL);
1324
1325   pthread_mutex_lock(&polls_lock);
1326   memset(&pivot, 0, sizeof(pivot));
1327   pivot.name = "";
1328   pivot.target = (char *)target;
1329   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1330   while(next && next->data) {
1331     noit_check_t *check = next->data;
1332     if(strcmp(check->target, target)) break;
1333     todo_count++;
1334     mtev_skiplist_next(tlist, &next);
1335   }
1336
1337   if(todo_count > 8192) todo = malloc(todo_count * sizeof(*todo));
1338
1339   memset(&pivot, 0, sizeof(pivot));
1340   pivot.name = "";
1341   pivot.target = (char *)target;
1342   mtev_skiplist_find_neighbors(tlist, &pivot, NULL, NULL, &next);
1343   while(next && next->data) {
1344     noit_check_t *check = next->data;
1345     if(strcmp(check->target, target)) break;
1346     if(count < todo_count) todo[count++] = check;
1347     mtev_skiplist_next(tlist, &next);
1348   }
1349   pthread_mutex_unlock(&polls_lock);
1350
1351   todo_count = count;
1352   count = 0;
1353   for(i=0;i<todo_count;i++)
1354     count += f(todo[i],closure);
1355
1356   if(todo != todo_onstack) free(todo);
1357   return count;
1358 }
1359
1360 int
1361 noit_poller_do(int (*f)(noit_check_t *, void *),
1362                void *closure) {
1363   mtev_skiplist_node *iter;
1364   int i, count = 0, max_count = 0;
1365   noit_check_t **todo;
1366
1367   if(polls_by_name.size == 0) return 0;
1368
1369   max_count = polls_by_name.size;
1370   todo = malloc(max_count * sizeof(*todo));
1371
1372   pthread_mutex_lock(&polls_lock);
1373   for(iter = mtev_skiplist_getlist(&polls_by_name); iter;
1374       mtev_skiplist_next(&polls_by_name, &iter)) {
1375     if(count < max_count) todo[count++] = (noit_check_t *)iter->data;
1376   }
1377   pthread_mutex_unlock(&polls_lock);
1378
1379   max_count = count;
1380   count = 0;
1381   for(i=0;i<max_count;i++)
1382     count += f(todo[i], closure);
1383   free(todo);
1384   return count;
1385 }
1386
1387 struct ip_module_collector_crutch {
1388   noit_check_t **array;
1389   const char *module;
1390   int idx;
1391   int allocd;
1392 };
1393 static int ip_module_collector(noit_check_t *check, void *cl) {
1394   struct ip_module_collector_crutch *c = cl;
1395   if(c->idx >= c->allocd) return 0;
1396   if(strcmp(check->module, c->module)) return 0;
1397   c->array[c->idx++] = check;
1398   return 1;
1399 }
1400 int
1401 noit_poller_lookup_by_ip_module(const char *ip, const char *mod,
1402                                 noit_check_t **checks, int nchecks) {
1403   struct ip_module_collector_crutch crutch;
1404   crutch.array = checks;
1405   crutch.allocd = nchecks;
1406   crutch.idx = 0;
1407   crutch.module = mod;
1408   return noit_poller_target_ip_do(ip, ip_module_collector, &crutch);
1409 }
1410 int
1411 noit_poller_lookup_by_module(const char *ip, const char *mod,
1412                              noit_check_t **checks, int nchecks) {
1413   struct ip_module_collector_crutch crutch;
1414   crutch.array = checks;
1415   crutch.allocd = nchecks;
1416   crutch.idx = 0;
1417   crutch.module = mod;
1418   return noit_poller_target_do(ip, ip_module_collector, &crutch);
1419 }
1420
1421
1422 int
1423 noit_check_xpath(char *xpath, int len,
1424                  const char *base, const char *arg) {
1425   uuid_t checkid;
1426   int base_trailing_slash;
1427   char argcopy[1024], *target, *module, *name;
1428
1429   base_trailing_slash = (base[strlen(base)-1] == '/');
1430   xpath[0] = '\0';
1431   argcopy[0] = '\0';
1432   if(arg) strlcpy(argcopy, arg, sizeof(argcopy));
1433
1434   if(uuid_parse(argcopy, checkid) == 0) {
1435     /* If they kill by uuid, we'll seek and destroy -- find it anywhere */
1436     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
1437              base, base_trailing_slash ? "" : "/", argcopy);
1438   }
1439   else if((module = strchr(argcopy, '`')) != NULL) {
1440     noit_check_t *check;
1441     char uuid_str[37];
1442     target = argcopy;
1443     *module++ = '\0';
1444     if((name = strchr(module+1, '`')) == NULL)
1445       name = module;
1446     else
1447       name++;
1448     check = noit_poller_lookup_by_name(target, name);
1449     if(!check) {
1450       return -1;
1451     }
1452     uuid_unparse_lower(check->checkid, uuid_str);
1453     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
1454              base, base_trailing_slash ? "" : "/", uuid_str);
1455   }
1456   return strlen(xpath);
1457 }
1458
1459 static int
1460 bad_check_initiate(noit_module_t *self, noit_check_t *check,
1461                    int once, noit_check_t *cause) {
1462   /* self is likely null here -- why it is bad, in fact */
1463   /* this is only suitable to call in one-offs */
1464   stats_t current;
1465   char buff[256];
1466   if(!once) return -1;
1467   if(!check) return -1;
1468   assert(!(check->flags & NP_RUNNING));
1469   check->flags |= NP_RUNNING;
1470   noit_check_stats_clear(check, &current);
1471   gettimeofday(&current.whence, NULL);
1472   current.duration = 0;
1473   current.available = NP_UNKNOWN;
1474   current.state = NP_UNKNOWN;
1475   snprintf(buff, sizeof(buff), "check[%s] implementation offline",
1476            check->module);
1477   current.status = buff;
1478   noit_check_set_stats(check, &current);
1479   check->flags &= ~NP_RUNNING;
1480   return 0;
1481 }
1482 void
1483 noit_check_stats_clear(noit_check_t *check, stats_t *s) {
1484   memset(s, 0, sizeof(*s));
1485   s->state = NP_UNKNOWN;
1486   s->available = NP_UNKNOWN;
1487 }
1488
1489 void
1490 __stats_add_metric(stats_t *newstate, metric_t *m) {
1491   mtev_hash_replace(&newstate->metrics, m->metric_name, strlen(m->metric_name),
1492                     m, NULL, (void (*)(void *))free_metric);
1493 }
1494
1495 static size_t
1496 noit_metric_sizes(metric_type_t type, const void *value) {
1497   switch(type) {
1498     case METRIC_INT32:
1499     case METRIC_UINT32:
1500       return sizeof(int32_t);
1501     case METRIC_INT64:
1502     case METRIC_UINT64:
1503       return sizeof(int64_t);
1504     case METRIC_DOUBLE:
1505       return sizeof(double);
1506     case METRIC_STRING: {
1507       int len = strlen((char*)value) + 1;
1508       return ((len >= text_size_limit) ? text_size_limit+1 : len);
1509     }
1510     case METRIC_GUESS:
1511       break;
1512   }
1513   assert(type != type);
1514   return 0;
1515 }
1516 static metric_type_t
1517 noit_metric_guess_type(const char *s, void **replacement) {
1518   char *copy, *cp, *trailer, *rpl;
1519   int negative = 0;
1520   metric_type_t type = METRIC_STRING;
1521
1522   if(!s) return METRIC_GUESS;
1523   copy = cp = strdup(s);
1524
1525   /* TRIM the string */
1526   while(*cp && isspace(*cp)) cp++; /* ltrim */
1527   s = cp; /* found a good starting point */
1528   while(*cp) cp++; /* advance to \0 */
1529   cp--; /* back up one */
1530   while(cp > s && isspace(*cp)) *cp-- = '\0'; /* rtrim */
1531
1532   /* Find the first space */
1533   cp = (char *)s;
1534   while(*cp && !isspace(*cp)) cp++;
1535   trailer = cp;
1536   cp--; /* backup one */
1537   if(cp > s && *cp == '%') *cp-- = '\0'; /* chop a last % is there is one */
1538
1539   while(*trailer && isspace(*trailer)) *trailer++ = '\0'; /* rtrim */
1540
1541   /* string was       '  -1.23e-01%  inodes used  ' */
1542   /* copy is (~ = \0) '  -1.23e-01~  inodes used~~' */
1543   /*                     ^           ^              */
1544   /*                     s           trailer        */
1545
1546   /* So, the trailer must not contain numbers */
1547   while(*trailer) { if(isdigit(*trailer)) goto notanumber; trailer++; }
1548
1549   /* And the 's' must be of the form:
1550    *  0) may start with a sign [-+]?
1551    *  1) [1-9][0-9]*
1552    *  2) [0]?.[0-9]+
1553    *  3) 0
1554    *  4) [1-9][0-9]*.[0-9]+
1555    *  5) all of the above ending with e[+-][0-9]+
1556    */
1557    rpl = (char *)s;
1558    /* CASE 0 */
1559    if(s[0] == '-' || s[0] == '+') {
1560      if(s[0] == '-') negative = 1;
1561      s++;
1562    }
1563
1564    if(s[0] == '.') goto decimal; /* CASE 2 */
1565    if(s[0] == '0') { /* CASE 2 & 3 */
1566      s++;
1567      if(!s[0]) goto scanint; /* CASE 3 */
1568      if(s[0] == '.') goto decimal; /* CASE 2 */
1569      goto notanumber;
1570    }
1571    if(s[0] >= '1' && s[0] <= '9') { /* CASE 1 & 4 */
1572      s++;
1573      while(isdigit(s[0])) s++; /* CASE 1 & 4 */
1574      if(!s[0]) goto scanint; /* CASE 1 */
1575      if(s[0] == '.') goto decimal; /* CASE 4 */
1576      goto notanumber;
1577    }
1578    /* Not case 1,2,3,4 */
1579    goto notanumber;
1580
1581   decimal:
1582    s++;
1583    if(!isdigit(s[0])) goto notanumber;
1584    s++;
1585    while(isdigit(s[0])) s++;
1586    if(!s[0]) goto scandouble;
1587    if(s[0] == 'e' || s[0] == 'E') goto exponent; /* CASE 5 */
1588    goto notanumber;
1589
1590   exponent:
1591    s++;
1592    if(s[0] != '-' && s[0] != '+') goto notanumber;
1593    s++;
1594    if(!isdigit(s[0])) goto notanumber;
1595    s++;
1596    while(isdigit(s[0])) s++;
1597    if(!s[0]) goto scandouble;
1598    goto notanumber;
1599
1600  scanint:
1601    if(negative) {
1602      int64_t *v;
1603      v = mtev_memory_safe_malloc(sizeof(*v));
1604      *v = strtoll(rpl, NULL, 10);
1605      *replacement = v;
1606      type = METRIC_INT64;
1607      goto alldone;
1608    }
1609    else {
1610      u_int64_t *v;
1611      v = mtev_memory_safe_malloc(sizeof(*v));
1612      *v = strtoull(rpl, NULL, 10);
1613      *replacement = v;
1614      type = METRIC_UINT64;
1615      goto alldone;
1616    }
1617  scandouble:
1618    {
1619      double *v;
1620      v = mtev_memory_safe_malloc(sizeof(*v));
1621      *v = strtod(rpl, NULL);
1622      *replacement = v;
1623      type = METRIC_DOUBLE;
1624      goto alldone;
1625    }
1626
1627  alldone:
1628  notanumber:
1629   free(copy);
1630   return type;
1631 }
1632
1633 static void
1634 cleanse_metric_name(char *m) {
1635   char *cp;
1636   for(cp = m; *cp; cp++)
1637     if(!isprint(*cp)) *cp=' ';
1638   for(cp--; *cp == ' ' && cp > m; cp--) /* always leave first char */
1639     *cp = '\0';
1640 }
1641
1642 int
1643 noit_stats_populate_metric(metric_t *m, const char *name, metric_type_t type,
1644                            const void *value) {
1645   void *replacement = NULL;
1646
1647   m->metric_name = mtev_memory_safe_strdup(name);
1648   cleanse_metric_name(m->metric_name);
1649
1650   if(type == METRIC_GUESS)
1651     type = noit_metric_guess_type((char *)value, &replacement);
1652   if(type == METRIC_GUESS) return -1;
1653
1654   m->metric_type = type;
1655
1656   if(replacement)
1657     m->metric_value.vp = replacement;
1658   else if(value) {
1659     size_t len;
1660     len = noit_metric_sizes(type, value);
1661     m->metric_value.vp = mtev_memory_safe_malloc(len);
1662     memcpy(m->metric_value.vp, value, len);
1663     if (type == METRIC_STRING) {
1664       m->metric_value.s[len-1] = 0;
1665     }
1666   }
1667   else m->metric_value.vp = NULL;
1668   return 0;
1669 }
1670
1671 metric_t *
1672 noit_stats_get_metric(noit_check_t *check,
1673                       stats_t *newstate, const char *name) {
1674   void *v;
1675   if(mtev_hash_retrieve(&newstate->metrics, name, strlen(name), &v))
1676     return (metric_t *)v;
1677   return NULL;
1678 }
1679
1680 void
1681 noit_stats_set_metric(noit_check_t *check,
1682                       stats_t *newstate, const char *name, metric_type_t type,
1683                       const void *value) {
1684   metric_t *m = mtev_memory_safe_calloc(1, sizeof(*m));
1685   if(noit_stats_populate_metric(m, name, type, value)) {
1686     free_metric(m);
1687     return;
1688   }
1689   noit_check_metric_count_add(1);
1690   check_stats_set_metric_hook_invoke(check, newstate, m);
1691   __stats_add_metric(newstate, m);
1692 }
1693 void
1694 noit_stats_set_metric_coerce(noit_check_t *check,
1695                              stats_t *stat, const char *name, metric_type_t t,
1696                              const char *v) {
1697   char *endptr;
1698   if(v == NULL) {
1699    bogus:
1700     check_stats_set_metric_coerce_hook_invoke(check, stat, name, t, v, mtev_false);
1701     noit_stats_set_metric(check, stat, name, t, NULL);
1702     return;
1703   }
1704   switch(t) {
1705     case METRIC_STRING:
1706       noit_stats_set_metric(check, stat, name, t, v);
1707       break;
1708     case METRIC_INT32:
1709     {
1710       int32_t val;
1711       val = strtol(v, &endptr, 10);
1712       if(endptr == v) goto bogus;
1713       noit_stats_set_metric(check, stat, name, t, &val);
1714       break;
1715     }
1716     case METRIC_UINT32:
1717     {
1718       u_int32_t val;
1719       val = strtoul(v, &endptr, 10);
1720       if(endptr == v) goto bogus;
1721       noit_stats_set_metric(check, stat, name, t, &val);
1722       break;
1723     }
1724     case METRIC_INT64:
1725     {
1726       int64_t val;
1727       val = strtoll(v, &endptr, 10);
1728       if(endptr == v) goto bogus;
1729       noit_stats_set_metric(check, stat, name, t, &val);
1730       break;
1731     }
1732     case METRIC_UINT64:
1733     {
1734       u_int64_t val;
1735       val = strtoull(v, &endptr, 10);
1736       if(endptr == v) goto bogus;
1737       noit_stats_set_metric(check, stat, name, t, &val);
1738       break;
1739     }
1740     case METRIC_DOUBLE:
1741     {
1742       double val;
1743       val = strtod(v, &endptr);
1744       if(endptr == v) goto bogus;
1745       noit_stats_set_metric(check, stat, name, t, &val);
1746       break;
1747     }
1748     case METRIC_GUESS:
1749       noit_stats_set_metric(check, stat, name, t, v);
1750       break;
1751   }
1752   check_stats_set_metric_coerce_hook_invoke(check, stat, name, t, v, mtev_true);
1753 }
1754 void
1755 noit_stats_log_immediate_metric(noit_check_t *check,
1756                                 const char *name, metric_type_t type,
1757                                 void *value) {
1758   struct timeval now;
1759   metric_t *m = mtev_memory_safe_malloc(sizeof(*m));
1760   if(noit_stats_populate_metric(m, name, type, value)) {
1761     free_metric(m);
1762     return;
1763   }
1764   gettimeofday(&now, NULL);
1765   noit_check_log_metric(check, &now, m);
1766   free_metric(m);
1767 }
1768
1769 void
1770 noit_check_passive_set_stats(noit_check_t *check, stats_t *newstate) {
1771   int i, nwatches = 0;
1772   mtev_skiplist_node *next;
1773   noit_check_t n;
1774   noit_check_t *watches[8192];
1775
1776   uuid_copy(n.checkid, check->checkid);
1777   n.period = 0;
1778
1779   noit_check_set_stats(check,newstate);
1780
1781   pthread_mutex_lock(&polls_lock);
1782   mtev_skiplist_find_neighbors(&watchlist, &n, NULL, NULL, &next);
1783   while(next && next->data && nwatches < 8192) {
1784     noit_check_t *wcheck = next->data;
1785     if(uuid_compare(n.checkid, wcheck->checkid)) break;
1786     watches[nwatches++] = wcheck;
1787     mtev_skiplist_next(&watchlist, &next);
1788   }
1789   pthread_mutex_unlock(&polls_lock);
1790
1791   for(i=0;i<nwatches;i++) {
1792     stats_t backup;
1793     noit_check_t *wcheck = watches[i];
1794     /* Swap the real check's stats into place */
1795     memcpy(&backup, &wcheck->stats.current, sizeof(stats_t));
1796     memcpy(&wcheck->stats.current, &check->stats.current, sizeof(stats_t));
1797
1798     if(check_passive_log_stats_hook_invoke(check) == MTEV_HOOK_CONTINUE) {
1799       /* Write out our status */
1800       noit_check_log_status(wcheck);
1801       /* Write out all metrics */
1802       noit_check_log_metrics(wcheck);
1803     }
1804     /* Swap them back out */
1805     memcpy(&wcheck->stats.current, &backup, sizeof(stats_t));
1806   }
1807 }
1808 void
1809 noit_check_set_stats(noit_check_t *check, stats_t *newstate) {
1810   int report_change = 0;
1811   char *cp;
1812   dep_list_t *dep;
1813   if(check->stats.previous.status)
1814     free(check->stats.previous.status);
1815   mtev_hash_destroy(&check->stats.previous.metrics, NULL,
1816                     (void (*)(void *))free_metric);
1817   memcpy(&check->stats.previous, &check->stats.current, sizeof(stats_t));
1818   if(newstate)
1819     memcpy(&check->stats.current, newstate, sizeof(stats_t));
1820   if(check->stats.current.status)
1821     check->stats.current.status = strdup(check->stats.current.status);
1822   for(cp = check->stats.current.status; cp && *cp; cp++)
1823     if(*cp == '\r' || *cp == '\n') *cp = ' ';
1824
1825   /* check for state changes */
1826   if(check->stats.current.available != NP_UNKNOWN &&
1827      check->stats.previous.available != NP_UNKNOWN &&
1828      check->stats.current.available != check->stats.previous.available)
1829     report_change = 1;
1830   if(check->stats.current.state != NP_UNKNOWN &&
1831      check->stats.previous.state != NP_UNKNOWN &&
1832      check->stats.current.state != check->stats.previous.state)
1833     report_change = 1;
1834
1835   mtevL(noit_debug, "%s`%s <- [%s]\n", check->target, check->name,
1836         check->stats.current.status);
1837   if(report_change) {
1838     mtevL(noit_debug, "%s`%s -> [%s:%s]\n",
1839           check->target, check->name,
1840           noit_check_available_string(check->stats.current.available),
1841           noit_check_state_string(check->stats.current.state));
1842   }
1843
1844   if(NOIT_CHECK_STATUS_ENABLED()) {
1845     char id[UUID_STR_LEN+1];
1846     uuid_unparse_lower(check->checkid, id);
1847     NOIT_CHECK_STATUS(id, check->module, check->name, check->target,
1848                       check->stats.current.available,
1849                       check->stats.current.state,
1850                       check->stats.current.status);
1851   }
1852
1853   if(check_log_stats_hook_invoke(check) == MTEV_HOOK_CONTINUE) {
1854     /* Write out the bundled information */
1855     noit_check_log_bundle(check);
1856   }
1857   /* count the check as complete */
1858   check_completion_count++;
1859
1860   for(dep = check->causal_checks; dep; dep = dep->next) {
1861     noit_module_t *mod;
1862     mod = noit_module_lookup(dep->check->module);
1863     if(!mod) {
1864       bad_check_initiate(mod, dep->check, 1, check);
1865     }
1866     else {
1867       mtevL(noit_debug, "Firing %s`%s in response to %s`%s\n",
1868             dep->check->target, dep->check->name,
1869             check->target, check->name);
1870       if((dep->check->flags & NP_DISABLED) == 0)
1871         if(mod->initiate_check)
1872           mod->initiate_check(mod, dep->check, 1, check);
1873     }
1874   }
1875 }
1876
1877 static int
1878 noit_console_show_watchlist(mtev_console_closure_t ncct,
1879                             int argc, char **argv,
1880                             mtev_console_state_t *dstate,
1881                             void *closure) {
1882   mtev_skiplist_node *iter, *fiter;
1883   int nwatches = 0, i;
1884   noit_check_t *watches[8192];
1885
1886   nc_printf(ncct, "%d active watches.\n", watchlist.size);
1887   pthread_mutex_lock(&polls_lock);
1888   for(iter = mtev_skiplist_getlist(&watchlist); iter && nwatches < 8192;
1889       mtev_skiplist_next(&watchlist, &iter)) {
1890     noit_check_t *check = iter->data;
1891     watches[nwatches++] = check;
1892   }
1893   pthread_mutex_unlock(&polls_lock);
1894
1895   for(i=0;i<nwatches;i++) {
1896     noit_check_t *check = watches[i];
1897     char uuid_str[UUID_STR_LEN + 1];
1898
1899     uuid_unparse_lower(check->checkid, uuid_str);
1900     nc_printf(ncct, "%s:\n\t[%s`%s`%s]\n\tPeriod: %dms\n\tFeeds[%d]:\n",
1901               uuid_str, check->target, check->module, check->name,
1902               check->period, check->feeds ? check->feeds->size : 0);
1903     if(check->feeds && check->feeds->size) {
1904       for(fiter = mtev_skiplist_getlist(check->feeds); fiter;
1905           mtev_skiplist_next(check->feeds, &fiter)) {
1906         nc_printf(ncct, "\t\t%s\n", (const char *)fiter->data);
1907       }
1908     }
1909   }
1910   return 0;
1911 }
1912
1913 static void
1914 nc_printf_check_brief(mtev_console_closure_t ncct,
1915                       noit_check_t *check) {
1916   char out[512];
1917   char uuid_str[37];
1918   snprintf(out, sizeof(out), "%s`%s (%s [%x])", check->target, check->name,
1919            check->target_ip, check->flags);
1920   uuid_unparse_lower(check->checkid, uuid_str);
1921   nc_printf(ncct, "%s %s\n", uuid_str, out);
1922   if(check->stats.current.status)
1923     nc_printf(ncct, "\t%s\n", check->stats.current.status);
1924 }
1925
1926 char *
1927 noit_console_conf_check_opts(mtev_console_closure_t ncct,
1928                              mtev_console_state_stack_t *stack,
1929                              mtev_console_state_t *dstate,
1930                              int argc, char **argv, int idx) {
1931   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1932   uuid_t key_id;
1933   int klen, i = 0;
1934   void *vcheck;
1935
1936   if(argc == 1) {
1937     if(!strncmp("new", argv[0], strlen(argv[0]))) {
1938       if(idx == i) return strdup("new");
1939       i++;
1940     }
1941     pthread_mutex_lock(&polls_lock);
1942     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
1943                          &vcheck)) {
1944       noit_check_t *check = (noit_check_t *)vcheck;
1945       char out[512];
1946       char uuid_str[37];
1947       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
1948       uuid_unparse_lower(check->checkid, uuid_str);
1949       if(!strncmp(out, argv[0], strlen(argv[0]))) {
1950         if(idx == i) {
1951           pthread_mutex_unlock(&polls_lock);
1952           return strdup(out);
1953         }
1954         i++;
1955       }
1956       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
1957         if(idx == i) {
1958           pthread_mutex_unlock(&polls_lock);
1959           return strdup(uuid_str);
1960         }
1961         i++;
1962       }
1963     }
1964     pthread_mutex_unlock(&polls_lock);
1965   }
1966   if(argc == 2) {
1967     cmd_info_t *cmd;
1968     if(!strcmp("new", argv[0])) return NULL;
1969     cmd = mtev_skiplist_find(&dstate->cmds, "attribute", NULL);
1970     if(!cmd) return NULL;
1971     return mtev_console_opt_delegate(ncct, stack, cmd->dstate, argc-1, argv+1, idx);
1972   }
1973   return NULL;
1974 }
1975
1976 char *
1977 noit_console_check_opts(mtev_console_closure_t ncct,
1978                         mtev_console_state_stack_t *stack,
1979                         mtev_console_state_t *dstate,
1980                         int argc, char **argv, int idx) {
1981   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
1982   uuid_t key_id;
1983   int klen, i = 0;
1984
1985   if(argc == 1) {
1986     void *vcheck;
1987     pthread_mutex_lock(&polls_lock);
1988     while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
1989                          &vcheck)) {
1990       char out[512];
1991       char uuid_str[37];
1992       noit_check_t *check = (noit_check_t *)vcheck;
1993       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
1994       uuid_unparse_lower(check->checkid, uuid_str);
1995       if(!strncmp(out, argv[0], strlen(argv[0]))) {
1996         if(idx == i) {
1997           pthread_mutex_unlock(&polls_lock);
1998           return strdup(out);
1999         }
2000         i++;
2001       }
2002       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
2003         if(idx == i) {
2004           pthread_mutex_unlock(&polls_lock);
2005           return strdup(uuid_str);
2006         }
2007         i++;
2008       }
2009     }
2010     pthread_mutex_unlock(&polls_lock);
2011   }
2012   if(argc == 2) {
2013     return mtev_console_opt_delegate(ncct, stack, dstate, argc-1, argv+1, idx);
2014   }
2015   return NULL;
2016 }
2017
2018 static int
2019 noit_console_show_checks(mtev_console_closure_t ncct,
2020                          int argc, char **argv,
2021                          mtev_console_state_t *dstate,
2022                          void *closure) {
2023   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
2024   uuid_t key_id;
2025   int klen, i = 0, nchecks;
2026   void *vcheck;
2027   noit_check_t **checks;
2028
2029   nchecks = mtev_hash_size(&polls);
2030   if(nchecks == 0) return 0;
2031   checks = malloc(nchecks * sizeof(*checks));
2032
2033   pthread_mutex_lock(&polls_lock);
2034   while(mtev_hash_next(&polls, &iter, (const char **)key_id, &klen,
2035                        &vcheck)) {
2036     if(i<nchecks) checks[i++] = vcheck;
2037   }
2038   pthread_mutex_unlock(&polls_lock);
2039
2040   nchecks = i;
2041   for(i=0;i<nchecks;i++)
2042     nc_printf_check_brief(ncct,checks[i]);
2043
2044   free(checks);
2045   return 0;
2046 }
2047
2048 static int
2049 noit_console_short_checks_sl(mtev_console_closure_t ncct,
2050                              mtev_skiplist *tlist) {
2051   int max_count, i = 0;
2052   noit_check_t **todo;
2053   mtev_skiplist_node *iter;
2054
2055   max_count = tlist->size;
2056   if(max_count == 0) return 0;
2057   todo = malloc(max_count * sizeof(*todo));
2058
2059   pthread_mutex_lock(&polls_lock);
2060   for(iter = mtev_skiplist_getlist(tlist); i < max_count && iter;
2061       mtev_skiplist_next(tlist, &iter)) {
2062     todo[i++] = iter->data;
2063   }
2064   pthread_mutex_unlock(&polls_lock);
2065
2066   max_count = i;
2067   for(i=0;i<max_count;i++)
2068     nc_printf_check_brief(ncct, todo[i]);
2069
2070   free(todo);
2071   return 0;
2072 }
2073 static int
2074 noit_console_show_checks_name(mtev_console_closure_t ncct,
2075                               int argc, char **argv,
2076                               mtev_console_state_t *dstate,
2077                               void *closure) {
2078   return noit_console_short_checks_sl(ncct, &polls_by_name);
2079 }
2080
2081 static int
2082 noit_console_show_checks_target(mtev_console_closure_t ncct,
2083                                    int argc, char **argv,
2084                                    mtev_console_state_t *dstate,
2085                                    void *closure) {
2086   return noit_console_short_checks_sl(ncct,
2087            mtev_skiplist_find(polls_by_name.index,
2088            __check_target_compare, NULL));
2089 }
2090
2091 static int
2092 noit_console_show_checks_target_ip(mtev_console_closure_t ncct,
2093                                    int argc, char **argv,
2094                                    mtev_console_state_t *dstate,
2095                                    void *closure) {
2096   return noit_console_short_checks_sl(ncct,
2097            mtev_skiplist_find(polls_by_name.index,
2098            __check_target_ip_compare, NULL));
2099 }
2100
2101 static void
2102 register_console_check_commands() {
2103   mtev_console_state_t *tl;
2104   cmd_info_t *showcmd;
2105
2106   tl = mtev_console_state_initial();
2107   showcmd = mtev_console_state_get_cmd(tl, "show");
2108   assert(showcmd && showcmd->dstate);
2109
2110   mtev_console_state_add_cmd(showcmd->dstate,
2111     NCSCMD("timing_slots", noit_console_show_timing_slots, NULL, NULL, NULL));
2112
2113   mtev_console_state_add_cmd(showcmd->dstate,
2114     NCSCMD("checks", noit_console_show_checks, NULL, NULL, NULL));
2115
2116   mtev_console_state_add_cmd(showcmd->dstate,
2117     NCSCMD("checks:name", noit_console_show_checks_name, NULL,
2118            NULL, NULL));
2119
2120   mtev_console_state_add_cmd(showcmd->dstate,
2121     NCSCMD("checks:target", noit_console_show_checks_target, NULL,
2122            NULL, NULL));
2123
2124   mtev_console_state_add_cmd(showcmd->dstate,
2125     NCSCMD("checks:target_ip", noit_console_show_checks_target_ip, NULL,
2126            NULL, NULL));
2127
2128   mtev_console_state_add_cmd(showcmd->dstate,
2129     NCSCMD("watches", noit_console_show_watchlist, NULL, NULL, NULL));
2130 }
2131
2132 int
2133 noit_check_register_module(const char *name) {
2134   int i;
2135   for(i=0; i<reg_module_id; i++)
2136     if(!strcmp(reg_module_names[i], name)) return i;
2137   if(reg_module_id >= MAX_MODULE_REGISTRATIONS) return -1;
2138   mtevL(noit_debug, "Registered module %s as %d\n", name, i);
2139   i = reg_module_id++;
2140   reg_module_names[i] = strdup(name);
2141   mtev_conf_set_namespace(reg_module_names[i]);
2142   return i;
2143 }
2144 int
2145 noit_check_registered_module_cnt() {
2146   return reg_module_id;
2147 }
2148 const char *
2149 noit_check_registered_module(int idx) {
2150   if(reg_module_used < 0) reg_module_used = reg_module_id;
2151   assert(reg_module_used == reg_module_id);
2152   if(idx >= reg_module_id || idx < 0) return NULL;
2153   return reg_module_names[idx];
2154 }
2155
2156 void
2157 noit_check_set_module_metadata(noit_check_t *c, int idx, void *md, void (*freefunc)(void *)) {
2158   struct vp_w_free *tuple;
2159   if(reg_module_used < 0) reg_module_used = reg_module_id;
2160   assert(reg_module_used == reg_module_id);
2161   if(idx >= reg_module_id || idx < 0) return;
2162   if(!c->module_metadata) c->module_metadata = calloc(reg_module_id, sizeof(void *));
2163   c->module_metadata[idx] = calloc(1, sizeof(struct vp_w_free));
2164   tuple = c->module_metadata[idx];
2165   tuple->ptr = md;
2166   tuple->freefunc = freefunc;
2167 }
2168 void
2169 noit_check_set_module_config(noit_check_t *c, int idx, mtev_hash_table *config) {
2170   if(reg_module_used < 0) reg_module_used = reg_module_id;
2171   assert(reg_module_used == reg_module_id);
2172   if(idx >= reg_module_id || idx < 0) return;
2173   if(!c->module_configs) c->module_configs = calloc(reg_module_id, sizeof(mtev_hash_table *));
2174   c->module_configs[idx] = config;
2175 }
2176 void *
2177 noit_check_get_module_metadata(noit_check_t *c, int idx) {
2178   struct vp_w_free *tuple;
2179   if(reg_module_used < 0) reg_module_used = reg_module_id;
2180   assert(reg_module_used == reg_module_id);
2181   if(idx >= reg_module_id || idx < 0 || !c->module_metadata) return NULL;
2182   tuple = c->module_metadata[idx];
2183   return tuple ? tuple->ptr : NULL;
2184 }
2185 mtev_hash_table *
2186 noit_check_get_module_config(noit_check_t *c, int idx) {
2187   if(reg_module_used < 0) reg_module_used = reg_module_id;
2188   assert(reg_module_used == reg_module_id);
2189   if(idx >= reg_module_id || idx < 0 || !c->module_configs) return NULL;
2190   return c->module_configs[idx];
2191 }
Note: See TracBrowser for help on using the browser.