root/src/noit_check.c

Revision 832c60c9213032868ca5cf0d28b130c84b19c340, 39.4 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 3 years ago)

merge in master

  • Property mode set to 100644
Line 
1 /*
2  * Copyright (c) 2007, OmniTI Computer Consulting, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are
7  * met:
8  *
9  *     * Redistributions of source code must retain the above copyright
10  *       notice, this list of conditions and the following disclaimer.
11  *     * Redistributions in binary form must reproduce the above
12  *       copyright notice, this list of conditions and the following
13  *       disclaimer in the documentation and/or other materials provided
14  *       with the distribution.
15  *     * Neither the name OmniTI Computer Consulting, Inc. nor the names
16  *       of its contributors may be used to endorse or promote products
17  *       derived from this software without specific prior written
18  *       permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #include "noit_defines.h"
34
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <unistd.h>
38 #include <ctype.h>
39 #include <assert.h>
40 #include <errno.h>
41 #include <netinet/in.h>
42 #include <arpa/inet.h>
43 #include <time.h>
44
45 #include "dtrace_probes.h"
46 #include "utils/noit_log.h"
47 #include "utils/noit_hash.h"
48 #include "utils/noit_skiplist.h"
49 #include "noit_conf.h"
50 #include "noit_check.h"
51 #include "noit_module.h"
52 #include "noit_console.h"
53 #include "noit_check_tools.h"
54 #include "noit_check_resolver.h"
55 #include "eventer/eventer.h"
56
57 /* 60 seconds of possible stutter */
58 #define MAX_INITIAL_STUTTER 60000
59
60 static u_int64_t check_completion_count = 0;
61 static noit_hash_table polls = NOIT_HASH_EMPTY;
62 static noit_skiplist watchlist = { 0 };
63 static noit_skiplist polls_by_name = { 0 };
64 static u_int32_t __config_load_generation = 0;
65
66 u_int64_t noit_check_completion_count() {
67   return check_completion_count;
68 }
69 static void register_console_check_commands();
70 static int check_recycle_bin_processor(eventer_t, int, void *,
71                                        struct timeval *);
72
73 const char *
74 noit_check_available_string(int16_t available) {
75   switch(available) {
76     case NP_AVAILABLE:    return "available";
77     case NP_UNAVAILABLE:  return "unavailable";
78     case NP_UNKNOWN:      return "unknown";
79   }
80   return NULL;
81 }
82 const char *
83 noit_check_state_string(int16_t state) {
84   switch(state) {
85     case NP_GOOD:         return "good";
86     case NP_BAD:          return "bad";
87     case NP_UNKNOWN:      return "unknown";
88   }
89   return NULL;
90 }
91 static int __check_name_compare(const void *a, const void *b) {
92   const noit_check_t *ac = a;
93   const noit_check_t *bc = b;
94   int rv;
95   if((rv = strcmp(ac->target, bc->target)) != 0) return rv;
96   if((rv = strcmp(ac->name, bc->name)) != 0) return rv;
97   return 0;
98 }
99 static int __watchlist_compare(const void *a, const void *b) {
100   const noit_check_t *ac = a;
101   const noit_check_t *bc = b;
102   int rv;
103   if((rv = memcmp(ac->checkid, bc->checkid, sizeof(ac->checkid))) != 0) return rv;
104   if(ac->period < bc->period) return -1;
105   if(ac->period == bc->period) return 0;
106   return 1;
107 }
108 int
109 noit_check_max_initial_stutter() {
110   int stutter;
111   if(!noit_conf_get_int(NULL, "/noit/checks/@max_initial_stutter", &stutter))
112     stutter = MAX_INITIAL_STUTTER;
113   return stutter;
114 }
115 void
116 noit_check_fake_last_check(noit_check_t *check,
117                            struct timeval *lc, struct timeval *_now) {
118   struct timeval now, period;
119   static int start_offset_ms = -1;
120   int offset = 0, max;
121
122   if(start_offset_ms == -1)
123     start_offset_ms = drand48() * noit_check_max_initial_stutter();
124   if(!(check->flags & NP_TRANSIENT)) {
125     max = noit_check_max_initial_stutter();
126     offset = start_offset_ms + drand48() * 1000;
127     offset = offset % MIN(max, check->period);
128     start_offset_ms += 1000;
129   }
130   period.tv_sec = (check->period - offset) / 1000;
131   period.tv_usec = ((check->period - offset) % 1000) * 1000;
132   if(!_now) {
133     gettimeofday(&now, NULL);
134     _now = &now;
135   }
136   sub_timeval(*_now, period, lc);
137 }
138 void
139 noit_poller_process_checks(const char *xpath) {
140   int i, flags, cnt = 0;
141   noit_conf_section_t *sec;
142   __config_load_generation++;
143   sec = noit_conf_get_sections(NULL, xpath, &cnt);
144   for(i=0; i<cnt; i++) {
145     void *vcheck;
146     char uuid_str[37];
147     char target[256] = "";
148     char module[256] = "";
149     char name[256] = "";
150     char filterset[256] = "";
151     char oncheck[1024] = "";
152     int no_period = 0;
153     int no_oncheck = 0;
154     int period = 0, timeout = 0;
155     noit_boolean disabled = noit_false, busted = noit_false;
156     uuid_t uuid, out_uuid;
157     noit_hash_table *options;
158
159 #define NEXT(...) noitL(noit_stderr, __VA_ARGS__); continue
160 #define MYATTR(type,a,...) noit_conf_get_##type(sec[i], "@" #a, __VA_ARGS__)
161 #define INHERIT(type,a,...) \
162   noit_conf_get_##type(sec[i], "ancestor-or-self::node()/@" #a, __VA_ARGS__)
163
164     if(!MYATTR(stringbuf, uuid, uuid_str, sizeof(uuid_str))) {
165       noitL(noit_stderr, "check %d has no uuid\n", i+1);
166       continue;
167     }
168
169     if(uuid_parse(uuid_str, uuid)) {
170       noitL(noit_stderr, "check uuid: '%s' is invalid\n", uuid_str);
171       continue;
172     }
173
174     if(!INHERIT(stringbuf, target, target, sizeof(target))) {
175       noitL(noit_stderr, "check uuid: '%s' has no target\n", uuid_str);
176       busted = noit_true;
177     }
178     if(!INHERIT(stringbuf, module, module, sizeof(module))) {
179       noitL(noit_stderr, "check uuid: '%s' has no module\n", uuid_str);
180       busted = noit_true;
181     }
182
183     if(!INHERIT(stringbuf, filterset, filterset, sizeof(filterset)))
184       filterset[0] = '\0';
185
186     if(!MYATTR(stringbuf, name, name, sizeof(name)))
187       strlcpy(name, module, sizeof(name));
188
189     if(!INHERIT(int, period, &period) || period == 0)
190       no_period = 1;
191
192     if(!INHERIT(stringbuf, oncheck, oncheck, sizeof(oncheck)) || !oncheck[0])
193       no_oncheck = 1;
194
195     if(no_period && no_oncheck) {
196       noitL(noit_stderr, "check uuid: '%s' has neither period nor oncheck\n",
197             uuid_str);
198       busted = noit_true;
199     }
200     if(!(no_period || no_oncheck)) {
201       noitL(noit_stderr, "check uuid: '%s' has oncheck and period.\n",
202             uuid_str);
203       busted = noit_true;
204     }
205     if(!INHERIT(int, timeout, &timeout)) {
206       noitL(noit_stderr, "check uuid: '%s' has no timeout\n", uuid_str);
207       busted = noit_true;
208     }
209     if(!no_period && timeout >= period) {
210       noitL(noit_stderr, "check uuid: '%s' timeout > period\n", uuid_str);
211       timeout = period/2;
212     }
213     options = noit_conf_get_hash(sec[i], "config");
214
215     INHERIT(boolean, disable, &disabled);
216     flags = 0;
217     if(busted) flags |= (NP_UNCONFIG|NP_DISABLED);
218     else if(disabled) flags |= NP_DISABLED;
219
220     if(noit_hash_retrieve(&polls, (char *)uuid, UUID_SIZE,
221                           &vcheck)) {
222       noit_check_t *existing_check = (noit_check_t *)vcheck;
223       /* Once set, it cannot be checked if the check is live */
224       assert(!existing_check->module || !existing_check->module[0] ||
225              !strcmp(existing_check->module, module) ||
226              !NOIT_CHECK_LIVE(existing_check));
227       /* Set it if it is unset or being changed */
228       if(!existing_check->module || !existing_check->module[0] ||
229          strcmp(existing_check->module, module)) {
230         if(existing_check->module) free(existing_check->module);
231         existing_check->module = strdup(module);
232       }
233       noit_check_update(existing_check, target, name, filterset, options,
234                            period, timeout, oncheck[0] ? oncheck : NULL,
235                            flags);
236       noitL(noit_debug, "reloaded uuid: %s\n", uuid_str);
237     }
238     else {
239       noit_poller_schedule(target, module, name, filterset, options,
240                            period, timeout, oncheck[0] ? oncheck : NULL,
241                            flags, uuid, out_uuid);
242       noitL(noit_debug, "loaded uuid: %s\n", uuid_str);
243     }
244
245     noit_hash_destroy(options, free, free);
246     free(options);
247   }
248   if(sec) free(sec);
249 }
250
251 int
252 noit_check_activate(noit_check_t *check) {
253   noit_module_t *mod;
254   if(NOIT_CHECK_LIVE(check)) return 0;
255   mod = noit_module_lookup(check->module);
256   if(mod && mod->initiate_check) {
257     if((check->flags & NP_DISABLED) == 0) {
258       mod->initiate_check(mod, check, 0, NULL);
259       return 1;
260     }
261     else
262       noitL(noit_debug, "Skipping %s`%s, disabled.\n",
263             check->target, check->name);
264   }
265   else {
266     if(!mod) {
267       noitL(noit_stderr, "Cannot find module '%s'\n", check->module);
268       check->flags |= NP_DISABLED;
269     }
270   }
271   return 0;
272 }
273
274 void
275 noit_poller_initiate() {
276   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
277   uuid_t key_id;
278   int klen;
279   void *vcheck;
280   while(noit_hash_next(&polls, &iter, (const char **)key_id, &klen,
281                        &vcheck)) {
282     noit_check_activate((noit_check_t *)vcheck);
283   }
284 }
285
286 void
287 noit_poller_flush_epoch(int oldest_allowed) {
288   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
289   uuid_t key_id;
290   int klen;
291   noit_check_t *tofree = NULL;
292   void *vcheck;
293
294   /* Cleanup any previous causal map */
295   while(noit_hash_next(&polls, &iter, (const char **)key_id, &klen,
296                        &vcheck)) {
297     noit_check_t *check = (noit_check_t *)vcheck;
298     /* We don't free the one we're looking at... we free it on the next
299      * pass.  This leaves out iterator in good shape.  We just need to
300      * remember to free it one last time outside the while loop, down...
301      */
302     if(tofree) {
303       noit_poller_deschedule(tofree->checkid);
304       tofree = NULL;
305     }
306     if(check->generation < oldest_allowed) {
307       tofree = check;
308     }
309   }
310   /* ... here */
311   if(tofree) noit_poller_deschedule(tofree->checkid);
312 }
313
314 void
315 noit_poller_make_causal_map() {
316   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
317   uuid_t key_id;
318   int klen;
319   void *vcheck;
320
321   /* Cleanup any previous causal map */
322   while(noit_hash_next(&polls, &iter, (const char **)key_id, &klen,
323                        &vcheck)) {
324     noit_check_t *check = (noit_check_t *)vcheck;
325     dep_list_t *dep;
326     while((dep = check->causal_checks) != NULL) {
327       check->causal_checks = dep->next;
328       free(dep);
329     }
330   }
331
332   memset(&iter, 0, sizeof(iter));
333   /* Walk all checks and add check dependencies to their parents */
334   while(noit_hash_next(&polls, &iter, (const char **)key_id, &klen,
335                        &vcheck)) {
336     noit_check_t *check = (noit_check_t *)vcheck, *parent;
337     if(check->oncheck) {
338       /* This service is causally triggered by another service */
339       char fullcheck[1024];
340       char *name = check->oncheck;
341       char *target = NULL;
342
343       noitL(noit_debug, "Searching for upstream trigger on %s\n", name);
344       if((target = strchr(check->oncheck, '`')) != NULL) {
345         strlcpy(fullcheck, check->oncheck, target - check->oncheck);
346         name = target + 1;
347         target = fullcheck;
348       }
349       else
350        target = check->target;
351
352       parent = noit_poller_lookup_by_name(target, name);
353       if(!parent) {
354         check->flags |= NP_DISABLED;
355         noitL(noit_stderr, "Disabling check %s`%s, can't find oncheck %s`%s\n",
356               check->target, check->name, target, name);
357       }
358       else {
359         dep_list_t *dep;
360         dep = malloc(sizeof(*dep));
361         dep->check = check;
362         dep->next = parent->causal_checks;
363         parent->causal_checks = dep;
364         noitL(noit_debug, "Causal map %s`%s --> %s`%s\n",
365               parent->target, parent->name, check->target, check->name);
366       }
367     }
368   }
369 }
370 void
371 noit_poller_reload(const char *xpath)
372 {
373   noit_poller_process_checks(xpath ? xpath : "/noit/checks//check");
374   if(!xpath) {
375     /* Full reload, we need to wipe old checks */
376     noit_poller_flush_epoch(__config_load_generation);
377   }
378   noit_poller_make_causal_map();
379   noit_poller_initiate();
380 }
381 void
382 noit_poller_init() {
383   srand48((getpid() << 16) & time(NULL));
384   noit_check_resolver_init();
385   noit_check_tools_init();
386   noit_skiplist_init(&polls_by_name);
387   noit_skiplist_set_compare(&polls_by_name, __check_name_compare,
388                             __check_name_compare);
389   noit_skiplist_init(&watchlist);
390   noit_skiplist_set_compare(&watchlist, __watchlist_compare,
391                             __watchlist_compare);
392   register_console_check_commands();
393   eventer_name_callback("check_recycle_bin_processor",
394                         check_recycle_bin_processor);
395   eventer_add_in_s_us(check_recycle_bin_processor, NULL, 60, 0);
396   noit_poller_reload(NULL);
397 }
398
399 int
400 noit_poller_check_count() {
401   return polls_by_name.size;
402 }
403
404 int
405 noit_poller_transient_check_count() {
406   return watchlist.size;
407 }
408
409 noit_check_t *
410 noit_check_clone(uuid_t in) {
411   noit_check_t *checker, *new_check;
412   void *vcheck;
413   if(noit_hash_retrieve(&polls,
414                         (char *)in, UUID_SIZE,
415                         &vcheck) == 0) {
416     return NULL;
417   }
418   checker = (noit_check_t *)vcheck;
419   if(checker->oncheck) {
420     return NULL;
421   }
422   new_check = calloc(1, sizeof(*new_check));
423   memcpy(new_check, checker, sizeof(*new_check));
424   new_check->target = strdup(new_check->target);
425   new_check->module = strdup(new_check->module);
426   new_check->name = strdup(new_check->name);
427   new_check->filterset = strdup(new_check->filterset);
428   new_check->flags = 0;
429   new_check->fire_event = NULL;
430   memset(&new_check->last_fire_time, 0, sizeof(new_check->last_fire_time));
431   memset(&new_check->stats, 0, sizeof(new_check->stats));
432   new_check->closure = NULL;
433   new_check->config = calloc(1, sizeof(*new_check->config));
434   noit_hash_merge_as_dict(new_check->config, checker->config);
435   return new_check;
436 }
437
438 noit_check_t *
439 noit_check_watch(uuid_t in, int period) {
440   /* First look for a copy that is being watched */
441   int minimum_pi = 1000, granularity_pi = 500;
442   noit_conf_section_t check_node;
443   char uuid_str[UUID_STR_LEN + 1];
444   char xpath[1024];
445   noit_check_t n, *f;
446
447   uuid_unparse_lower(in, uuid_str);
448   /* Find the check */
449   snprintf(xpath, sizeof(xpath), "//checks//check[@uuid=\"%s\"]", uuid_str);
450   check_node = noit_conf_get_section(NULL, xpath);
451   noit_conf_get_int(NULL, "//checks/@transient_min_period", &minimum_pi);
452   noit_conf_get_int(NULL, "//checks/@transient_period_granularity", &granularity_pi);
453   if(check_node) {
454     noit_conf_get_int(check_node,
455                       "ancestor-or-self::node()/@transient_min_period",
456                       &minimum_pi);
457     noit_conf_get_int(check_node,
458                       "ancestor-or-self::node()/@transient_period_granularity",
459                       &granularity_pi);
460   }
461
462   /* apply the bounds */
463   period /= granularity_pi;
464   period *= granularity_pi;
465   period = MAX(period, minimum_pi);
466
467   uuid_copy(n.checkid, in);
468   n.period = period;
469
470   f = noit_skiplist_find(&watchlist, &n, NULL);
471   if(f) return f;
472   f = noit_check_clone(in);
473   if(!f) return NULL;
474   f->period = period;
475   f->timeout = period - 10;
476   f->flags |= NP_TRANSIENT;
477   noitL(noit_debug, "Watching %s@%d\n", uuid_str, period);
478   noit_skiplist_insert(&watchlist, f);
479   return f;
480 }
481
482 noit_check_t *
483 noit_check_get_watch(uuid_t in, int period) {
484   noit_check_t n, *f;
485
486   uuid_copy(n.checkid, in);
487   n.period = period;
488
489   f = noit_skiplist_find(&watchlist, &n, NULL);
490   return f;
491 }
492
493 void
494 noit_check_transient_add_feed(noit_check_t *check, const char *feed) {
495   char *feedcopy;
496   if(!check->feeds) {
497     check->feeds = calloc(1, sizeof(*check->feeds));
498     noit_skiplist_init(check->feeds);
499     noit_skiplist_set_compare(check->feeds,
500                               (noit_skiplist_comparator_t)strcmp,
501                               (noit_skiplist_comparator_t)strcmp);
502   }
503   feedcopy = strdup(feed);
504   /* No error on failure -- it's already there */
505   if(noit_skiplist_insert(check->feeds, feedcopy) == NULL) free(feedcopy);
506   noitL(noit_debug, "check %s`%s @ %dms has %d feed(s): %s.\n",
507         check->target, check->name, check->period, check->feeds->size, feed);
508 }
509 void
510 noit_check_transient_remove_feed(noit_check_t *check, const char *feed) {
511   if(!check->feeds) return;
512   if(feed) {
513     noitL(noit_debug, "check %s`%s @ %dms removing 1 of %d feeds: %s.\n",
514           check->target, check->name, check->period, check->feeds->size, feed);
515     noit_skiplist_remove(check->feeds, feed, free);
516   }
517   if(check->feeds->size == 0) {
518     char uuid_str[UUID_STR_LEN + 1];
519     uuid_unparse_lower(check->checkid, uuid_str);
520     noitL(noit_debug, "Unwatching %s@%d\n", uuid_str, check->period);
521     noit_skiplist_remove(&watchlist, check, NULL);
522     noit_skiplist_destroy(check->feeds, free);
523     free(check->feeds);
524     check->feeds = NULL;
525     if(check->flags & NP_TRANSIENT) {
526       noitL(noit_debug, "check %s`%s @ %dms has no more listeners.\n",
527             check->target, check->name, check->period);
528       check->flags |= NP_KILLED;
529     }
530   }
531 }
532
533 noit_boolean
534 noit_check_is_valid_target(const char *target) {
535   int8_t family;
536   int rv;
537   union {
538     struct in_addr addr4;
539     struct in6_addr addr6;
540   } a;
541
542   family = AF_INET;
543   rv = inet_pton(family, target, &a);
544   if(rv != 1) {
545     family = AF_INET6;
546     rv = inet_pton(family, target, &a);
547     if(rv != 1) {
548       return noit_false;
549     }
550   }
551   return noit_true;
552 }
553 int
554 noit_check_set_ip(noit_check_t *new_check,
555                   const char *ip_str) {
556   int8_t family;
557   int rv, failed = 0;
558   union {
559     struct in_addr addr4;
560     struct in6_addr addr6;
561   } a;
562
563
564   family = AF_INET;
565   rv = inet_pton(family, ip_str, &a);
566   if(rv != 1) {
567     family = AF_INET6;
568     rv = inet_pton(family, ip_str, &a);
569     if(rv != 1) {
570       family = AF_INET;
571       memset(&a, 0, sizeof(a));
572       failed = -1;
573     }
574   }
575
576   new_check->target_family = family;
577   memcpy(&new_check->target_addr, &a, sizeof(a));
578   new_check->target_ip[0] = '\0';
579   if(failed == 0)
580     if(inet_ntop(new_check->target_family,
581                  &new_check->target_addr,
582                  new_check->target_ip,
583                  sizeof(new_check->target_ip)) == NULL) {
584       noitL(noit_error, "inet_ntop failed [%s] -> %d\n", ip_str, errno);
585     }
586   return failed;
587 }
588 int
589 noit_check_resolve(noit_check_t *check) {
590   uint8_t family_pref = AF_INET;
591   char ipaddr[INET6_ADDRSTRLEN];
592   if(!NOIT_CHECK_SHOULD_RESOLVE(check)) return 1; /* success, not required */
593   noit_check_resolver_remind(check->target);
594   if(noit_check_resolver_fetch(check->target, ipaddr, sizeof(ipaddr),
595                                family_pref) >= 0) {
596     check->flags |= NP_RESOLVED;
597     noit_check_set_ip(check, ipaddr);
598     return 0;
599   }
600   check->flags &= ~NP_RESOLVED;
601   return -1;
602 }
603 int
604 noit_check_update(noit_check_t *new_check,
605                   const char *target,
606                   const char *name,
607                   const char *filterset,
608                   noit_hash_table *config,
609                   u_int32_t period,
610                   u_int32_t timeout,
611                   const char *oncheck,
612                   int flags) {
613   int mask = NP_DISABLED | NP_UNCONFIG;
614
615   new_check->generation = __config_load_generation;
616   if(new_check->target) free(new_check->target);
617   new_check->target = strdup(target);
618
619   if(noit_check_set_ip(new_check, target)) {
620     noit_boolean should_resolve;
621     new_check->flags |= NP_RESOLVE;
622     new_check->flags &= ~NP_RESOLVED;
623     if(noit_conf_get_boolean(NULL, "//checks/@resolve_targets",
624                              &should_resolve) && should_resolve == noit_false)
625      
626       flags |= NP_DISABLED | NP_UNCONFIG;
627     noit_check_resolve(new_check);
628   }
629
630   if(new_check->name) free(new_check->name);
631   new_check->name = name ? strdup(name): NULL;
632   if(new_check->filterset) free(new_check->filterset);
633   new_check->filterset = filterset ? strdup(filterset): NULL;
634
635   if(config != NULL) {
636     noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
637     const char *k;
638     int klen;
639     void *data;
640     if(new_check->config) noit_hash_delete_all(new_check->config, free, free);
641     else new_check->config = calloc(1, sizeof(*new_check->config));
642     while(noit_hash_next(config, &iter, &k, &klen, &data)) {
643       noit_hash_store(new_check->config, strdup(k), klen, strdup((char *)data));
644     }
645   }
646   if(new_check->oncheck) free(new_check->oncheck);
647   new_check->oncheck = oncheck ? strdup(oncheck) : NULL;
648   new_check->period = period;
649   new_check->timeout = timeout;
650
651   /* Unset what could be set.. then set what should be set */
652   new_check->flags = (new_check->flags & ~mask) | flags;
653
654   if(!(new_check->flags & NP_TRANSIENT)) {
655     /* This remove could fail -- no big deal */
656     noit_skiplist_remove(&polls_by_name, new_check, NULL);
657
658     /* This insert could fail.. which means we have a conflict on
659      * target`name.  That should result in the check being disabled. */
660     if(!noit_skiplist_insert(&polls_by_name, new_check)) {
661       noitL(noit_stderr, "Check %s`%s disabled due to naming conflict\n",
662             new_check->target, new_check->name);
663       new_check->flags |= NP_DISABLED;
664     }
665   }
666   noit_check_log_check(new_check);
667   return 0;
668 }
669 int
670 noit_poller_schedule(const char *target,
671                      const char *module,
672                      const char *name,
673                      const char *filterset,
674                      noit_hash_table *config,
675                      u_int32_t period,
676                      u_int32_t timeout,
677                      const char *oncheck,
678                      int flags,
679                      uuid_t in,
680                      uuid_t out) {
681   noit_check_t *new_check;
682   new_check = calloc(1, sizeof(*new_check));
683   if(!new_check) return -1;
684
685   /* The module and the UUID can never be changed */
686   new_check->module = strdup(module);
687   if(uuid_is_null(in))
688     uuid_generate(new_check->checkid);
689   else
690     uuid_copy(new_check->checkid, in);
691
692   noit_check_update(new_check, target, name, filterset, config,
693                     period, timeout, oncheck, flags);
694   assert(noit_hash_store(&polls,
695                          (char *)new_check->checkid, UUID_SIZE,
696                          new_check));
697   uuid_copy(out, new_check->checkid);
698
699   return 0;
700 }
701
702 /* A quick little list of recycleable checks.  This list never really
703  * grows large, so no sense in thinking too hard about the algorithmic
704  * complexity.
705  */
706 struct _checker_rcb {
707   noit_check_t *checker;
708   struct _checker_rcb *next;
709 };
710 static struct _checker_rcb *checker_rcb = NULL;
711 static void recycle_check(noit_check_t *checker) {
712   struct _checker_rcb *n = malloc(sizeof(*n));
713   n->checker = checker;
714   n->next = checker_rcb;
715   checker_rcb = n;
716 }
717 void
718 noit_poller_free_check(noit_check_t *checker) {
719   noit_module_t *mod;
720
721   if(checker->flags & NP_RUNNING) {
722     recycle_check(checker);
723     return;
724   }
725
726   mod = noit_module_lookup(checker->module);
727   if(mod && mod->cleanup) mod->cleanup(mod, checker);
728   if(checker->fire_event) {
729      eventer_remove(checker->fire_event);
730      free(checker->fire_event->closure);
731      eventer_free(checker->fire_event);
732      checker->fire_event = NULL;
733   }
734   if(checker->closure) free(checker->closure);
735   if(checker->target) free(checker->target);
736   if(checker->module) free(checker->module);
737   if(checker->name) free(checker->name);
738   if(checker->config) {
739     noit_hash_destroy(checker->config, free, free);
740     free(checker->config);
741     checker->config = NULL;
742   }
743   free(checker);
744 }
745 static int
746 check_recycle_bin_processor(eventer_t e, int mask, void *closure,
747                             struct timeval *now) {
748   static struct timeval one_minute = { 60L, 0L };
749   struct _checker_rcb *prev = NULL, *curr = checker_rcb;
750   noitL(noit_debug, "Scanning check recycle bin\n");
751   while(curr) {
752     if(!(curr->checker->flags & NP_RUNNING)) {
753       noitL(noit_debug, "Check is ready to free.\n");
754       noit_poller_free_check(curr->checker);
755       if(prev) prev->next = curr->next;
756       else checker_rcb = curr->next;
757       free(curr);
758       curr = prev ? prev->next : checker_rcb;
759     }
760     else {
761       prev = curr;
762       curr = curr->next;
763     }
764   }
765   add_timeval(*now, one_minute, &e->whence);
766   return EVENTER_TIMER;
767 }
768
769 int
770 noit_poller_deschedule(uuid_t in) {
771   void *vcheck;
772   noit_check_t *checker;
773   if(noit_hash_retrieve(&polls,
774                         (char *)in, UUID_SIZE,
775                         &vcheck) == 0) {
776     return -1;
777   }
778   checker = (noit_check_t *)vcheck;
779   checker->flags |= (NP_DISABLED|NP_KILLED);
780
781   noit_check_log_delete(checker);
782
783   noit_skiplist_remove(&polls_by_name, checker, NULL);
784   noit_hash_delete(&polls, (char *)in, UUID_SIZE, NULL, NULL);
785
786   noit_poller_free_check(checker);
787   return 0;
788 }
789
790 noit_check_t *
791 noit_poller_lookup(uuid_t in) {
792   void *vcheck;
793   if(noit_hash_retrieve(&polls, (char *)in, UUID_SIZE, &vcheck))
794     return (noit_check_t *)vcheck;
795   return NULL;
796 }
797 noit_check_t *
798 noit_poller_lookup_by_name(char *target, char *name) {
799   noit_check_t *check, *tmp_check;
800   tmp_check = calloc(1, sizeof(*tmp_check));
801   tmp_check->target = target;
802   tmp_check->name = name;
803   check = noit_skiplist_find(&polls_by_name, tmp_check, NULL);
804   free(tmp_check);
805   return check;
806 }
807 int
808 noit_poller_target_do(char *target, int (*f)(noit_check_t *, void *),
809                       void *closure) {
810   int count = 0;
811   noit_check_t pivot;
812   noit_skiplist_node *next;
813
814   memset(&pivot, 0, sizeof(pivot));
815   pivot.target = target;
816   pivot.name = "";
817   noit_skiplist_find_neighbors(&polls_by_name, &pivot, NULL, NULL, &next);
818   while(next && next->data) {
819     noit_check_t *check = next->data;
820     if(strcmp(check->target, target)) break;
821     count += f(check,closure);
822     noit_skiplist_next(&polls_by_name, &next);
823   }
824   return count;
825 }
826
827 int
828 noit_check_xpath(char *xpath, int len,
829                  const char *base, const char *arg) {
830   uuid_t checkid;
831   int base_trailing_slash;
832   char argcopy[1024], *target, *module, *name;
833
834   base_trailing_slash = (base[strlen(base)-1] == '/');
835   xpath[0] = '\0';
836   argcopy[0] = '\0';
837   if(arg) strlcpy(argcopy, arg, sizeof(argcopy));
838
839   if(uuid_parse(argcopy, checkid) == 0) {
840     /* If they kill by uuid, we'll seek and destroy -- find it anywhere */
841     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
842              base, base_trailing_slash ? "" : "/", argcopy);
843   }
844   else if((module = strchr(argcopy, '`')) != NULL) {
845     noit_check_t *check;
846     char uuid_str[37];
847     target = argcopy;
848     *module++ = '\0';
849     if((name = strchr(module+1, '`')) == NULL)
850       name = module;
851     else
852       name++;
853     check = noit_poller_lookup_by_name(target, name);
854     if(!check) {
855       return -1;
856     }
857     uuid_unparse_lower(check->checkid, uuid_str);
858     snprintf(xpath, len, "/noit/checks%s%s/check[@uuid=\"%s\"]",
859              base, base_trailing_slash ? "" : "/", uuid_str);
860   }
861   return strlen(xpath);
862 }
863
864 void
865 noit_check_stats_clear(stats_t *s) {
866   memset(s, 0, sizeof(*s));
867   s->state = NP_UNKNOWN;
868   s->available = NP_UNKNOWN;
869 }
870 void
871 free_metric(metric_t *m) {
872   if(!m) return;
873   if(m->metric_name) free(m->metric_name);
874   if(m->metric_value.i) free(m->metric_value.i);
875   free(m);
876 }
877
878 void
879 __stats_add_metric(stats_t *newstate, metric_t *m) {
880   noit_hash_replace(&newstate->metrics, m->metric_name, strlen(m->metric_name),
881                     m, NULL, (void (*)(void *))free_metric);
882 }
883
884 static size_t
885 noit_metric_sizes(metric_type_t type, void *value) {
886   switch(type) {
887     case METRIC_INT32:
888     case METRIC_UINT32:
889       return sizeof(int32_t);
890     case METRIC_INT64:
891     case METRIC_UINT64:
892       return sizeof(int64_t);
893     case METRIC_DOUBLE:
894       return sizeof(double);
895     case METRIC_STRING:
896       return strlen((char *)value) + 1;
897     case METRIC_GUESS:
898       break;
899   }
900   assert(type != type);
901   return 0;
902 }
903 static metric_type_t
904 noit_metric_guess_type(const char *s, void **replacement) {
905   char *copy, *cp, *trailer, *rpl;
906   int negative = 0;
907   metric_type_t type = METRIC_STRING;
908
909   if(!s) return METRIC_GUESS;
910   copy = cp = strdup(s);
911
912   /* TRIM the string */
913   while(*cp && isspace(*cp)) cp++; /* ltrim */
914   s = cp; /* found a good starting point */
915   while(*cp) cp++; /* advance to \0 */
916   cp--; /* back up one */
917   while(cp > s && isspace(*cp)) *cp-- = '\0'; /* rtrim */
918
919   /* Find the first space */
920   cp = (char *)s;
921   while(*cp && !isspace(*cp)) cp++;
922   trailer = cp;
923   cp--; /* backup one */
924   if(cp > s && *cp == '%') *cp-- = '\0'; /* chop a last % is there is one */
925
926   while(*trailer && isspace(*trailer)) *trailer++ = '\0'; /* rtrim */
927
928   /* string was       '  -1.23e-01%  inodes used  ' */
929   /* copy is (~ = \0) '  -1.23e-01~  inodes used~~' */
930   /*                     ^           ^              */
931   /*                     s           trailer        */
932
933   /* So, the trailer must not contain numbers */
934   while(*trailer) { if(isdigit(*trailer)) goto notanumber; trailer++; }
935
936   /* And the 's' must be of the form:
937    *  0) may start with a sign [-+]?
938    *  1) [1-9][0-9]*
939    *  2) [0]?.[0-9]+
940    *  3) 0
941    *  4) [1-9][0-9]*.[0-9]+
942    *  5) all of the above ending with e[+-][0-9]+
943    */
944    rpl = (char *)s;
945    /* CASE 0 */
946    if(s[0] == '-' || s[0] == '+') {
947      if(s[0] == '-') negative = 1;
948      s++;
949    }
950
951    if(s[0] == '.') goto decimal; /* CASE 2 */
952    if(s[0] == '0') { /* CASE 2 & 3 */
953      s++;
954      if(!s[0]) goto scanint; /* CASE 3 */
955      if(s[0] == '.') goto decimal; /* CASE 2 */
956      goto notanumber;
957    }
958    if(s[0] >= '1' && s[0] <= '9') { /* CASE 1 & 4 */
959      s++;
960      while(isdigit(s[0])) s++; /* CASE 1 & 4 */
961      if(!s[0]) goto scanint; /* CASE 1 */
962      if(s[0] == '.') goto decimal; /* CASE 4 */
963      goto notanumber;
964    }
965    /* Not case 1,2,3,4 */
966    goto notanumber;
967
968   decimal:
969    s++;
970    if(!isdigit(s[0])) goto notanumber;
971    s++;
972    while(isdigit(s[0])) s++;
973    if(!s[0]) goto scandouble;
974    if(s[0] == 'e' || s[0] == 'E') goto exponent; /* CASE 5 */
975    goto notanumber;
976
977   exponent:
978    s++;
979    if(s[0] != '-' && s[0] != '+') goto notanumber;
980    s++;
981    if(!isdigit(s[0])) goto notanumber;
982    s++;
983    while(isdigit(s[0])) s++;
984    if(!s[0]) goto scandouble;
985    goto notanumber;
986
987  scanint:
988    if(negative) {
989      int64_t *v;
990      v = calloc(1, sizeof(*v));
991      *v = strtoll(rpl, NULL, 10);
992      *replacement = v;
993      type = METRIC_INT64;
994      goto alldone;
995    }
996    else {
997      u_int64_t *v;
998      v = calloc(1, sizeof(*v));
999      *v = strtoull(rpl, NULL, 10);
1000      *replacement = v;
1001      type = METRIC_UINT64;
1002      goto alldone;
1003    }
1004  scandouble:
1005    {
1006      double *v;
1007      v = calloc(1, sizeof(*v));
1008      *v = strtod(rpl, NULL);
1009      *replacement = v;
1010      type = METRIC_DOUBLE;
1011      goto alldone;
1012    }
1013
1014  alldone:
1015  notanumber:
1016   free(copy);
1017   return type;
1018 }
1019 int
1020 noit_stats_populate_metric(metric_t *m, const char *name, metric_type_t type,
1021                            void *value) {
1022   void *replacement = NULL;
1023   if(type == METRIC_GUESS)
1024     type = noit_metric_guess_type((char *)value, &replacement);
1025   if(type == METRIC_GUESS) return -1;
1026
1027   m->metric_name = strdup(name);
1028   m->metric_type = type;
1029   if(replacement)
1030     m->metric_value.vp = replacement;
1031   else if(value) {
1032     size_t len;
1033     len = noit_metric_sizes(type, value);
1034     m->metric_value.vp = calloc(1, len);
1035     memcpy(m->metric_value.vp, value, len);
1036   }
1037   return 0;
1038 }
1039 void
1040 noit_stats_set_metric(stats_t *newstate, const char *name, metric_type_t type,
1041                       void *value) {
1042   metric_t *m = calloc(1, sizeof(*m));
1043   if(noit_stats_populate_metric(m, name, type, value)) {
1044     free_metric(m);
1045     return;
1046   }
1047   __stats_add_metric(newstate, m);
1048 }
1049 void
1050 noit_stats_log_immediate_metric(noit_check_t *check,
1051                                 const char *name, metric_type_t type,
1052                                 void *value) {
1053   struct timeval now;
1054   metric_t *m = calloc(1, sizeof(*m));
1055   if(noit_stats_populate_metric(m, name, type, value)) {
1056     free_metric(m);
1057     return;
1058   }
1059   gettimeofday(&now, NULL);
1060   noit_check_log_metric(check, &now, m);
1061   free_metric(m);
1062 }
1063
1064 void
1065 noit_check_passive_set_stats(struct _noit_module *module,
1066                              noit_check_t *check, stats_t *newstate) {
1067   noit_skiplist_node *next;
1068   noit_check_t n;
1069
1070   uuid_copy(n.checkid, check->checkid);
1071   n.period = 0;
1072
1073   noit_check_set_stats(module,check,newstate);
1074   noit_skiplist_find_neighbors(&watchlist, &n, NULL, NULL, &next);
1075   while(next && next->data) {
1076     stats_t backup;
1077     noit_check_t *wcheck = next->data;
1078     if(uuid_compare(n.checkid, wcheck->checkid)) break;
1079
1080     /* Swap the real check's stats into place */
1081     memcpy(&backup, &wcheck->stats.current, sizeof(stats_t));
1082     memcpy(&wcheck->stats.current, newstate, sizeof(stats_t));
1083     /* Write out our status */
1084     noit_check_log_status(wcheck);
1085     /* Write out all metrics */
1086     noit_check_log_metrics(wcheck);
1087     /* Swap them back out */
1088     memcpy(&wcheck->stats.current, &backup, sizeof(stats_t));
1089
1090     noit_skiplist_next(&watchlist, &next);
1091   }
1092 }
1093 void
1094 noit_check_set_stats(struct _noit_module *module,
1095                      noit_check_t *check, stats_t *newstate) {
1096   int report_change = 0;
1097   char *cp;
1098   dep_list_t *dep;
1099   if(check->stats.previous.status)
1100     free(check->stats.previous.status);
1101   noit_hash_destroy(&check->stats.previous.metrics, NULL,
1102                     (void (*)(void *))free_metric);
1103   memcpy(&check->stats.previous, &check->stats.current, sizeof(stats_t));
1104   memcpy(&check->stats.current, newstate, sizeof(stats_t));
1105   if(check->stats.current.status)
1106     check->stats.current.status = strdup(check->stats.current.status);
1107   for(cp = check->stats.current.status; cp && *cp; cp++)
1108     if(*cp == '\r' || *cp == '\n') *cp = ' ';
1109
1110   /* check for state changes */
1111   if(check->stats.current.available != NP_UNKNOWN &&
1112      check->stats.previous.available != NP_UNKNOWN &&
1113      check->stats.current.available != check->stats.previous.available)
1114     report_change = 1;
1115   if(check->stats.current.state != NP_UNKNOWN &&
1116      check->stats.previous.state != NP_UNKNOWN &&
1117      check->stats.current.state != check->stats.previous.state)
1118     report_change = 1;
1119
1120   noitL(noit_debug, "%s`%s <- [%s]\n", check->target, check->name,
1121         check->stats.current.status);
1122   if(report_change) {
1123     noitL(noit_debug, "%s`%s -> [%s:%s]\n",
1124           check->target, check->name,
1125           noit_check_available_string(check->stats.current.available),
1126           noit_check_state_string(check->stats.current.state));
1127   }
1128
1129   if(NOIT_CHECK_STATUS_ENABLED()) {
1130     char id[UUID_STR_LEN+1];
1131     uuid_unparse_lower(check->checkid, id);
1132     NOIT_CHECK_STATUS(id, check->module, check->name, check->target,
1133                       check->stats.current.available,
1134                       check->stats.current.state,
1135                       check->stats.current.status);
1136   }
1137
1138   /* Write out the bundled information */
1139   noit_check_log_bundle(check);
1140   /* count the check as complete */
1141   check_completion_count++;
1142
1143   for(dep = check->causal_checks; dep; dep = dep->next) {
1144     noit_module_t *mod;
1145     mod = noit_module_lookup(dep->check->module);
1146     assert(mod);
1147     noitL(noit_debug, "Firing %s`%s in response to %s`%s\n",
1148           dep->check->target, dep->check->name,
1149           check->target, check->name);
1150     if((dep->check->flags & NP_DISABLED) == 0)
1151       if(mod->initiate_check)
1152         mod->initiate_check(mod, dep->check, 1, check);
1153   }
1154 }
1155
1156 static int
1157 noit_console_show_watchlist(noit_console_closure_t ncct,
1158                             int argc, char **argv,
1159                             noit_console_state_t *dstate,
1160                             void *closure) {
1161   noit_skiplist_node *iter, *fiter;
1162   nc_printf(ncct, "%d active watches.\n", watchlist.size);
1163   for(iter = noit_skiplist_getlist(&watchlist); iter;
1164       noit_skiplist_next(&watchlist, &iter)) {
1165     char uuid_str[UUID_STR_LEN + 1];
1166     noit_check_t *check = iter->data;
1167
1168     uuid_unparse_lower(check->checkid, uuid_str);
1169     nc_printf(ncct, "%s:\n\t[%s`%s`%s]\n\tPeriod: %dms\n\tFeeds[%d]:\n",
1170               uuid_str, check->target, check->module, check->name,
1171               check->period, check->feeds ? check->feeds->size : 0);
1172     if(check->feeds && check->feeds->size) {
1173       for(fiter = noit_skiplist_getlist(check->feeds); fiter;
1174           noit_skiplist_next(check->feeds, &fiter)) {
1175         nc_printf(ncct, "\t\t%s\n", (const char *)fiter->data);
1176       }
1177     }
1178   }
1179   return 0;
1180 }
1181
1182 static void
1183 nc_printf_check_brief(noit_console_closure_t ncct,
1184                       noit_check_t *check) {
1185   char out[512];
1186   char uuid_str[37];
1187   snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
1188   uuid_unparse_lower(check->checkid, uuid_str);
1189   nc_printf(ncct, "%s %s\n", uuid_str, out);
1190   if(check->stats.current.status)
1191     nc_printf(ncct, "\t%s\n", check->stats.current.status);
1192 }
1193
1194 char *
1195 noit_console_conf_check_opts(noit_console_closure_t ncct,
1196                              noit_console_state_stack_t *stack,
1197                              noit_console_state_t *dstate,
1198                              int argc, char **argv, int idx) {
1199   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
1200   uuid_t key_id;
1201   int klen, i = 0;
1202   void *vcheck;
1203
1204   if(argc == 1) {
1205     if(!strncmp("new", argv[0], strlen(argv[0]))) {
1206       if(idx == i) return strdup("new");
1207       i++;
1208     }
1209     while(noit_hash_next(&polls, &iter, (const char **)key_id, &klen,
1210                          &vcheck)) {
1211       noit_check_t *check = (noit_check_t *)vcheck;
1212       char out[512];
1213       char uuid_str[37];
1214       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
1215       uuid_unparse_lower(check->checkid, uuid_str);
1216       if(!strncmp(out, argv[0], strlen(argv[0]))) {
1217         if(idx == i) return strdup(out);
1218         i++;
1219       }
1220       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
1221         if(idx == i) return strdup(uuid_str);
1222         i++;
1223       }
1224     }
1225   }
1226   if(argc == 2) {
1227     cmd_info_t *cmd;
1228     if(!strcmp("new", argv[0])) return NULL;
1229     cmd = noit_skiplist_find(&dstate->cmds, "attribute", NULL);
1230     if(!cmd) return NULL;
1231     return noit_console_opt_delegate(ncct, stack, cmd->dstate, argc-1, argv+1, idx);
1232   }
1233   return NULL;
1234 }
1235
1236 char *
1237 noit_console_check_opts(noit_console_closure_t ncct,
1238                         noit_console_state_stack_t *stack,
1239                         noit_console_state_t *dstate,
1240                         int argc, char **argv, int idx) {
1241   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
1242   uuid_t key_id;
1243   int klen, i = 0;
1244
1245   if(argc == 1) {
1246     void *vcheck;
1247     while(noit_hash_next(&polls, &iter, (const char **)key_id, &klen,
1248                          &vcheck)) {
1249       char out[512];
1250       char uuid_str[37];
1251       noit_check_t *check = (noit_check_t *)vcheck;
1252       snprintf(out, sizeof(out), "%s`%s", check->target, check->name);
1253       uuid_unparse_lower(check->checkid, uuid_str);
1254       if(!strncmp(out, argv[0], strlen(argv[0]))) {
1255         if(idx == i) return strdup(out);
1256         i++;
1257       }
1258       if(!strncmp(uuid_str, argv[0], strlen(argv[0]))) {
1259         if(idx == i) return strdup(uuid_str);
1260         i++;
1261       }
1262     }
1263   }
1264   if(argc == 2) {
1265     return noit_console_opt_delegate(ncct, stack, dstate, argc-1, argv+1, idx);
1266   }
1267   return NULL;
1268 }
1269
1270 static int
1271 noit_console_show_checks(noit_console_closure_t ncct,
1272                          int argc, char **argv,
1273                          noit_console_state_t *dstate,
1274                          void *closure) {
1275   noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
1276   uuid_t key_id;
1277   int klen;
1278   void *vcheck;
1279
1280   while(noit_hash_next(&polls, &iter, (const char **)key_id, &klen,
1281                        &vcheck)) {
1282     nc_printf_check_brief(ncct, (noit_check_t *)vcheck);
1283   }
1284   return 0;
1285 }
1286
1287 static void
1288 register_console_check_commands() {
1289   noit_console_state_t *tl;
1290   cmd_info_t *showcmd;
1291
1292   tl = noit_console_state_initial();
1293   showcmd = noit_console_state_get_cmd(tl, "show");
1294   assert(showcmd && showcmd->dstate);
1295
1296   noit_console_state_add_cmd(showcmd->dstate,
1297     NCSCMD("checks", noit_console_show_checks, NULL, NULL, NULL));
1298
1299   noit_console_state_add_cmd(showcmd->dstate,
1300     NCSCMD("watches", noit_console_show_watchlist, NULL, NULL, NULL));
1301 }
1302
Note: See TracBrowser for help on using the browser.