root/src/modules/httptrap.c

Revision 4496cc4e7850184a250647f0a956c335c15f5f18, 19.0 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 2 years ago)

Allow checks to note that they are passive.

If a check doesn't "do" anything to collect metrics and instead
passively allows metrics to arrive, then when you clone the check
to a transient copy, the metrics will always be blank. By
allowing modules to note that the check acts this way, we can
be smarter elsewhere. We should be able to use this to allow
transient streaming of things like statsd, collectd and traps.

  • Property mode set to 100644
Line 
1 /*
2  * Copyright (c) 2011, OmniTI Computer Consulting, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are
7  * met:
8  *
9  *     * Redistributions of source code must retain the above copyright
10  *       notice, this list of conditions and the following disclaimer.
11  *     * Redistributions in binary form must reproduce the above
12  *       copyright notice, this list of conditions and the following
13  *       disclaimer in the documentation and/or other materials provided
14  *       with the distribution.
15  *     * Neither the name OmniTI Computer Consulting, Inc. nor the names
16  *       of its contributors may be used to endorse or promote products
17  *       derived from this software without specific prior written
18  *       permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
31
32 #include "noit_defines.h"
33
34 #include <stdio.h>
35 #include <unistd.h>
36 #include <errno.h>
37 #include <assert.h>
38 #include <math.h>
39 #include <ctype.h>
40
41 #include "noit_module.h"
42 #include "noit_check.h"
43 #include "noit_check_tools.h"
44 #include "noit_rest.h"
45 #include "yajl-lib/yajl_parse.h"
46 #include "utils/noit_log.h"
47 #include "utils/noit_hash.h"
48
49 #define MAX_DEPTH 32
50
51 static noit_log_stream_t nlerr = NULL;
52 static noit_log_stream_t nldeb = NULL;
53
54 typedef struct _mod_config {
55   noit_hash_table *options;
56   noit_boolean asynch_metrics;
57 } httptrap_mod_config_t;
58
59 typedef struct httptrap_closure_s {
60   noit_module_t *self;
61   stats_t current;
62   int stats_count;
63 } httptrap_closure_t;
64
65 struct value_list {
66   char *v;
67   struct value_list *next;
68 };
69 struct rest_json_payload {
70   noit_check_t *check;
71   stats_t *stats;
72   yajl_handle parser;
73   int len;
74   int complete;
75   char *error;
76   int depth;
77   char *keys[MAX_DEPTH];
78   char array_depth[MAX_DEPTH];
79   unsigned char last_special_key;
80   unsigned char saw_complex_type;
81   metric_type_t last_type;
82   struct value_list *last_value;
83   int cnt;
84   noit_boolean immediate;
85 };
86
87 #define NEW_LV(json,a) do { \
88   struct value_list *nlv = malloc(sizeof(*nlv)); \
89   nlv->v = a; \
90   nlv->next = json->last_value; \
91   json->last_value = nlv; \
92 } while(0)
93
94 static noit_boolean
95 noit_httptrap_check_aynsch(noit_module_t *self,
96                            noit_check_t *check) {
97   const char *config_val;
98   httptrap_mod_config_t *conf;
99   if(!self) return noit_true;
100   conf = noit_module_get_userdata(self);
101   if(!conf) return noit_true;
102   noit_boolean is_asynch = conf->asynch_metrics;
103   if(noit_hash_retr_str(check->config,
104                         "asynch_metrics", strlen("asynch_metrics"),
105                         (const char **)&config_val)) {
106     if(!strcasecmp(config_val, "false") || !strcasecmp(config_val, "off"))
107       is_asynch = noit_false;
108   }
109
110   if(is_asynch) check->flags |= NP_SUPPRESS_METRICS;
111   else check->flags &= ~NP_SUPPRESS_METRICS;
112   return is_asynch;
113 }
114
115 static void
116 set_array_key(struct rest_json_payload *json) {
117   if(json->array_depth[json->depth] > 0) {
118     char str[256];
119     int strLen;
120     snprintf(str, sizeof(str), "%d", json->array_depth[json->depth] - 1);
121     json->array_depth[json->depth]++;
122     strLen = strlen(str);
123     if(json->keys[json->depth]) free(json->keys[json->depth]);
124     json->keys[json->depth] = NULL;
125     if(json->depth == 0) {
126       json->keys[json->depth] = malloc(strLen+1);
127       memcpy(json->keys[json->depth], str, strLen);
128       json->keys[json->depth][strLen] = '\0';
129     }
130     else {
131       int uplen = strlen(json->keys[json->depth-1]);
132       if(uplen + 1 + strLen > 255) return;
133       json->keys[json->depth] = malloc(uplen + 1 + strLen + 1);
134       memcpy(json->keys[json->depth], json->keys[json->depth-1], uplen);
135       json->keys[json->depth][uplen] = '`';
136       memcpy(json->keys[json->depth] + uplen + 1, str, strLen);
137       json->keys[json->depth][uplen + 1 + strLen] = '\0';
138     }
139   }
140 }
141 static int
142 httptrap_yajl_cb_null(void *ctx) {
143   struct rest_json_payload *json = ctx;
144   set_array_key(json);
145   if(json->last_special_key == 0x2) {
146     NEW_LV(json, NULL);
147     return 1;
148   }
149   if(json->last_special_key) return 0;
150   noit_stats_set_metric(json->check, json->stats,
151       json->keys[json->depth], METRIC_INT32, NULL);
152   if(json->immediate)
153     noit_stats_log_immediate_metric(json->check,
154         json->keys[json->depth], METRIC_INT32, NULL);
155   json->cnt++;
156   return 1;
157 }
158 static int
159 httptrap_yajl_cb_boolean(void *ctx, int boolVal) {
160   int ival;
161   struct rest_json_payload *json = ctx;
162   set_array_key(json);
163   if(json->last_special_key == 0x2) {
164     NEW_LV(json, strdup(boolVal ? "1" : "0"));
165     return 1;
166   }
167   if(json->last_special_key) return 0;
168   ival = boolVal ? 1 : 0;
169   noit_stats_set_metric(json->check, json->stats,
170       json->keys[json->depth], METRIC_INT32, &ival);
171   if(json->immediate)
172     noit_stats_log_immediate_metric(json->check,
173         json->keys[json->depth], METRIC_INT32, &ival);
174   json->cnt++;
175   return 1;
176 }
177 static int
178 httptrap_yajl_cb_number(void *ctx, const char * numberVal,
179                         size_t numberLen) {
180   char val[128];
181   struct rest_json_payload *json = ctx;
182   set_array_key(json);
183   if(json->last_special_key == 0x2) {
184     char *str;
185     str = malloc(numberLen+1);
186     memcpy(str, numberVal, numberLen);
187     str[numberLen] = '\0';
188     NEW_LV(json, str);
189     return 1;
190   }
191   if(json->last_special_key) return 0;
192   if(numberLen > sizeof(val)-1) numberLen = sizeof(val)-1;
193   memcpy(val, numberVal, numberLen);
194   val[numberLen] = '\0';
195   noit_stats_set_metric(json->check, json->stats,
196       json->keys[json->depth], METRIC_GUESS, val);
197   if(json->immediate)
198     noit_stats_log_immediate_metric(json->check,
199         json->keys[json->depth], METRIC_GUESS, val);
200   json->cnt++;
201   return 1;
202 }
203 static int
204 httptrap_yajl_cb_string(void *ctx, const unsigned char * stringVal,
205                         size_t stringLen) {
206   struct rest_json_payload *json = ctx;
207   char val[4096];
208   set_array_key(json);
209   if(json->last_special_key == 0x1) {
210     if(stringLen != 1) return 0;
211     if(*stringVal == 'L' || *stringVal == 'l' ||
212         *stringVal == 'I' || *stringVal == 'i' ||
213         *stringVal == 'n' || *stringVal == 's') {
214       json->last_type = *stringVal;
215       json->saw_complex_type |= 0x1;
216       return 1;
217     }
218     return 0;
219   }
220   else if(json->last_special_key == 0x2) {
221     char *str;
222     str = malloc(stringLen+1);
223     memcpy(str, stringVal, stringLen);
224     str[stringLen] = '\0';
225     NEW_LV(json, str);
226     json->saw_complex_type |= 0x2;
227     return 1;
228   }
229   if(stringLen > sizeof(val)-1) stringLen = sizeof(val)-1;
230   memcpy(val, stringVal, stringLen);
231   val[stringLen] = '\0';
232   noit_stats_set_metric(json->check, json->stats,
233       json->keys[json->depth], METRIC_GUESS, val);
234   if(json->immediate)
235     noit_stats_log_immediate_metric(json->check,
236         json->keys[json->depth], METRIC_GUESS, val);
237   json->cnt++;
238   return 1;
239 }
240 static int
241 httptrap_yajl_cb_start_map(void *ctx) {
242   struct rest_json_payload *json = ctx;
243   set_array_key(json);
244   json->depth++;
245   if(json->depth >= MAX_DEPTH) return 0;
246   return 1;
247 }
248 static int
249 httptrap_yajl_cb_end_map(void *ctx) {
250   struct value_list *p, *last_p = NULL;
251   struct rest_json_payload *json = ctx;
252   json->depth--;
253   if(json->saw_complex_type == 0x3) {
254     long double total = 0, cnt = 0;
255     noit_boolean use_avg = noit_false;
256     for(p=json->last_value;p;p=p->next) {
257       noit_stats_set_metric_coerce(json->check, json->stats,
258           json->keys[json->depth], json->last_type, p->v);
259       last_p = p;
260       if(json->last_type == 'L' || json->last_type == 'l' ||
261          json->last_type == 'I' || json->last_type == 'i' ||
262          json->last_type == 'n') {
263         total += strtold(p->v, NULL);
264         cnt = cnt + 1;
265         use_avg = noit_true;
266       }
267       json->cnt++;
268     }
269     if(json->immediate && last_p != NULL) {
270       if(use_avg) {
271         double avg = total / cnt;
272         noit_stats_log_immediate_metric(json->check,
273             json->keys[json->depth], 'n', &avg);
274       }
275       else {
276         noit_stats_log_immediate_metric(json->check,
277             json->keys[json->depth], json->last_type, last_p->v);
278       }
279     }
280   }
281   json->saw_complex_type = 0;
282   for(p=json->last_value;p;) {
283     struct value_list *savenext;
284     savenext = p->next;
285     if(p->v) free(p->v);
286     savenext = p->next;
287     free(p);
288     p = savenext;
289   }
290   json->last_value = NULL;
291   return 1;
292 }
293 static int
294 httptrap_yajl_cb_start_array(void *ctx) {
295   struct rest_json_payload *json = ctx;
296   json->depth++;
297   json->array_depth[json->depth]++;
298   return 1;
299 }
300 static int
301 httptrap_yajl_cb_end_array(void *ctx) {
302   struct rest_json_payload *json = ctx;
303   json->array_depth[json->depth] = 0;
304   json->depth--;
305   return 1;
306 }
307 static int
308 httptrap_yajl_cb_map_key(void *ctx, const unsigned char * key,
309                          size_t stringLen) {
310   struct rest_json_payload *json = ctx;
311   if(stringLen > 255) return 0;
312   if(json->keys[json->depth]) free(json->keys[json->depth]);
313   json->keys[json->depth] = NULL;
314   if(stringLen == 5 && memcmp(key, "_type", 5) == 0) {
315     json->last_special_key = 0x1;
316     if(json->depth > 0) json->keys[json->depth] = strdup(json->keys[json->depth-1]);
317     return 1;
318   }
319   if(stringLen == 6 && memcmp(key, "_value", 6) == 0) {
320     if(json->depth > 0) json->keys[json->depth] = strdup(json->keys[json->depth-1]);
321     json->last_special_key = 0x2;
322     json->saw_complex_type |= 0x2;
323     return 1;
324   }
325   json->last_special_key = 0;
326   if(json->depth == 0) {
327     json->keys[json->depth] = malloc(stringLen+1);
328     memcpy(json->keys[json->depth], key, stringLen);
329     json->keys[json->depth][stringLen] = '\0';
330   }
331   else {
332     int uplen = strlen(json->keys[json->depth-1]);
333     if(uplen + 1 + stringLen > 255) return 0;
334     json->keys[json->depth] = malloc(uplen + 1 + stringLen + 1);
335     memcpy(json->keys[json->depth], json->keys[json->depth-1], uplen);
336     json->keys[json->depth][uplen] = '`';
337     memcpy(json->keys[json->depth] + uplen + 1, key, stringLen);
338     json->keys[json->depth][uplen + 1 + stringLen] = '\0';
339   }
340   return 1;
341 }
342 static yajl_callbacks httptrap_yajl_callbacks = {
343   .yajl_null = httptrap_yajl_cb_null,
344   .yajl_boolean = httptrap_yajl_cb_boolean,
345   .yajl_number = httptrap_yajl_cb_number,
346   .yajl_string = httptrap_yajl_cb_string,
347   .yajl_start_map = httptrap_yajl_cb_start_map,
348   .yajl_map_key = httptrap_yajl_cb_map_key,
349   .yajl_end_map = httptrap_yajl_cb_end_map,
350   .yajl_start_array = httptrap_yajl_cb_start_array,
351   .yajl_end_array = httptrap_yajl_cb_end_array
352 };
353
354 static void
355 rest_json_payload_free(void *f) {
356   int i;
357   struct rest_json_payload *json = f;
358   if(json->parser) yajl_free(json->parser);
359   if(json->error) free(json->error);
360   for(i=0;i<MAX_DEPTH;i++)
361     if(json->keys[i]) free(json->keys[i]);
362   if(json->last_value) free(json->last_value);
363   free(json);
364 }
365
366 static struct rest_json_payload *
367 rest_get_json_upload(noit_http_rest_closure_t *restc,
368                     int *mask, int *complete) {
369   struct rest_json_payload *rxc;
370   noit_http_request *req = noit_http_session_request(restc->http_ctx);
371   httptrap_closure_t *ccl;
372   int content_length;
373   char buffer[32768];
374
375   content_length = noit_http_request_content_length(req);
376   rxc = restc->call_closure;
377   ccl = rxc->check->closure;
378   rxc->immediate = noit_httptrap_check_aynsch(ccl->self, rxc->check);
379   while(!rxc->complete) {
380     int len;
381     len = noit_http_session_req_consume(
382             restc->http_ctx, buffer,
383             MIN(content_length - rxc->len, sizeof(buffer)),
384             mask);
385     if(len > 0) {
386       yajl_status status;
387       status = yajl_parse(rxc->parser, (unsigned char *)buffer, len);
388       if(status != yajl_status_ok) {
389         unsigned char *err;
390         *complete = 1;
391         err = yajl_get_error(rxc->parser, 0, (unsigned char *)buffer, len);
392         rxc->error = strdup((char *)err);
393         yajl_free_error(rxc->parser, err);
394         return rxc;
395       }
396       rxc->len += len;
397     }
398     if(len < 0 && errno == EAGAIN) return NULL;
399     else if(len < 0) {
400       *complete = 1;
401       return NULL;
402     }
403     if(rxc->len == content_length) {
404       rxc->complete = 1;
405       yajl_complete_parse(rxc->parser);
406     }
407   }
408
409   *complete = 1;
410   return rxc;
411 }
412
413 static void clear_closure(noit_check_t *check, httptrap_closure_t *ccl) {
414   ccl->stats_count = 0;
415   noit_check_stats_clear(check, &ccl->current);
416 }
417
418 static int httptrap_submit(noit_module_t *self, noit_check_t *check,
419                            noit_check_t *cause) {
420   httptrap_closure_t *ccl;
421   struct timeval duration;
422   /* We are passive, so we don't do anything for transient checks */
423   if(check->flags & NP_TRANSIENT) return 0;
424
425   noit_httptrap_check_aynsch(self, check);
426   if(!check->closure) {
427     ccl = check->closure = (void *)calloc(1, sizeof(httptrap_closure_t));
428     memset(ccl, 0, sizeof(httptrap_closure_t));
429     ccl->self = self;
430   } else {
431     // Don't count the first run
432     char human_buffer[256];
433     ccl = (httptrap_closure_t*)check->closure;
434     gettimeofday(&ccl->current.whence, NULL);
435     sub_timeval(ccl->current.whence, check->last_fire_time, &duration);
436     ccl->current.duration = duration.tv_sec * 1000 + duration.tv_usec / 1000;
437
438     snprintf(human_buffer, sizeof(human_buffer),
439              "dur=%d,run=%d,stats=%d", ccl->current.duration,
440              check->generation, ccl->stats_count);
441     noitL(nldeb, "httptrap(%s) [%s]\n", check->target, human_buffer);
442
443     // Not sure what to do here
444     ccl->current.available = (ccl->stats_count > 0) ?
445         NP_AVAILABLE : NP_UNAVAILABLE;
446     ccl->current.state = (ccl->stats_count > 0) ?
447         NP_GOOD : NP_BAD;
448     ccl->current.status = human_buffer;
449     if(check->last_fire_time.tv_sec)
450       noit_check_passive_set_stats(check, &ccl->current);
451
452     memcpy(&check->last_fire_time, &ccl->current.whence, sizeof(duration));
453   }
454   clear_closure(check, ccl);
455   return 0;
456 }
457
458 static int
459 push_payload_at_check(struct rest_json_payload *rxc) {
460   httptrap_closure_t *ccl;
461   noit_boolean immediate;
462   char key[256];
463
464   if (!rxc->check || strcmp(rxc->check->module, "httptrap")) return 0;
465   if (rxc->check->closure == NULL) return 0;
466   ccl = rxc->check->closure;
467   immediate = noit_httptrap_check_aynsch(ccl->self,rxc->check);
468
469   /* do it here */
470   ccl->stats_count = rxc->cnt;
471   return rxc->cnt;
472 }
473
474 static int
475 rest_httptrap_handler(noit_http_rest_closure_t *restc,
476                       int npats, char **pats) {
477   int mask, complete = 0, cnt;
478   struct rest_json_payload *rxc = NULL;
479   const char *error = "internal error", *secret = NULL;
480   noit_http_session_ctx *ctx = restc->http_ctx;
481   char json_out[128];
482   noit_check_t *check;
483   uuid_t check_id;
484
485   if(npats != 2) {
486     error = "bad uri";
487     goto error;
488   }
489   if(uuid_parse(pats[0], check_id)) {
490     error = "uuid parse error";
491     goto error;
492   }
493
494   if(restc->call_closure == NULL) {
495     httptrap_closure_t *ccl;
496     rxc = restc->call_closure = calloc(1, sizeof(*rxc));
497     check = noit_poller_lookup(check_id);
498     if(!check || strcmp(check->module, "httptrap")) {
499       error = "no such httptrap check";
500       goto error;
501     }
502     noit_hash_retr_str(check->config, "secret", strlen("secret"), &secret);
503     if(!secret) secret = "";
504     if(strcmp(pats[1], secret)) {
505       error = "secret mismatch";
506       goto error;
507     }
508     rxc->check = check;
509     ccl = check->closure;
510     if(!ccl) {
511       error = "noitd is booting, try again in a bit";
512       goto error;
513     }
514     rxc->stats = &ccl->current;
515     rxc->parser = yajl_alloc(&httptrap_yajl_callbacks, NULL, rxc);
516     rxc->depth = -1;
517     yajl_config(rxc->parser, yajl_allow_comments, 1);
518     yajl_config(rxc->parser, yajl_dont_validate_strings, 1);
519     yajl_config(rxc->parser, yajl_allow_trailing_garbage, 1);
520     yajl_config(rxc->parser, yajl_allow_partial_values, 1);
521     restc->call_closure_free = rest_json_payload_free;
522   }
523
524   rxc = rest_get_json_upload(restc, &mask, &complete);
525   if(rxc == NULL && !complete) return mask;
526
527   if(!rxc) goto error;
528   if(rxc->error) goto error;
529
530   cnt = push_payload_at_check(rxc);
531
532   noit_http_response_ok(ctx, "application/json");
533   snprintf(json_out, sizeof(json_out),
534            "{ \"stats\": %d }", cnt);
535   noit_http_response_append(ctx, json_out, strlen(json_out));
536   noit_http_response_end(ctx);
537   return 0;
538
539  error:
540   noit_http_response_server_error(ctx, "application/json");
541   noit_http_response_append(ctx, "{ error: \"", 10);
542   if(rxc && rxc->error) error = rxc->error;
543   noit_http_response_append(ctx, error, strlen(error));
544   noit_http_response_append(ctx, "\" }", 3);
545   noit_http_response_end(ctx);
546   return 0;
547 }
548
549 static int noit_httptrap_initiate_check(noit_module_t *self,
550                                         noit_check_t *check,
551                                         int once, noit_check_t *cause) {
552   check->flags |= NP_PASSIVE_COLLECTION;
553   if (check->closure == NULL) {
554     httptrap_closure_t *ccl;
555     ccl = check->closure = (void *)calloc(1, sizeof(httptrap_closure_t));
556     ccl->self = self;
557   }
558   INITIATE_CHECK(httptrap_submit, self, check, cause);
559   return 0;
560 }
561
562 static int noit_httptrap_config(noit_module_t *self, noit_hash_table *options) {
563   httptrap_mod_config_t *conf;
564   conf = noit_module_get_userdata(self);
565   if(conf) {
566     if(conf->options) {
567       noit_hash_destroy(conf->options, free, free);
568       free(conf->options);
569     }
570   }
571   else
572     conf = calloc(1, sizeof(*conf));
573   conf->options = options;
574   noit_module_set_userdata(self, conf);
575   return 1;
576 }
577
578 static int noit_httptrap_onload(noit_image_t *self) {
579   if(!nlerr) nlerr = noit_log_stream_find("error/httptrap");
580   if(!nldeb) nldeb = noit_log_stream_find("debug/httptrap");
581   if(!nlerr) nlerr = noit_error;
582   if(!nldeb) nldeb = noit_debug;
583   return 0;
584 }
585
586 static int noit_httptrap_init(noit_module_t *self) {
587   const char *config_val;
588   httptrap_mod_config_t *conf;
589   conf = noit_module_get_userdata(self);
590
591   conf->asynch_metrics = noit_true;
592   if(noit_hash_retr_str(conf->options,
593                         "asynch_metrics", strlen("asynch_metrics"),
594                         (const char **)&config_val)) {
595     if(!strcasecmp(config_val, "false") || !strcasecmp(config_val, "off"))
596       conf->asynch_metrics = noit_false;
597   }
598
599   noit_module_set_userdata(self, conf);
600
601   /* register rest handler */
602   noit_http_rest_register("PUT", "/module/httptrap/",
603                           "^(" UUID_REGEX ")/([^/]*).*$",
604                           rest_httptrap_handler);
605   return 0;
606 }
607
608 #include "httptrap.xmlh"
609 noit_module_t httptrap = {
610   {
611     NOIT_MODULE_MAGIC,
612     NOIT_MODULE_ABI_VERSION,
613     "httptrap",
614     "httptrap collection",
615     httptrap_xml_description,
616     noit_httptrap_onload
617   },
618   noit_httptrap_config,
619   noit_httptrap_init,
620   noit_httptrap_initiate_check,
621   NULL
622 };
Note: See TracBrowser for help on using the browser.