Changeset bd6d64f84276dfd6f5c69702a74e5dbc691b389c

Show
Ignore:
Timestamp:
03/13/12 16:40:19 (2 years ago)
Author:
Theo Schlossnagle <jesus@omniti.com>
git-committer:
Theo Schlossnagle <jesus@omniti.com> 1331656819 -0400
git-parent:

[204ecdc5ad373fb076d81ab8a63fc09df931be4e]

git-author:
Theo Schlossnagle <jesus@omniti.com> 1331656819 -0400
Message:

import yajl and switch the httptrap check to use it

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • configure.in

    rb5eb9e5 rbd6d64f  
    735735src/udns/Makefile 
    736736src/json-lib/Makefile 
     737src/yajl-lib/Makefile 
    737738src/man/Makefile 
    738739src/modules/Makefile 
  • src/Makefile.in

    r880bb29 rbd6d64f  
    7979        bundle.pb-c.o 
    8080 
    81 NOIT_LIBS=libudns libnoitedit libeventer libjlog libnoit_utils libnoitjson 
     81NOIT_LIBS=libudns libnoitedit libeventer libjlog libnoit_utils libnoitjson libnoityajl 
    8282NOIT_LIB_FILES=udns/libudns.o noitedit/libnoitedit.a eventer/libeventer.a \ 
    83         jlog/libjlog.a utils/libnoit_utils.a json-lib/libnoitjson.a 
     83        jlog/libjlog.a utils/libnoit_utils.a json-lib/libnoitjson.a yajl-lib/libnoityajl.a 
    8484 
    8585STRATCON_OBJS=stratcond.o noit_main.o noit_listener.o \ 
     
    9494        bundle.pb-c.o 
    9595 
    96 STRATCON_LIBS=libudns libnoitedit libeventer libjlog libnoit_utils libnoitjson 
     96STRATCON_LIBS=libudns libnoitedit libeventer libjlog libnoit_utils libnoitjson libnoityajl 
    9797STRATCON_LIB_FILES=udns/libudns.o noitedit/libnoitedit.a eventer/libeventer.a \ 
    98         jlog/libjlog.a utils/libnoit_utils.a json-lib/libnoitjson.a 
     98        jlog/libjlog.a utils/libnoit_utils.a json-lib/libnoitjson.a yajl-lib/libnoityajl.a 
    9999 
    100100all:    reversion $(NOIT_LIBS) $(STRATCON_LIBS) $(TARGETS) java-bits make-modules make-man 
     
    136136libnoitjson: 
    137137        @(cd json-lib && $(MAKE) -s libnoitjson.a) 
     138 
     139libnoityajl: 
     140        @(cd yajl-lib && $(MAKE) -s libnoityajl.a) 
    138141 
    139142jlog/libjlog.a: libjlog 
     
    143146utils/libnoit_utils.a:  libnoit_utils 
    144147json-lib/libnoitjson.a: libnoitjson 
     148yajl-lib/libnoityajl.a: libnoityajl 
    145149 
    146150.PHONY: reversion 
     
    191195                utils/*.o \ 
    192196                json-lib/*.o \ 
     197                yajl-lib/*.o \ 
    193198                -Ljlog -ljlog \ 
    194199                -Lnoitedit -lnoitedit \ 
     
    206211                utils/*.o \ 
    207212                json-lib/*.o \ 
     213                yajl-lib/*.o \ 
    208214                -Ljlog -ljlog \ 
    209215                -Lnoitedit -lnoitedit \ 
     
    315321        (cd utils && $(MAKE) clean) 
    316322        (cd json-lib && $(MAKE) clean) 
     323        (cd yajl-lib && $(MAKE) clean) 
    317324        (cd lua && $(MAKE) clean) 
    318325        (cd java && $(MAKE) clean) 
     
    325332 
    326333distclean-subdirs: 
    327         for dir in jlog eventer udns man modules modules-lua noitedit utils json-lib lua java ; do \ 
     334        for dir in jlog eventer udns man modules modules-lua noitedit utils json-lib yajl-lib lua java ; do \ 
    328335                (cd $$dir && $(MAKE) distclean) ; \ 
    329336        done 
  • src/modules/httptrap.c

    r7a237a2 rbd6d64f  
    4343#include "noit_check_tools.h" 
    4444#include "noit_rest.h" 
    45 #include "json-lib/json.h" 
     45#include "yajl-lib/yajl_parse.h" 
    4646#include "utils/noit_log.h" 
    4747#include "utils/noit_hash.h" 
    4848 
     49#define MAX_DEPTH 32 
    4950 
    5051static noit_log_stream_t nlerr = NULL; 
     
    6263} httptrap_closure_t; 
    6364 
     65struct value_list { 
     66  char *v; 
     67  struct value_list *next; 
     68}; 
    6469struct rest_json_payload { 
    65   struct json_tokener *tok; 
    66   struct json_object *root; 
     70  noit_check_t *check; 
     71  stats_t *stats; 
     72  yajl_handle parser; 
    6773  int len; 
    6874  int complete; 
    6975  char *error; 
    70   int nput; 
     76  int depth; 
     77  char *keys[MAX_DEPTH]; 
     78  char array_depth[MAX_DEPTH]; 
     79  unsigned char last_special_key; 
     80  unsigned char saw_complex_type; 
     81  metric_type_t last_type; 
     82  struct value_list *last_value; 
     83  int cnt; 
     84}; 
     85 
     86#define NEW_LV(json,a) do { \ 
     87  struct value_list *nlv = malloc(sizeof(*nlv)); \ 
     88  nlv->v = a; \ 
     89  nlv->next = json->last_value; \ 
     90  json->last_value = nlv; \ 
     91} while(0) 
     92 
     93static void 
     94set_array_key(struct rest_json_payload *json) { 
     95  if(json->array_depth[json->depth] > 0) { 
     96    char str[256]; 
     97    int strLen; 
     98    snprintf(str, sizeof(str), "%d", json->array_depth[json->depth] - 1); 
     99    json->array_depth[json->depth]++; 
     100    strLen = strlen(str); 
     101    if(json->keys[json->depth]) free(json->keys[json->depth]); 
     102    json->keys[json->depth] = NULL; 
     103    if(json->depth == 0) { 
     104      json->keys[json->depth] = malloc(strLen+1); 
     105      memcpy(json->keys[json->depth], str, strLen); 
     106      json->keys[json->depth][strLen] = '\0'; 
     107    } 
     108    else { 
     109      int uplen = strlen(json->keys[json->depth-1]); 
     110      if(uplen + 1 + strLen > 255) return; 
     111      json->keys[json->depth] = malloc(uplen + 1 + strLen + 1); 
     112      memcpy(json->keys[json->depth], json->keys[json->depth-1], uplen); 
     113      json->keys[json->depth][uplen] = '`'; 
     114      memcpy(json->keys[json->depth] + uplen + 1, str, strLen); 
     115      json->keys[json->depth][uplen + 1 + strLen] = '\0'; 
     116    } 
     117  } 
     118
     119static int 
     120httptrap_yajl_cb_null(void *ctx) { 
     121  struct rest_json_payload *json = ctx; 
     122  set_array_key(json); 
     123  if(json->last_special_key == 0x2) { 
     124    NEW_LV(json, NULL); 
     125    return 1; 
     126  } 
     127  if(json->last_special_key) return 0; 
     128  noit_stats_set_metric(json->check, json->stats, 
     129      json->keys[json->depth], METRIC_INT32, NULL); 
     130  json->cnt++; 
     131  return 1; 
     132
     133static int 
     134httptrap_yajl_cb_boolean(void *ctx, int boolVal) { 
     135  int ival; 
     136  struct rest_json_payload *json = ctx; 
     137  set_array_key(json); 
     138  if(json->last_special_key == 0x2) { 
     139    NEW_LV(json, strdup(boolVal ? "1" : "0")); 
     140    return 1; 
     141  } 
     142  if(json->last_special_key) return 0; 
     143  ival = boolVal ? 1 : 0; 
     144  noit_stats_set_metric(json->check, json->stats, 
     145      json->keys[json->depth], METRIC_INT32, &ival); 
     146  json->cnt++; 
     147  return 1; 
     148
     149static int 
     150httptrap_yajl_cb_number(void *ctx, const char * numberVal, 
     151                        size_t numberLen) { 
     152  char val[128]; 
     153  struct rest_json_payload *json = ctx; 
     154  set_array_key(json); 
     155  if(json->last_special_key == 0x2) { 
     156    char *str; 
     157    str = malloc(numberLen+1); 
     158    memcpy(str, numberVal, numberLen); 
     159    str[numberLen] = '\0'; 
     160    NEW_LV(json, str); 
     161    return 1; 
     162  } 
     163  if(json->last_special_key) return 0; 
     164  if(numberLen > sizeof(val)-1) numberLen = sizeof(val)-1; 
     165  memcpy(val, numberVal, numberLen); 
     166  val[numberLen] = '\0'; 
     167  noit_stats_set_metric(json->check, json->stats, 
     168      json->keys[json->depth], METRIC_GUESS, val); 
     169  json->cnt++; 
     170  return 1; 
     171
     172static int 
     173httptrap_yajl_cb_string(void *ctx, const unsigned char * stringVal, 
     174                        size_t stringLen) { 
     175  struct rest_json_payload *json = ctx; 
     176  char val[4096]; 
     177  set_array_key(json); 
     178  if(json->last_special_key == 0x1) { 
     179    if(stringLen != 1) return 0; 
     180    if(*stringVal == 'L' || *stringVal == 'l' || 
     181        *stringVal == 'I' || *stringVal == 'i' || 
     182        *stringVal == 'n' || *stringVal == 's') { 
     183      json->last_type = *stringVal; 
     184      json->saw_complex_type |= 0x1; 
     185      return 1; 
     186    } 
     187    return 0; 
     188  } 
     189  else if(json->last_special_key == 0x2) { 
     190    char *str; 
     191    str = malloc(stringLen+1); 
     192    memcpy(str, stringVal, stringLen); 
     193    str[stringLen] = '\0'; 
     194    NEW_LV(json, str); 
     195    json->saw_complex_type |= 0x2; 
     196    return 1; 
     197  } 
     198  if(stringLen > sizeof(val)-1) stringLen = sizeof(val)-1; 
     199  memcpy(val, stringVal, stringLen); 
     200  val[stringLen] = '\0'; 
     201  noit_stats_set_metric(json->check, json->stats, 
     202      json->keys[json->depth], METRIC_GUESS, val); 
     203  json->cnt++; 
     204  return 1; 
     205
     206static int 
     207httptrap_yajl_cb_start_map(void *ctx) { 
     208  struct rest_json_payload *json = ctx; 
     209  set_array_key(json); 
     210  json->depth++; 
     211  if(json->depth >= MAX_DEPTH) return 0; 
     212  return 1; 
     213
     214static int 
     215httptrap_yajl_cb_end_map(void *ctx) { 
     216  struct value_list *p; 
     217  struct rest_json_payload *json = ctx; 
     218  json->depth--; 
     219  if(json->saw_complex_type == 0x3) { 
     220    for(p=json->last_value;p;p=p->next) { 
     221      noit_stats_set_metric_coerce(json->check, json->stats, 
     222          json->keys[json->depth], json->last_type, p->v); 
     223      json->cnt++; 
     224    } 
     225  } 
     226  json->saw_complex_type = 0; 
     227  for(p=json->last_value;p;) { 
     228    struct value_list *savenext; 
     229    savenext = p->next; 
     230    if(p->v) free(p->v); 
     231    savenext = p->next; 
     232    free(p); 
     233    p = savenext; 
     234  } 
     235  json->last_value = NULL; 
     236  return 1; 
     237
     238static int 
     239httptrap_yajl_cb_start_array(void *ctx) { 
     240  struct rest_json_payload *json = ctx; 
     241  json->depth++; 
     242  json->array_depth[json->depth]++; 
     243  return 1; 
     244
     245static int 
     246httptrap_yajl_cb_end_array(void *ctx) { 
     247  struct rest_json_payload *json = ctx; 
     248  json->array_depth[json->depth] = 0; 
     249  json->depth--; 
     250  return 1; 
     251
     252static int 
     253httptrap_yajl_cb_map_key(void *ctx, const unsigned char * key, 
     254                         size_t stringLen) { 
     255  struct rest_json_payload *json = ctx; 
     256  if(stringLen > 255) return 0; 
     257  if(json->keys[json->depth]) free(json->keys[json->depth]); 
     258  json->keys[json->depth] = NULL; 
     259  if(stringLen == 5 && memcmp(key, "_type", 5) == 0) { 
     260    json->last_special_key = 0x1; 
     261    if(json->depth > 0) json->keys[json->depth] = strdup(json->keys[json->depth-1]); 
     262    return 1; 
     263  } 
     264  if(stringLen == 6 && memcmp(key, "_value", 6) == 0) { 
     265    if(json->depth > 0) json->keys[json->depth] = strdup(json->keys[json->depth-1]); 
     266    json->last_special_key = 0x2; 
     267    json->saw_complex_type |= 0x2; 
     268    return 1; 
     269  } 
     270  json->last_special_key = 0; 
     271  if(json->depth == 0) { 
     272    json->keys[json->depth] = malloc(stringLen+1); 
     273    memcpy(json->keys[json->depth], key, stringLen); 
     274    json->keys[json->depth][stringLen] = '\0'; 
     275  } 
     276  else { 
     277    int uplen = strlen(json->keys[json->depth-1]); 
     278    if(uplen + 1 + stringLen > 255) return 0; 
     279    json->keys[json->depth] = malloc(uplen + 1 + stringLen + 1); 
     280    memcpy(json->keys[json->depth], json->keys[json->depth-1], uplen); 
     281    json->keys[json->depth][uplen] = '`'; 
     282    memcpy(json->keys[json->depth] + uplen + 1, key, stringLen); 
     283    json->keys[json->depth][uplen + 1 + stringLen] = '\0'; 
     284  } 
     285  return 1; 
     286
     287static yajl_callbacks httptrap_yajl_callbacks = { 
     288  .yajl_null = httptrap_yajl_cb_null, 
     289  .yajl_boolean = httptrap_yajl_cb_boolean, 
     290  .yajl_number = httptrap_yajl_cb_number, 
     291  .yajl_string = httptrap_yajl_cb_string, 
     292  .yajl_start_map = httptrap_yajl_cb_start_map, 
     293  .yajl_map_key = httptrap_yajl_cb_map_key, 
     294  .yajl_end_map = httptrap_yajl_cb_end_map, 
     295  .yajl_start_array = httptrap_yajl_cb_start_array, 
     296  .yajl_end_array = httptrap_yajl_cb_end_array 
    71297}; 
    72298 
    73299static void 
    74300rest_json_payload_free(void *f) { 
     301  int i; 
    75302  struct rest_json_payload *json = f; 
    76   if(json->tok) json_tokener_free(json->tok); 
    77   if(json->root) json_object_put(json->root); 
     303  if(json->parser) yajl_free(json->parser); 
    78304  if(json->error) free(json->error); 
     305  for(i=0;i<MAX_DEPTH;i++) 
     306    if(json->keys[i]) free(json->keys[i]); 
     307  if(json->last_value) free(json->last_value); 
    79308  free(json); 
    80309} 
     
    85314  struct rest_json_payload *rxc; 
    86315  noit_http_request *req = noit_http_session_request(restc->http_ctx); 
     316  httptrap_closure_t *ccl; 
    87317  int content_length; 
    88318  char buffer[32768]; 
    89319 
    90320  content_length = noit_http_request_content_length(req); 
    91   if(restc->call_closure == NULL) { 
    92     rxc = restc->call_closure = calloc(1, sizeof(*rxc)); 
    93     rxc->tok = json_tokener_new(); 
    94     restc->call_closure_free = rest_json_payload_free; 
    95   } 
    96321  rxc = restc->call_closure; 
     322  ccl = rxc->check->closure; 
    97323  while(!rxc->complete) { 
    98324    int len; 
     
    102328            mask); 
    103329    if(len > 0) { 
    104       struct json_object *o; 
    105       o = json_tokener_parse_ex(rxc->tok, buffer, len); 
     330      yajl_status status; 
     331      status = yajl_parse(rxc->parser, buffer, len); 
     332      if(status != yajl_status_ok) { 
     333        unsigned char *err; 
     334        *complete = 1; 
     335        err = yajl_get_error(rxc->parser, 0, buffer, len); 
     336        rxc->error = strdup(err); 
     337        yajl_free_error(rxc->parser, err); 
     338        return rxc; 
     339      } 
    106340      rxc->len += len; 
    107       if(!is_error(o)) { 
    108         rxc->root = o; 
    109       } 
    110341    } 
    111342    if(len < 0 && errno == EAGAIN) return NULL; 
     
    116347    if(rxc->len == content_length) { 
    117348      rxc->complete = 1; 
     349      yajl_complete_parse(rxc->parser); 
    118350    } 
    119351  } 
     
    186418} 
    187419 
     420/* 
    188421static int 
    189422json_parse_descent(noit_check_t *check, noit_boolean immediate, 
     
    219452        table = json_object_get_object(o); 
    220453        if(table->count == 2) { 
    221           /* this is the special key: { _type: , _value: } notation */ 
    222454          json_object *type; 
    223455          type = json_object_object_get(o, "_type"); 
     
    297529  return cnt; 
    298530} 
    299 static int 
    300 push_payload_at_check(noit_check_t *check, json_object *root) { 
     531*/ 
     532static int 
     533push_payload_at_check(struct rest_json_payload *rxc) { 
    301534  httptrap_closure_t *ccl; 
    302535  noit_boolean immediate; 
    303536  char key[256]; 
    304   int cnt; 
    305  
    306   if (check->closure == NULL) return 0; 
    307   ccl = check->closure; 
    308   if (!check || strcmp(check->module, "httptrap")) return 0; 
    309   immediate = noit_httptrap_check_aynsch(ccl->self,check); 
     537 
     538  if (!rxc->check || strcmp(rxc->check->module, "httptrap")) return 0; 
     539  if (rxc->check->closure == NULL) return 0; 
     540  ccl = rxc->check->closure; 
     541  immediate = noit_httptrap_check_aynsch(ccl->self,rxc->check); 
    310542 
    311543  /* do it here */ 
    312   key[0] = '\0'; 
    313   cnt = json_parse_descent(check, immediate, root, key); 
    314   ccl->stats_count += cnt; 
    315   return cnt; 
     544  ccl->stats_count = rxc->cnt; 
     545  return rxc->cnt; 
    316546} 
    317547 
     
    336566  } 
    337567 
     568  if(restc->call_closure == NULL) { 
     569    httptrap_closure_t *ccl; 
     570    rxc = restc->call_closure = calloc(1, sizeof(*rxc)); 
     571    check = noit_poller_lookup(check_id); 
     572    if(!check || strcmp(check->module, "httptrap")) { 
     573      error = "no such httptrap check"; 
     574      goto error; 
     575    } 
     576    noit_hash_retr_str(check->config, "secret", strlen("secret"), &secret); 
     577    if(!secret) secret = ""; 
     578    if(strcmp(pats[1], secret)) { 
     579      error = "secret mismatch"; 
     580      goto error; 
     581    } 
     582    rxc->check = check; 
     583    ccl = check->closure; 
     584    if(!ccl) { 
     585      error = "noitd is booting, try again in a bit"; 
     586      goto error; 
     587    } 
     588    rxc->stats = &ccl->current; 
     589    rxc->parser = yajl_alloc(&httptrap_yajl_callbacks, NULL, rxc); 
     590    rxc->depth = -1; 
     591    yajl_config(rxc->parser, yajl_allow_comments, 1); 
     592    yajl_config(rxc->parser, yajl_dont_validate_strings, 1); 
     593    yajl_config(rxc->parser, yajl_allow_trailing_garbage, 1); 
     594    yajl_config(rxc->parser, yajl_allow_partial_values, 1); 
     595    restc->call_closure_free = rest_json_payload_free; 
     596  } 
     597 
    338598  rxc = rest_get_json_upload(restc, &mask, &complete); 
    339599  if(rxc == NULL && !complete) return mask; 
    340600 
    341   check = noit_poller_lookup(check_id); 
    342   if(!check || strcmp(check->module, "httptrap")) { 
    343     error = "no such httptrap check"; 
    344     goto error; 
    345   } 
    346   noit_hash_retr_str(check->config, "secret", strlen("secret"), &secret); 
    347   if(!secret) secret = ""; 
    348   if(strcmp(pats[1], secret)) { 
    349     error = "secret mismatch"; 
    350     goto error; 
    351   } 
    352  
    353601  if(!rxc) goto error; 
    354   if(!rxc->root) { 
    355     error = "parse failure"; 
    356     goto error; 
    357   } 
    358602  if(rxc->error) goto error; 
    359603 
    360   cnt = push_payload_at_check(check, rxc->root); 
     604  cnt = push_payload_at_check(rxc); 
    361605 
    362606  noit_http_response_ok(ctx, "application/json"); 
     
    430674  /* register rest handler */ 
    431675  noit_http_rest_register("PUT", "/module/httptrap/", 
    432                           "^(" UUID_REGEX ")/([^/]*)$", 
     676                          "^(" UUID_REGEX ")/([^/]*).*$", 
    433677                          rest_httptrap_handler); 
    434678  return 0;