Changeset 47a77d02f1268e637a4807266064909be566e1f7

Show
Ignore:
Timestamp:
10/24/09 18:57:58 (4 years ago)
Author:
Theo Schlossnagle <jesus@omniti.com>
git-committer:
Theo Schlossnagle <jesus@omniti.com> 1256410678 +0000
git-parent:

[698542882ef39bf51805101908d69d90b9a2291a]

git-author:
Theo Schlossnagle <jesus@omniti.com> 1256410678 +0000
Message:

flag-day(stratcond,webconsole,database) get realtime graphs working on uuids

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • sql/sprocs/stratcon.get_storage_node_for_sid.sql

    rb2014c0 r47a77d0  
    1 CREATE OR REPLACE FUNCTION stratcon.choose_storage_node_for_sid(in v_noit text, out o_storage_node_id int, out o_fqdn text, out o_dsn text)  
     1CREATE OR REPLACE FUNCTION stratcon.choose_storage_node_for_sid(in v_uuid uuid, in v_noit text, out o_storage_node_id int, out o_fqdn text, out o_dsn text)  
    22RETURNS RECORD  
    33AS $$ 
    44BEGIN 
    5     -- we could be smart with v_noit 
     5    -- we could be smart with v_uuid and/or v_noit 
    66    SELECT storage_node_id, fqdn, dsn from stratcon.storage_node order by random() limit 1 INTO o_storage_node_id, o_dsn;  
    77END 
    …… 
    1010;  
    1111 
    12 GRANT EXECUTE ON FUNCTION stratcon.choose_storage_node_for_sid(text) TO stratcon; 
     12GRANT EXECUTE ON FUNCTION stratcon.choose_storage_node_for_sid(uuid, text) TO stratcon; 

  • sql/sprocs/stratcon.map_uuid_to_sid.sql

    r08e498f r47a77d0  
    1 CREATE OR REPLACE FUNCTION stratcon.map_uuid_to_sid(in v_uuid uuid, in v_noit text, out o_sid int, out o_storage_node_id int, out o_fqdn text, out o_dsn text)  
     1CREATE OR REPLACE FUNCTION stratcon.map_uuid_to_sid(in v_uuid uuid, in v_noit text, out o_sid int, out o_noit text, out o_storage_node_id int, out o_fqdn text, out o_dsn text)  
    22RETURNS RECORD  
    33AS $$ 
    …… 
    66    -- but I think we had a reason to use it 
    77 
    8     SELECT sid, storage_node_id, fqdn, dsn 
     8    SELECT sid, storage_node_id, noit, fqdn, dsn 
    99      FROM stratcon.map_uuid_to_sid LEFT JOIN stratcon.storage_node USING (storage_node_id) 
    10      WHERE id = v_uuid INTO o_sid, o_storage_node_id, o_fqdn, o_dsn;  
     10     WHERE id = v_uuid INTO o_sid, o_storage_node_id, o_noit, o_fqdn, o_dsn;  
    1111    IF NOT FOUND THEN 
    1212 
    1313        SELECT nextval('stratcon.map_uuid_to_sid_sid_seq') INTO o_sid;   
    14         SELECT * FROM stratcon.choose_storage_node_for_sid(v_noit) INTO o_storage_node_id, o_fqdn, o_dsn;  
     14        SELECT * FROM stratcon.choose_storage_node_for_sid(v_uuid, v_noit) INTO o_storage_node_id, o_fqdn, o_dsn;  
    1515 
    16         INSERT INTO stratcon.map_uuid_to_sid(id,sid,storage_node_id) VALUES (v_uuid, o_sid, o_storage_node_id);  
     16        INSERT INTO stratcon.map_uuid_to_sid(id,sid,noit,storage_node_id) VALUES (v_uuid, o_sid, v_noit, o_storage_node_id);  
    1717 
    1818    END IF; 

  • sql/tables/stratcon.map_uuid_to_sid.sql

    ra34bc12 r47a77d0  
    2222    id uuid NOT NULL, 
    2323    sid serial NOT NULL, 
     24    noit text NOT NULL, 
    2425    storage_node_id integer REFERENCES stratcon.storage_node 
    2526); 

  • src/noit_check_rest.c

    recb7471 r47a77d0  
    4646 
    4747#define FAIL(a) do { error = (a); goto error; } while(0) 
    48 #define UUID_REGEX "[0-9a-fA-F]{4}(?:[0-9a-fA-F]{4}-){4}[0-9a-fA-F]{12}" 
    4948 
    5049#define NODE_CONTENT(parent, k, v) do { \ 

  • src/noit_defines.h

    r88a7178 r47a77d0  
    135135#include "noitedit/strlcpy.h" 
    136136 
     137#define UUID_REGEX "[0-9a-fA-F]{4}(?:[0-9a-fA-F]{4}-){4}[0-9a-fA-F]{12}" 
     138 
    137139#endif 

  • src/stratcon.conf.in

    rf742d08 r47a77d0  
    8484      <allchecks><![CDATA[ 
    8585        SELECT remote_address, id, target, module, name 
    86           FROM noit.get_checks() 
     86          FROM check_currently 
    8787      ]]></allchecks> 
    8888      <findcheck><![CDATA[ 
    8989        SELECT remote_address, id, target, module, name 
    90           FROM noit.get_check($1,$2) 
     90          FROM check_currently 
     91         WHERE sid = $1 
    9192      ]]></findcheck> 
    9293      <allstoragenodes><![CDATA[ 
    …… 
    100101      ]]></findstoragenode> 
    101102      <mapallchecks><![CDATA[ 
    102         SELECT id, sid, storage_node_id, fqdn, dsn 
     103        SELECT id, sid, noit as remote_cn, storage_node_id, fqdn, dsn 
    103104          FROM stratcon.map_uuid_to_sid LEFT JOIN stratcon.storage_node USING (storage_node_id) 
    104105      ]]></mapallchecks> 

  • src/stratcon_datastore.c

    rcdb26b4 r47a77d0  
    161161static int stratcon_database_connect(conn_q *cq); 
    162162static int uuid_to_sid(const char *uuid_str_in, const char *remote_cn); 
     163static int storage_node_quick_lookup(const char *uuid_str, 
     164                                     const char *remote_cn, 
     165                                     int *sid_out, int *storagenode_id_out, 
     166                                     const char **remote_cn_out, 
     167                                     const char **fqdn_out, 
     168                                     const char **dsn_out); 
    163169 
    164170static void 
    …… 
    178184typedef struct { 
    179185  char *uuid_str; 
     186  char *remote_cn; 
    180187  int storagenode_id; 
    181188  int sid; 
    …… 
    476483  char buff[1024]; 
    477484  conn_q *cq; 
    478   cq = get_conn_q_for_metanode(); 
    479  
    480485  if(!(mask & EVENTER_ASYNCH_WORK)) return 0; 
    481486  if(mask & EVENTER_ASYNCH_CLEANUP) return 0; 
    482487 
     488  cq = get_conn_q_for_metanode(); 
    483489  stratcon_database_connect(cq); 
    484490  d = calloc(1, sizeof(*d)); 
    …… 
    538544} 
    539545execute_outcome_t 
    540 stratcon_datastore_find(conn_q *cq, ds_rt_detail *d) { 
     546stratcon_datastore_find(ds_rt_detail *d) { 
     547  conn_q *cq; 
    541548  char *val; 
    542549  int row_count; 
    543550  struct realtime_tracker *node; 
    544551 
    545   GET_QUERY(check_find); 
    546552  for(node = d->rt; node; node = node->next) { 
     553    char uuid_str[UUID_STR_LEN+1]; 
     554    const char *fqdn, *dsn, *remote_cn; 
     555    int storagenode_id; 
     556 
     557    uuid_unparse_lower(node->checkid, uuid_str); 
     558    if(storage_node_quick_lookup(uuid_str, NULL, &node->sid, 
     559                                 &storagenode_id, &remote_cn, &fqdn, &dsn)) 
     560      continue; 
     561 
     562    cq = get_conn_q_for_remote(NULL, remote_cn, fqdn, dsn); 
     563    stratcon_database_connect(cq); 
     564 
     565    GET_QUERY(check_find); 
    547566    DECLARE_PARAM_INT(node->sid); 
    548567    PG_EXEC(check_find); 
    …… 
    575594    free_params((ds_single_detail *)d); 
    576595    d->nparams = 0; 
     596    release_conn_q(cq); 
    577597  } 
    578598  return DS_EXEC_SUCCESS; 
    …… 
    878898                                 struct timeval *now) { 
    879899  ds_rt_detail *dsjd = closure; 
    880   conn_q *cq; 
    881900  if(!(mask & EVENTER_ASYNCH_WORK)) return 0; 
    882  
    883   cq = get_conn_q_for_metanode(); 
    884   stratcon_database_connect(cq); 
     901  if(mask & EVENTER_ASYNCH_CLEANUP) return 0; 
     902 
    885903  assert(dsjd->rt); 
    886   stratcon_datastore_find(cq, dsjd); 
     904  stratcon_datastore_find(dsjd); 
    887905  if(dsjd->completion_event) 
    888906    eventer_add(dsjd->completion_event); 
    …… 
    890908  free_params((ds_single_detail *)dsjd); 
    891909  free(dsjd); 
    892   release_conn_q(cq); 
    893910  return 0; 
    894911} 
    …… 
    11601177  return ij; 
    11611178} 
    1162 static void 
     1179static int 
    11631180storage_node_quick_lookup(const char *uuid_str, const char *remote_cn, 
    11641181                          int *sid_out, int *storagenode_id_out, 
    1165                           char **fqdn_out, char **dsn_out) { 
     1182                          const char **remote_cn_out, 
     1183                          const char **fqdn_out, const char **dsn_out) { 
    11661184  /* only called from the main thread -- no safety issues */ 
    11671185  void *vuuidinfo, *vinfo; 
    …… 
    11731191  if(!noit_hash_retrieve(&uuid_to_info_cache, uuid_str, strlen(uuid_str), 
    11741192                         &vuuidinfo)) { 
    1175     int row_count; 
     1193    int row_count = 0; 
    11761194    char *tmpint; 
    11771195    ds_single_detail *d; 
    11781196    conn_q *cq; 
     1197 
     1198    /* We can't do a database lookup without the remote_cn */ 
     1199    if(!remote_cn) return -1; 
     1200 
    11791201    d = calloc(1, sizeof(*d)); 
    11801202    cq = get_conn_q_for_metanode(); 
    …… 
    12021224    free(d); 
    12031225    release_conn_q(cq); 
     1226    if(row_count != 1) return -1; 
    12041227    /* Place in cache */ 
    12051228    if(fqdn) fqdn = strdup(fqdn); 
    …… 
    12071230    uuidinfo->sid = sid; 
    12081231    uuidinfo->uuid_str = strdup(uuid_str); 
     1232    uuidinfo->remote_cn = strdup(remote_cn); 
    12091233    noit_hash_store(&uuid_to_info_cache, 
    12101234                    uuidinfo->uuid_str, strlen(uuidinfo->uuid_str), uuidinfo); 
    …… 
    12511275  if(fqdn_out) *fqdn_out = fqdn ? fqdn : (info ? info->fqdn : NULL); 
    12521276  if(dsn_out) *dsn_out = dsn ? dsn : (info ? info->dsn : NULL); 
     1277  if(remote_cn_out) *remote_cn_out = uuidinfo->remote_cn; 
    12531278  if(storagenode_id_out) *storagenode_id_out = uuidinfo->storagenode_id; 
    12541279  if(sid_out) *sid_out = uuidinfo->sid; 
     1280  return 0; 
    12551281} 
    12561282static int 
    …… 
    12591285  int sid = 0; 
    12601286  strlcpy(uuid_str, uuid_str_in, sizeof(uuid_str)); 
    1261   storage_node_quick_lookup(uuid_str, remote_cn, &sid, NULL, NULL, NULL); 
     1287  storage_node_quick_lookup(uuid_str, remote_cn, &sid, NULL, NULL, NULL, NULL); 
    12621288  return sid; 
    12631289} 
    …… 
    12661292                           const char *remote_cn, const char *line) { 
    12671293  interim_journal_t *ij = NULL; 
    1268   char uuid_str[UUID_STR_LEN+1], *cp, *fqdn, *dsn; 
     1294  char uuid_str[UUID_STR_LEN+1], *cp; 
     1295  const char *fqdn, *dsn; 
    12691296  int storagenode_id = 0; 
    12701297  uuid_t checkid; 
    …… 
    12791306        if(!uuid_parse(uuid_str, checkid)) { 
    12801307          storage_node_quick_lookup(uuid_str, remote_cn, NULL, 
    1281                                     &storagenode_id, &fqdn, &dsn); 
     1308                                    &storagenode_id, NULL, &fqdn, &dsn); 
    12821309          ij = interim_journal_get(remote, remote_cn, storagenode_id, fqdn); 
    12831310        } 
    …… 
    15411568  for(i=0; i<cnt; i++) { 
    15421569    void *vinfo; 
    1543     char *tmpint, *fqdn, *dsn, *uuid_str; 
     1570    char *tmpint, *fqdn, *dsn, *uuid_str, *remote_cn; 
    15441571    int sid, storagenode_id; 
    15451572    uuid_info *uuidinfo; 
    …… 
    15511578    PG_GET_STR_COL(fqdn, i, "fqdn"); 
    15521579    PG_GET_STR_COL(dsn, i, "dsn"); 
     1580    PG_GET_STR_COL(remote_cn, i, "remote_cn"); 
    15531581    PG_GET_STR_COL(tmpint, i, "storage_node_id"); 
    15541582    storagenode_id = tmpint ? atoi(tmpint) : 0; 
    …… 
    15561584    uuidinfo = calloc(1, sizeof(*uuidinfo)); 
    15571585    uuidinfo->uuid_str = strdup(uuid_str); 
     1586    uuidinfo->remote_cn = strdup(remote_cn); 
    15581587    uuidinfo->sid = sid; 
    15591588    noit_hash_store(&uuid_to_info_cache, 

  • src/stratcon_iep.c

    r5360a1e r47a77d0  
    536536 
    537537  if(op == DS_OP_CHKPT) { 
    538     eventer_add((eventer_t) operand); 
     538    eventer_add(completion); 
    539539    return; 
    540540  } 

  • src/stratcon_realtime_http.c

    r5360a1e r47a77d0  
    198198       interest; 
    199199       interest = strtok_r(NULL, "/", &brk)) { 
     200    uuid_t in_uuid; 
    200201    struct realtime_tracker *node; 
    201202    char *interval; 
    …… 
    206207    else 
    207208      *interval++ = '\0'; 
     209    if(uuid_parse(interest, in_uuid)) continue; 
    208210    node = calloc(1, sizeof(*node)); 
    209211    node->rc = rc; 
    210     node->sid = atoi(interest); 
     212    uuid_copy(node->checkid, in_uuid); 
    211213    node->interval = atoi(interval); 
    212214    node->next = rc->checklist; 
    …… 
    426428    /* Each interest references the ctx */ 
    427429    for(node = rc->checklist; node; node = node->next) { 
     430      char uuid_str[UUID_STR_LEN+1]; 
    428431      noit_atomic_inc32(&ctx->ref_cnt); 
    429       noitL(noit_error, "Resolving sid: %d\n", node->sid); 
     432      uuid_unparse_lower(node->checkid, uuid_str); 
     433      noitL(noit_error, "Resolving uuid: %s\n", uuid_str); 
    430434    } 
    431435    completion = eventer_alloc(); 
    …… 
    477481                        stratcon_realtime_http_handler); 
    478482  assert(noit_http_rest_register( 
    479     "GET", "/data/", "^((?:\\d+(?:@\\d+)?)(?:/\\d+(?:@\\d+)?)*)$", 
     483    "GET", "/data/", 
     484           "^((?:" UUID_REGEX "(?:@\\d+)?)(?:/" UUID_REGEX "(?:@\\d+)?)*)$", 
    480485    rest_stream_data 
    481486  ) == 0); 

  • src/stratcon_realtime_http.h

    r88a7178 r47a77d0  
    4040 */ 
    4141struct realtime_tracker { 
    42   int sid;        /* set by request */ 
     42  uuid_t checkid; /* set by request */ 
    4343  int interval;   /* set by request */ 
    4444  char *noit;     /* resolved by datastore */ 
    45   uuid_t checkid; /* resolved by datastore */ 
     45  int sid;        /* resolved by datastore */ 
    4646  struct realtime_tracker *next; /* next in series */ 
    4747  eventer_t conn; /* used to track noitd connection feeding this */ 

  • ui/web/etc/httpd.conf.in

    r8f79bcd r47a77d0  
    1212LoadModule dir_module /usr/libexec/apache2/mod_dir.so 
    1313LoadModule rewrite_module /usr/libexec/apache2/mod_rewrite.so 
     14LoadModule proxy_module /usr/libexec/apache2/mod_proxy.so 
     15LoadModule proxy_http_module /usr/libexec/apache2/mod_proxy_http.so 
    1416 
    1517<Directory "/"> 
    …… 
    4648DirectoryIndex index.php 
    4749DefaultType text/plain 
     50 
     51RewriteEngine On 
     52RewriteRule ^(/data/.+)$ http://localhost:80$1 [P,L,QSA] 

  • ui/web/htdocs/datapoint_browse_json.php

    r462702b r47a77d0  
    4949                   ); 
    5050    if($item['unique']) { 
    51       $jitem['text'] = '<a href="javascript:graph_add_datapoint({\'sid\' : ' . $item['sid'] . ', \'module\' : \'' . $item['module'] . '\', \'target\' : \'' . $item['target'] . '\', \'name\' : \'' . $item['target'] . '`' . $item['metric_name'] . '\', \'metric_name\' : \'' . $item['metric_name'] . '\', \'metric_type\' : \'' . $item['metric_type'] . '\'})">' . $item[$want] . '</a>'; 
     51      $jitem['text'] = '<a href="javascript:graph_add_datapoint({\'id\' : \'' . $item['id'] . '\', \'sid\' : ' . $item['sid'] . ', \'module\' : \'' . $item['module'] . '\', \'target\' : \'' . $item['target'] . '\', \'name\' : \'' . $item['target'] . '`' . $item['metric_name'] . '\', \'metric_name\' : \'' . $item['metric_name'] . '\', \'metric_type\' : \'' . $item['metric_type'] . '\'})">' . $item[$want] . '</a>'; 
    5252      if($item['metric_type'] == "numeric") { 
    5353        $jitem['classes'] = 'metric numeric'; 

  • ui/web/htdocs/js/recon.js

    r4ce18ee r47a77d0  
    911911        return function() { 
    912912            graph_add_datapoint({ 
     913                'id': ds_c.id, 
    913914                'sid': ds_c.sid, 
    914915                'name': ds_c.target + '`' + ds_c.metric_name, 
    …… 
    992993        //we assume we never give datasets we dont want to graph sids, like composites 
    993994        for (var i = 0; i < g.datapoints.length; i++) { 
    994             if (g.datapoints[i].sid) { 
    995                 sidneed[g.datapoints[i].sid] = polltime; 
    996             } 
    997         } 
    998         for (var sid in sidneed) { 
    999             sids += "/" + sid + "@" + sidneed[sid]; 
     995            if (g.datapoints[i].id) { 
     996                sidneed[g.datapoints[i].id] = polltime; 
     997            } 
     998        } 
     999        for (var id in sidneed) { 
     1000            sids += "/" + id + "@" + sidneed[id]; 
    10001001        } 
    10011002        //console.log("sids request: http://" +recon_realtime_hostname+"/data"+sids);