Changeset 8e9cf57ebf8fd9d4375a893d47cef1aaecce0662

Timestamp:

11/29/11 21:26:35 (2 years ago)

Author:

Theo Schlossnagle <jesus@omniti.com>

git-committer:

Theo Schlossnagle <jesus@omniti.com> 1322601995 -0500

git-parent:

[e09f057cc61b7872af870562a9ca11fc52860502]

git-author:

Theo Schlossnagle <jesus@omniti.com> 1322601995 -0500

Message:

on-disk backing store implementation

Files:

• src/noit_check_rest.c (modified) (1 diff)
• src/noit_conf.c (modified) (11 diffs)
• src/noit_conf.h (modified) (1 diff)
• src/noit_conf_checks.c (modified) (6 diffs)
• src/noit_filters_rest.c (modified) (1 diff)
• src/noit_xml.c (modified) (2 diffs)
• src/noit_xml.h (modified) (1 diff)
• src/stratcon_jlog_streamer.c (modified) (1 diff)

src/noit_check_rest.c

@@ -467 +467 @@
-  /* delete this here */
-  noit_poller_deschedule(check->checkid);
+  CONF_REMOVE(node);
-  xmlUnlinkNode(node);
-  xmlFreeNode(node);

src/noit_conf.c

@@ -36 +36 @@
-#include <unistd.h>
-#include <fcntl.h>
+#include <dirent.h>
-#include <errno.h>
-#include <assert.h>
@@ -46 +47 @@
-#include "noit_check.h"
-#include "noit_console.h"
+#include "noit_xml.h"
-#include "utils/noit_hash.h"
-#include "utils/noit_log.h"
@@ -63 +65 @@
-static xmlDocPtr master_config = NULL;
-static int config_include_cnt = -1;
+static int backingstore_include_cnt = -1;
+
-static struct {
-  xmlNodePtr insertion_point;
@@ -68 +72 @@
-  xmlDocPtr doc;
-  xmlNodePtr root;
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-static char *root_node_name = NULL;
@@ -81 +99 @@
-static u_int32_t __config_coalesce = 0;
-static u_int32_t __config_coalesce_time = 0;
+static u_int64_t max_gen_count = 0;
-void noit_conf_coalesce_changes(u_int32_t seconds) {
-  __config_coalesce_time = seconds;
@@ -96 +115 @@
-  void *jc_closure;
-};
+
+static void
+noit_xml_userdata_free(noit_xml_userdata_t *n) {
+  if(n->name) free(n->name);
+  if(n->path) free(n->path);
+}
+
-static int
-noit_conf_watch_config_and_journal(eventer_t e, int mask, void *closure,
@@ -239 +265 @@
-  config_include_nodes = NULL;
-  config_include_cnt = -1;
+
+  if(backingstore_include_nodes) {
+    int i;
+    for(i=0; i<backingstore_include_cnt; i++) {
+      if(backingstore_include_nodes[i].doc) {
+        xmlNodePtr n;
+        for(n=backingstore_include_nodes[i].insertion_point->children;
+            n; n = n->next)
+          n->parent = backingstore_include_nodes[i].root;
+        backingstore_include_nodes[i].insertion_point->children =
+          backingstore_include_nodes[i].old_children;
+        xmlFreeDoc(backingstore_include_nodes[i].doc);
+      }
+    }
+    free(backingstore_include_nodes);
+  }
+  backingstore_include_nodes = NULL;
+  backingstore_include_cnt = -1;
-}
-void
@@ -272 +316 @@
-  }
-}
+static u_int64_t
+usec_now() {
+  u_int64_t usec;
+  struct timeval tv;
+  gettimeofday(&tv, NULL);
+  usec = tv.tv_sec * 1000000UL;
+  usec += tv.tv_usec;
+  return usec;
+}
+void
+noit_conf_backingstore_remove(noit_conf_section_t vnode) {
+  xmlNodePtr node = vnode;
+  noit_xml_userdata_t *subctx = node->_private;
+  if(subctx) {
+    noitL(noit_debug, "marking %s for removal\n", subctx->path);
+    if(!backingstore_freelist) backingstore_freelist = subctx;
+    else {
+      noit_xml_userdata_t *fl = backingstore_freelist;
+      while(fl->freelist) fl = fl->freelist;
+      fl->freelist = subctx;
+    }
+    node->_private = NULL;
+  }
+  /* If we're deleted, we'll mark the parent as dirty */
+  if(node->parent) noit_conf_backingstore_dirty(node->parent);
+}
+void
+noit_conf_backingstore_dirty(noit_conf_section_t vnode) {
+  xmlNodePtr node = vnode;
+  noit_xml_userdata_t *subctx = node->_private;
+  if(subctx) {
+    subctx->dirty_time = usec_now();
+    return;
+  }
+  if(node->parent) noit_conf_backingstore_dirty(node->parent);
+}
+int
+noit_conf_backingstore_write(noit_xml_userdata_t *ctx, noit_boolean skip,
+                             xmlAttrPtr attrs, xmlNodePtr node) {
+  int failure = 0;
+  char newpath[PATH_MAX];
+  xmlNodePtr n;
+  snprintf(newpath, sizeof(newpath), "%s/.attrs", ctx->path);
+  if(attrs) {
+    xmlDocPtr tmpdoc;
+    xmlNodePtr tmpnode;
+    noitL(noit_debug, " **> %s\n", newpath);
+    tmpdoc = xmlNewDoc((xmlChar *)"1.0");
+    tmpnode = xmlNewNode(NULL, ctx->name ? (xmlChar *)ctx->name : (xmlChar *)"stub");
+    xmlDocSetRootElement(tmpdoc, tmpnode);
+    tmpnode->properties = attrs;
+    failure = noit_xmlSaveToFile(tmpdoc, newpath);
+    tmpnode->properties = NULL;
+    xmlFreeDoc(tmpdoc);
+    if(failure) return -1;
+  }
+  else if(!skip) {
+    unlink(newpath);
+  }
+  for(n = node; n; n = n->next) {
+    int leaf;
+    noit_xml_userdata_t *subctx;
+    subctx = n->_private;
+    leaf = is_stopnode(n);
+    if(!subctx) { /* This has never been written out */
+      subctx = calloc(1, sizeof(*subctx));
+      subctx->name = strdup((char *)n->name);
+      snprintf(newpath, sizeof(newpath), "%s/%s#%llu", ctx->path, n->name, ++max_gen_count);
+      if(leaf) strlcat(newpath, ".xml", sizeof(newpath));
+      subctx->path = strdup(newpath);
+      subctx->dirty_time = usec_now();
+      n->_private = subctx;
+      noitL(noit_debug, " !!> %s\n", subctx->path);
+    }
+    if(leaf) {
+      xmlDocPtr tmpdoc;
+      xmlNodePtr tmpnode;
+      if(subctx->dirty_time > last_config_flush) {
+        tmpdoc = xmlNewDoc((xmlChar *)"1.0");
+        tmpnode = xmlNewNode(NULL, n->name);
+        xmlDocSetRootElement(tmpdoc, tmpnode);
+        tmpnode->properties = n->properties;
+        tmpnode->children = n->children;
+        failure = noit_xmlSaveToFile(tmpdoc, subctx->path);
+        tmpnode->properties = NULL;
+        tmpnode->children = NULL;
+        xmlFreeDoc(tmpdoc);
+        noitL(noit_debug, " ==> %s\n", subctx->path);
+        if(failure) return -1;
+      }
+    }
+    else {
+      noit_boolean skip_attrs;
+      skip_attrs = leaf || (subctx->dirty_time <= last_config_flush);
+      noitL(noit_debug, " --> %s\n", subctx->path);
+      if(noit_conf_backingstore_write(subctx, skip_attrs, skip_attrs ? NULL : n->properties, n->children))
+        return -1;
+    }
+  }
+  return 0;
+}
+void
+noit_conf_shatter_write(xmlDocPtr doc) {
+  if(backingstore_freelist) {
+    noit_xml_userdata_t *fl, *last;
+    for(fl = backingstore_freelist; fl; ) {
+      last = fl;
+      fl = fl->freelist;
+      /* If it is a file, we'll unlink it, otherwise,
+       * we need to delete the attributes and the directory.
+       */
+      if(unlink(last->path)) {
+        char attrpath[PATH_MAX];
+        snprintf(attrpath, sizeof(attrpath), "%s/.attrs", last->path);
+        unlink(attrpath);
+        if(rmdir(last->path) && errno != ENOENT) {
+          /* This shouldn't happen, but if it does we risk
+           * leaving a mess. Don't do that.
+           */
+          noitL(noit_error, "backingstore mess %s: %s\n",
+                last->path, strerror(errno));
+        }
+      }
+      noit_xml_userdata_free(last);
+    }
+    backingstore_freelist = NULL;
+  }
+  if(backingstore_include_nodes) {
+    int i;
+    for(i=0; i<backingstore_include_cnt; i++) {
+      if(backingstore_include_nodes[i].doc) {
+        xmlNodePtr n;
+        noit_xml_userdata_t *what = backingstore_include_nodes[i].doc->_private;
+
+        for(n=backingstore_include_nodes[i].insertion_point->children;
+            n; n = n->next)
+          n->parent = backingstore_include_nodes[i].root;
+        backingstore_include_nodes[i].insertion_point->children =
+          backingstore_include_nodes[i].old_children;
+        noit_conf_backingstore_write(what, noit_false, NULL, backingstore_include_nodes[i].root->children);
+      }
+    }
+    last_config_flush = usec_now();
+  }
+}
+void
+noit_conf_shatter_postwrite(xmlDocPtr doc) {
+  if(backingstore_include_nodes) {
+    int i;
+    for(i=0; i<backingstore_include_cnt; i++) {
+      if(backingstore_include_nodes[i].doc) {
+        xmlNodePtr n;
+        backingstore_include_nodes[i].insertion_point->children =
+          backingstore_include_nodes[i].root->children;
+        for(n=backingstore_include_nodes[i].insertion_point->children;
+            n; n = n->next)
+          n->parent = backingstore_include_nodes[i].insertion_point;
+      }
+    }
+  }
+}
+
+int
+noit_conf_read_into_node(xmlNodePtr node, const char *path) {
+  DIR *dirroot;
+  struct dirent *de, *entry;
+  char filepath[PATH_MAX];
+  xmlDocPtr doc;
+  xmlNodePtr root = NULL;
+  xmlAttrPtr a;
+  struct stat sb;
+  int size, rv;
+
+  noitL(noit_debug, "read backing store: %s\n", path);
+  snprintf(filepath, sizeof(filepath), "%s/.attrs", path);
+  while((rv = stat(filepath, &sb)) < 0 && errno == EINTR);
+  if(rv == 0) {
+    doc = xmlParseFile(filepath);
+    if(doc) root = xmlDocGetRootElement(doc);
+    if(doc && root) {
+      node->properties = root->properties;
+      for(a = node->properties; a; a = a->next) {
+        a->parent = node;
+        a->doc = node->doc;
+      }
+      root->properties = NULL;
+      xmlFreeDoc(doc);
+      doc = NULL;
+    }
+  }
+#ifdef _PC_NAME_MAX
+  size = pathconf(path, _PC_NAME_MAX);
+#endif
+  size = MAX(size, PATH_MAX + 128);
+  de = alloca(size);
+  dirroot = opendir(path);
+  if(!dirroot) return -1;
+  while(portable_readdir_r(dirroot, de, &entry) == 0 && entry != NULL) {
+    noit_xml_userdata_t *udata;
+    char name[PATH_MAX];
+    char *sep;
+    xmlNodePtr child;
+    u_int64_t gen;
+
+    sep = strchr(entry->d_name, '#');
+    if(!sep) continue;
+    snprintf(filepath, sizeof(filepath), "%s/%s", path, entry->d_name);
+    while((rv = stat(filepath, &sb)) < 0 && errno == EINTR);
+    if(rv == 0) {
+      strlcpy(name, entry->d_name, sizeof(name));
+      name[sep - entry->d_name] = '\0';
+      gen = strtoull(sep+1, NULL, 10);
+      if(gen > max_gen_count) max_gen_count = gen;
+
+      if(S_ISDIR(sb.st_mode)) {
+        noitL(noit_debug, "<DIR< %s\n", entry->d_name);
+        child = xmlNewNode(NULL, (xmlChar *)name);
+        noit_conf_read_into_node(child, filepath);
+        udata = calloc(1, sizeof(*udata));
+        udata->name = strdup(name);
+        udata->path = strdup(filepath);
+        child->_private = udata;
+        xmlAddChild(node, child);
+      }
+      else if(S_ISREG(sb.st_mode)) {
+        xmlDocPtr cdoc;
+        xmlNodePtr cnode = NULL;
+        noitL(noit_debug, "<FILE< %s\n", entry->d_name);
+        cdoc = xmlParseFile(filepath);
+        if(cdoc) {
+          cnode = xmlDocGetRootElement(cdoc);
+          xmlDocSetRootElement(cdoc, xmlNewNode(NULL, (xmlChar *)"dummy"));
+          if(cnode) {
+            udata = calloc(1, sizeof(*udata));
+            udata->name = strdup(name);
+            udata->path = strdup(filepath);
+            cnode->_private = udata;
+            xmlAddChild(node, cnode);
+          }
+          xmlFreeDoc(cdoc);
+        }
+      }
+    }
+  }
+  closedir(dirroot);
+  return 0;
+}
+
+xmlDocPtr
+noit_conf_read_backing_store(const char *path) {
+  xmlDocPtr doc;
+  xmlNodePtr root;
+  noit_xml_userdata_t *what;
+
+  doc = xmlNewDoc((xmlChar *)"1.0");
+  what = calloc(1, sizeof(*what));
+  what->path = strdup(path);
+  doc->_private = what;
+  root = xmlNewNode(NULL, (xmlChar *)"stub");
+  xmlDocSetRootElement(doc, root);
+  noit_conf_read_into_node(root, path);
+  return doc;
+}
-int
-noit_conf_magic_mix(const char *parentfile, xmlDocPtr doc) {
@@ -280 +587 @@
-
-  assert(config_include_cnt == -1);
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-  config_include_cnt = 0;
-  mix_ctxt = xmlXPathNewContext(doc);
-  pobj = xmlXPathEval((xmlChar *)"//include[@file]", mix_ctxt);
-  if(!pobj) goto out;
@@ -826 +1197 @@
-  }
-  noit_conf_kansas_city_shuffle_undo(master_config);
+  noit_conf_shatter_write(master_config);
-  len = xmlSaveFormatFileTo(out, master_config, "utf8", 1);
+  noit_conf_shatter_postwrite(master_config);
-  noit_conf_kansas_city_shuffle_redo(master_config);
-  close(fd);
@@ -1191 +1564 @@
-  if(delete) {
-    node = (noit_conf_section_t)xmlXPathNodeSetItem(pobj->nodesetval, 0);
+    CONF_REMOVE(node);
-    xmlUnlinkNode(node);
-    noit_conf_mark_changed();

src/noit_conf.h

@@ -130 +130 @@
-API_EXPORT(int) noit_conf_log_init_rotate(const char *, noit_boolean);
-
+API_EXPORT(void) noit_conf_backingstore_remove(noit_conf_section_t node);
+API_EXPORT(void) noit_conf_backingstore_dirty(noit_conf_section_t node);
+
+#define CONF_REMOVE(n) do { \
+  noit_conf_backingstore_remove(n); \
+} while(0)
+
+#define CONF_DIRTY(n) do { \
+ noit_conf_backingstore_dirty(n); \
+ noit_conf_mark_changed(); \
+} while(0)
+
-#define EXPOSE_CHECKER(name) \
-  API_EXPORT(pcre *) noit_conf_get_valid_##name##_checker()

src/noit_conf_checks.c

@@ -173 +173 @@
-    if(val)
-      xmlSetProp(node, (xmlChar *)attrinfo->name, (xmlChar *)val);
+    CONF_DIRTY(node);
-    noit_conf_mark_changed();
-  }
@@ -212 +213 @@
-    }
-    else {
+      CONF_DIRTY(newnode);
-      noit_conf_mark_changed();
-      rv = 0;
@@ -586 +588 @@
-        for(j=1;j<argc;j++)
-          xmlUnsetProp(node, (xmlChar *)argv[j]);
+        CONF_DIRTY(node);
-      } else {
-        nc_printf(ncct, "descheduling %s\n", uuid_conf);
-        noit_poller_deschedule(checkid);
+        CONF_REMOVE(node);
-        xmlUnlinkNode(node);
-      }
@@ -904 +908 @@
-    xmlNodePtr toremove;
-    toremove = xmlXPathNodeSetItem(pobj->nodesetval, 0);
+    CONF_REMOVE(toremove);
-    xmlUnlinkNode(toremove);
-  }
@@ -934 +939 @@
-    /* Now we create a child */
-    xmlNewChild(confignode, NULL, (xmlChar *)name, (xmlChar *)value);
+    CONF_DIRTY(confignode);
-  }
-  noit_conf_mark_changed();
@@ -1006 +1012 @@
-  if(value)
-    xmlSetProp(node, (xmlChar *)attrinfo->name, (xmlChar *)value);
+  CONF_DIRTY(node);
-  noit_conf_mark_changed();
-  rv = 0;

src/noit_filters_rest.c

@@ -148 +148 @@
-  if(!node) goto not_found;
-  if(noit_filter_remove(node) == 0) goto not_found;
+  CONF_REMOVE(node);
-  xmlUnlinkNode(node);
-  xmlFreeNode(node);

src/noit_xml.c

@@ -31 +31 @@
- */
-
+#include "noit_defines.h"
-#include "noit_xml.h"
+#include "utils/noit_log.h"
+#include "utils/noit_mkdir.h"
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
-#include <assert.h>
-
@@ -85 +91 @@
-}
-
+int
+noit_xmlSaveToFile(xmlDocPtr doc, const char *filename) {
+  int rv = -1, fd, have_backup = 0, once;
+  char tmpfile[PATH_MAX], bakfile[PATH_MAX];
+  xmlOutputBufferPtr out;
+  xmlCharEncodingHandlerPtr enc;
+
+  snprintf(tmpfile, sizeof(tmpfile), "%s.tmp", filename);
+  snprintf(bakfile, sizeof(bakfile), "%s.bak", filename);
+  once = 1;
+ mkbkup:
+  if(link(filename, bakfile) == 0) {
+    unlink(filename);
+    have_backup = 1;
+  }
+  else if (errno != ENOENT) {
+    if(!once) return -1;
+    once = 0;
+    if(errno == EEXIST) {
+      unlink(bakfile);
+      goto mkbkup;
+    }
+    noitL(noit_error, "Cannot create backup for %s: %s\n",
+          filename, strerror(errno));
+    return -1;
+  }
+  else {
+    noitL(noit_debug, "link(%s,%s) -> %s\n", filename, bakfile, strerror(errno));
+  }
+
+ retry:
+  fd = open(tmpfile, O_WRONLY | O_CREAT | O_EXCL, 0640);
+  if(fd < 0) {
+    if(!once) return -1;
+    once = 0;
+    if(errno == ENOENT) {
+      if(mkdir_for_file(tmpfile, 0750) == 0) goto retry;
+    }
+    if(errno == EEXIST) {
+      unlink(tmpfile);
+      goto retry;
+    }
+    return -1;
+  }
+  enc = xmlGetCharEncodingHandler(XML_CHAR_ENCODING_UTF8);
+  out = xmlOutputBufferCreateFd(fd, enc);
+  assert(out);
+  xmlSaveFormatFileTo(out, doc, "utf8", 1);
+  close(fd);
+  rv = 0;
+  if(link(tmpfile, filename)) {
+    rv = -1;
+    if(link(bakfile, filename) != 0) have_backup = 0;
+  }
+  unlink(tmpfile);
+  if(have_backup)
+    unlink(bakfile);
+  return rv;
+}

src/noit_xml.h

@@ -40 +40 @@
-  noit_xmlSaveToBuffer(xmlDocPtr doc);
-
+API_EXPORT(int)
+  noit_xmlSaveToFile(xmlDocPtr doc, const char *filename);
+
-#endif

src/stratcon_jlog_streamer.c

@@ -1463 +1463 @@
-      pthread_mutex_unlock(&noit_ip_by_cn_lock);
-    }
+    CONF_REMOVE(noit_configs[i]);
-    xmlUnlinkNode(noit_configs[i]);
-    xmlFreeNode(noit_configs[i]);