Changeset 3081176b656a255da1e587a44933eabc58ee9119

Timestamp:

07/01/11 08:00:58 (2 years ago)

Author:

Dan Di Spaltro <dan@cloudkick.com>

git-committer:

Dan Di Spaltro <dan@cloudkick.com> 1309507258 -0700

git-parent:

[5079a1f4e83875fdd95077bb5fae8079eb8b4bd9]

git-author:

Dan Di Spaltro <dan@cloudkick.com> 1309507258 -0700

Message:

Add autodetection of the libprotobuf-c library
Add the generated bundle file
use preprocessor macros to compile out support of protobufs.
Add 'B' metrics with the discussed format over IRC, enough information to route,
while preserving the cross language decoding.

Files:

• configure.in (modified) (1 diff)
• src/Makefile.in (modified) (2 diffs)
• src/bundle.pb-c.c (added)
• src/bundle.pb-c.h (added)
• src/bundle.proto (added)
• src/noit_check.c (modified) (1 diff)
• src/noit_check.h (modified) (1 diff)
• src/noit_check_log.c (modified) (4 diffs)
• src/noit_config.h.in (modified) (1 diff)

configure.in

@@ -204 +204 @@
-)
-
+
+old_LIBS="$LIBS"
+#LIBS="$LIBS `pkg-config --libs thrift_c_glib`"
+LIBS="$LIBS `pkg-config --libs libprotobuf-c`"
+AC_CHECK_LIB(protobuf-c, protobuf_c_message_get_packed_size,
+        [
+                AC_DEFINE([HAVE_PROTOBUFS])
+        ]
+        [
+    NOIT_OBJS_EXTRA="$NOIT_OBJS_EXTRA bundle.pb-c.o"
+                CFLAGS="$CFLAGS `pkg-config --cflags libprotobuf-c`"
+        ],
+        [
+                AC_MSG_WARN([protobuf-c not found, skipping bundled metrics])
+                LIBS=$old_LIBS
+        ]
+)
+AC_SUBST(NOIT_OBJS_EXTRA)
+
-old_LIBS=$LIBS
-old_CFLAGS=$CFLAGS

src/Makefile.in

@@ -72 +72 @@
-        noit_capabilities_listener.o noit_xml.o \
-        noit_jlog_listener.o noit_livestream_listener.o noit_filters.o \
-
+ @NOIT_OBJS_EXTRA@
-
-NOIT_LIBS=libudns libnoitedit libeventer libjlog libnoit_utils libnoitjson
@@ -146 +146 @@
-
-man/stratcond.usage.h:  make-man
+
-
-noitd.o:        man/noitd.usage.h

src/noit_check.c

@@ -1126 +1126 @@
-  }
-
+  /* Write out the bundled information */
+  noit_check_log_bundle(check);
-  /* Write out our status */
-  noit_check_log_status(check);

src/noit_check.h

@@ -267 +267 @@
-API_EXPORT(void) noit_check_log_status(noit_check_t *check);
-API_EXPORT(void) noit_check_log_delete(noit_check_t *check);
+API_EXPORT(void) noit_check_log_bundle(noit_check_t *check);
-API_EXPORT(void) noit_check_log_metrics(noit_check_t *check);
-API_EXPORT(void) noit_check_log_metric(noit_check_t *check,

src/noit_check_log.c

@@ -41 +41 @@
-#include "jlog/jlog.h"
-
+#ifdef HAVE_PROTOBUFS
+#include "bundle.pb-c.h"
+#include <zlib.h>
+#include "utils/noit_b64.h"
+#endif
+
-/* Log format is tab delimited:
- * NOIT CONFIG (implemented in noit_conf.c):
@@ -56 +62 @@
- * METRICS:
- *  'M' TIMESTAMP UUID NAME TYPE VALUE
+ *
+ * BUNDLE
+ *  'B' TIMESTAMP UUID TARGET MODULE NAME strlen(base64(gzipped(payload))) base64(gzipped(payload))
+ *  
- */
-
@@ -62 +72 @@
-static noit_log_stream_t metrics_log = NULL;
-static noit_log_stream_t delete_log = NULL;
+static noit_log_stream_t bundle_log = NULL;
+
+#ifdef HAVE_PROTOBUFS
+static bool use_compression = true;
+#endif
+
-#define SECPART(a) ((unsigned long)(a)->tv_sec)
-#define MSECPART(a) ((unsigned long)((a)->tv_usec / 1000))
@@ -274 +290 @@
-}
-
+
+#ifdef HAVE_PROTOBUFS
+static int
+_noit_check_log_bundle_metric(noit_log_stream_t ls, Metric *metric, metric_t *m) {
+  metric->metrictype = (int)m->metric_type;
+
+  metric->name = m->metric_name;
+  switch (m->metric_type) {
+    case METRIC_INT32:
+      metric->has_valuei32 = true;
+      metric->valuei32 = *(m->metric_value.i); break;
+    case METRIC_UINT32:
+      metric->has_valueui32 = true;
+      metric->valueui32 = *(m->metric_value.I); break;
+    case METRIC_INT64:
+      metric->has_valuei64 = true;
+      metric->valuei64 = *(m->metric_value.l); break;
+    case METRIC_UINT64:
+      metric->has_valueui64 = true;
+      metric->valueui64 = *(m->metric_value.L); break;
+    case METRIC_DOUBLE:
+      metric->has_valuedbl = true;
+      metric->valuedbl = *(m->metric_value.n); break;
+    case METRIC_STRING:
+      metric->valuestr = m->metric_value.s; break;
+    default:
+      return -1;
+  }
+  return 0;
+}
+
+static int
+_noit_check_compress_b64(char *buf_in,
+                         uLong len_in,
+                         char ** buf_out,
+                         uLong * len_out) {
+  uLong initial_dlen, dlen;
+  char *compbuff, *b64buff;
+
+  // Compress saves 25% of space (ex 470 -> 330)
+  if (use_compression) {
+    /* Compress */
+    initial_dlen = dlen = compressBound(len_in);
+    compbuff = malloc(initial_dlen);
+    if(!compbuff) return -1;
+    if(Z_OK != compress2((Bytef *)compbuff, &dlen,
+                         (Bytef *)buf_in, len_in, 9)) {
+      noitL(noit_error, "Error compressing bundled metrics.\n");
+      free(compbuff);
+      return -1;
+    }
+  } else {
+    // Or don't
+    dlen = len_in;
+    compbuff = buf_in;
+  }
+
+  /* Encode */
+  // Problems with the calculation?
+  initial_dlen = ((dlen + 2) / 3 * 4);
+  b64buff = malloc(initial_dlen);
+  if (!b64buff) {
+    if (use_compression) {
+      // Free only if we malloc'd
+      free(compbuff);
+    }
+    return -1;
+  }
+  dlen = noit_b64_encode((unsigned char *)compbuff, dlen,
+                         (char *)b64buff, initial_dlen);
+  if (use_compression) {
+    // Free only if we malloc'd
+    free(compbuff);
+  }
+  if(dlen == 0) {
+    noitL(noit_error, "Error base64'ing bundled metrics.\n");
+    free(b64buff);
+    return -1;
+  }
+  *buf_out = b64buff;
+  *len_out = dlen;
+  return 0;
+}
+
+
+static int
+_noit_check_log_bundle_serialize(noit_log_stream_t ls, noit_check_t *check) {
+  int rv = 0;
+  char uuid_str[256*3+37];
+  noit_hash_iter iter = NOIT_HASH_ITER_ZERO;
+  noit_hash_iter iter2 = NOIT_HASH_ITER_ZERO;
+  const char *key;
+  int klen, i=0, size, j;
+  uLong out_size;
+  stats_t *c;
+  void *vm;
+  char *buf, *out_buf;
+  Bundle bundle = BUNDLE__INIT;
+  SETUP_LOG(bundle, );
+  MAKE_CHECK_UUID_STR(uuid_str, sizeof(uuid_str), metrics_log, check);
+
+  // Get a bundle
+  c = &check->stats.current;
+
+  // Set attributes
+  bundle.available = c->available;
+  bundle.state = c->state;
+  bundle.duration = c->duration;
+  bundle.status = c->status;
+  
+
+  // Just count
+  while(noit_hash_next(&c->metrics, &iter, &key, &klen, &vm)) {
+    bundle.n_metrics++;
+  }
+
+  bundle.metrics = malloc(bundle.n_metrics * sizeof(Metric*));
+
+  // Now convert
+  while(noit_hash_next(&c->metrics, &iter2, &key, &klen, &vm)) {
+    /* If we apply the filter set and it returns false, we don't log */
+    metric_t *m = (metric_t *)vm;
+    bundle.metrics[i] = malloc(sizeof(Metric));
+    metric__init(bundle.metrics[i]);
+    _noit_check_log_bundle_metric(ls, bundle.metrics[i], m);
+    i++;
+  }
+
+  size = bundle__get_packed_size(&bundle);
+  buf = malloc(size);
+  bundle__pack(&bundle, (uint8_t*)buf);
+
+  // Compress + B64
+  _noit_check_compress_b64(buf, size, &out_buf, &out_size);
+  noit_log(ls, &(c->whence), __FILE__, __LINE__,
+                "B\t%lu.%03lu\t%s\t%s\t%s\t%s\t%.*s\n", SECPART(&(c->whence)), MSECPART(&(c->whence)),
+                     uuid_str, check->name, check->module, check->target,
+                     (unsigned int)out_size, out_buf);
+
+  free(buf);
+  free(out_buf);
+  // Free all the resources
+  for (j=0; j<i; j++) {
+    free(bundle.metrics[j]);
+  }
+  free(bundle.metrics);
+  return rv;
+}
+#endif
+
+static int
+_noit_check_log_bundle(noit_log_stream_t ls, noit_check_t *check) {
+#ifdef HAVE_PROTOBUFS
+  _noit_check_log_bundle_serialize(bundle_log, check);
+#else 
+  noitL(noit_error, "Trying to use bundled metrics without libprotobuf-c, recompile with support for libprotobuf-c to use bundled metrics.\n");
+  abort();
+#endif
+}
+
+void
+noit_check_log_bundle(noit_check_t *check) {
+  handle_extra_feeds(check, _noit_check_log_bundle);
+  if(!(check->flags & (NP_TRANSIENT | NP_SUPPRESS_STATUS | NP_SUPPRESS_METRICS))) {
+    SETUP_LOG(bundle, return);
+    _noit_check_log_bundle(bundle_log, check);
+  }
+}
+
-void
-noit_check_log_metric(noit_check_t *check, struct timeval *whence,

src/noit_config.h.in

@@ -87 +87 @@
-#define IFS_CH '/'
-#undef WORKING_SEM_INIT
+#undef HAVE_PROTOBUFS
-
-#undef HAVE_GETPWNAM_R