[Reconnoiter-devel] [reconnoiter commit] Reconnoiter branch, master, updated. a23f396910835ccd3d57d8f12cb36126660012e2

git at labs.omniti.com git at labs.omniti.com
Wed Nov 9 08:30:56 EST 2011


Pushed by: jesus
The branch, master has been updated
       via  a23f396910835ccd3d57d8f12cb36126660012e2 (commit)
       via  aed22aaeceba5be6d574a0721d482578a3e4e74f (commit)
       via  61903febed2d9becbc4659f7d3cde0e19ae87d14 (commit)
       via  23e807a144fa1f89d334518ea4a32eacfe154008 (commit)
       via  201e9d3d678bc8669cead70a8c52df2da7aa0585 (commit)
       via  09fef40cac145611c80b1048abd0fbcbd2a6cd70 (commit)
       via  f5982323e614fec6b6ca1900cedb708f89020355 (commit)
      from  fbf2b3181e1cc48410fe39bb0f0c291d73095a8f (commit)

Summary of changes:
 docs/config/modules/noit.module.http.xml |   28 ++++++++++++++++++++++++++++
 src/modules-lua/noit/HttpClient.lua      |   25 ++++++++++++++++++-------
 src/modules-lua/noit/module/http.lua     |   13 ++++++++++---
 3 files changed, 56 insertions(+), 10 deletions(-)

Log:
commit a23f396910835ccd3d57d8f12cb36126660012e2
Author: Theo Schlossnagle <jesus at omniti.com>
Date:   Wed Nov 9 08:26:10 2011 -0500

    merge in docs for read_limit

diff --git a/docs/config/modules/noit.module.http.xml b/docs/config/modules/noit.module.http.xml
index c3733e7..54720ae 100644
--- a/docs/config/modules/noit.module.http.xml
+++ b/docs/config/modules/noit.module.http.xml
@@ -425,6 +425,34 @@
         </listitem>
       </varlistentry>
     </variablelist>
+    <variablelist>
+      <varlistentry>
+        <term>read_limit</term>
+        <listitem>
+          <variablelist>
+            <varlistentry>
+              <term>required</term>
+              <listitem>
+                <para>optional</para>
+              </listitem>
+            </varlistentry>
+            <varlistentry>
+              <term>default</term>
+              <listitem>
+                <para>0</para>
+              </listitem>
+            </varlistentry>
+            <varlistentry>
+              <term>allowed</term>
+              <listitem>
+                <para>\d+</para>
+              </listitem>
+            </varlistentry>
+          </variablelist>
+          <para>Sets an approximate limit on the data read (0 means no limit).</para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
   </section>
   <example>
     <title>Checking an HTTP and HTTPS URL.</title>

commit aed22aaeceba5be6d574a0721d482578a3e4e74f
Merge: fbf2b31 61903fe
Author: Theo Schlossnagle <jesus at omniti.com>
Date:   Wed Nov 9 08:24:24 2011 -0500

    Merge branch 'http_length' of https://github.com/rphillips/reconnoiter into http_length
    
    There was a bug in http chunked encoding where the limits were never
    enforced.  Also, the default was set to no limit to emulate previous
    behaviour.

diff --cc src/modules-lua/noit/HttpClient.lua
index eb76e3d,cc44863..82798bd
--- a/src/modules-lua/noit/HttpClient.lua
+++ b/src/modules-lua/noit/HttpClient.lua
@@@ -160,6 -165,12 +165,12 @@@ function te_chunked(self, content_enc_f
          local decoded = content_enc_func(str)
          self.content_bytes = self.content_bytes + string.len(decoded)
          if self.hooks.consume ~= nil then self.hooks.consume(decoded) end
+         if read_limit and read_limit > 0 then
 -          if string.len(self.content_bytes) > read_limit then
++          if self.content_bytes > read_limit then
+             self.truncated = true
+             return
+           end
+         end
          -- each chunk ('cept a 0 size one) is followed by a \r\n
          str = self.e:read("\n")
          if str ~= "\r\n" and str ~= "\n" then error("short chunked boundary read") end
diff --cc src/modules-lua/noit/module/http.lua
index 06a7575,259d949..1a9ca52
--- a/src/modules-lua/noit/module/http.lua
+++ b/src/modules-lua/noit/module/http.lua
@@@ -91,11 -91,10 +91,14 @@@ function onload(image
                 required="optional"
                 default="10000"
                 allowed="\d+">This sets the PCRE internal match limit (see pcreapi documentation).</parameter>
 +    <parameter name="include_body"
 +               required="optional"
 +               allowed="^(?:true|false|on|off)$"
 +               default="false">Include whole response body as a metric with the key 'body'.</parameter>
- 
+     <parameter name="read_limit"
+                required="optional"
 -               default="102400"
 -               allowed="\d+">Sets the limit on the data read.</parameter>
++               default="0"
++               allowed="\d+">Sets an approximate limit on the data read (0 means no limit).</parameter>
    </checkconfig>
    <examples>
      <example>
@@@ -274,7 -273,7 +277,8 @@@ function initiate(module, check
      local max_len = 80
      local pcre_match_limit = check.config.pcre_match_limit or 10000
      local redirects = check.config.redirects or 0
 -    local read_limit = tonumber(check.config.read_limit) or 102400
 +    local include_body = false
++    local read_limit = tonumber(check.config.read_limit) or 0
  
      -- expect the worst
      check.bad()

commit 61903febed2d9becbc4659f7d3cde0e19ae87d14
Author: Ryan Phillips <ryan.phillips at rackspace.com>
Date:   Mon Oct 31 14:08:24 2011 -0500

    add truncated metric and fix a bug

diff --git a/src/modules-lua/noit/HttpClient.lua b/src/modules-lua/noit/HttpClient.lua
index 008f66f..cc44863 100644
--- a/src/modules-lua/noit/HttpClient.lua
+++ b/src/modules-lua/noit/HttpClient.lua
@@ -43,6 +43,7 @@ function HttpClient:connect(target, port, ssl)
     self.e = noit.socket(target)
     self.target = target
     self.port = port
+    self.truncated = nil
     local rv, err = self.e:connect(self.target, self.port)
     if rv ~= 0 then
         return rv, err
@@ -133,6 +134,7 @@ function te_length(self, content_enc_func, read_limit)
     local len = tonumber(self.headers["content-length"])
     if read_limit and read_limit > 0 and len > read_limit then
       len = read_limit
+      self.truncated = true
     end
     repeat
         local str = self.e:read(len)
@@ -165,6 +167,7 @@ function te_chunked(self, content_enc_func, read_limit)
         if self.hooks.consume ~= nil then self.hooks.consume(decoded) end
         if read_limit and read_limit > 0 then
           if string.len(self.content_bytes) > read_limit then
+            self.truncated = true
             return
           end
         end
diff --git a/src/modules-lua/noit/module/http.lua b/src/modules-lua/noit/module/http.lua
index 47d5168..259d949 100644
--- a/src/modules-lua/noit/module/http.lua
+++ b/src/modules-lua/noit/module/http.lua
@@ -273,7 +273,7 @@ function initiate(module, check)
     local max_len = 80
     local pcre_match_limit = check.config.pcre_match_limit or 10000
     local redirects = check.config.redirects or 0
-    local read_limit = check.config.read_limit or 102400
+    local read_limit = tonumber(check.config.read_limit) or 102400
 
     -- expect the worst
     check.bad()
@@ -451,6 +451,9 @@ function initiate(module, check)
       good = true
     end
 
+    -- truncated response
+    check.metric_uint32("truncated", client.truncated and 1 or 0)
+
     -- turnaround time
     local seconds = elapsed(check, "duration", starttime, endtime)
     status = status .. ',rt=' .. seconds .. 's'

commit 23e807a144fa1f89d334518ea4a32eacfe154008
Author: Ryan Phillips <ryan.phillips at rackspace.com>
Date:   Thu Oct 27 13:46:41 2011 -0500

    check for valid read_limit

diff --git a/src/modules-lua/noit/HttpClient.lua b/src/modules-lua/noit/HttpClient.lua
index 12debab..008f66f 100644
--- a/src/modules-lua/noit/HttpClient.lua
+++ b/src/modules-lua/noit/HttpClient.lua
@@ -131,7 +131,7 @@ end
 
 function te_length(self, content_enc_func, read_limit)
     local len = tonumber(self.headers["content-length"])
-    if read_limit > 0 and len > read_limit then
+    if read_limit and read_limit > 0 and len > read_limit then
       len = read_limit
     end
     repeat
@@ -163,7 +163,7 @@ function te_chunked(self, content_enc_func, read_limit)
         local decoded = content_enc_func(str)
         self.content_bytes = self.content_bytes + string.len(decoded)
         if self.hooks.consume ~= nil then self.hooks.consume(decoded) end
-        if read_limit > 0 then
+        if read_limit and read_limit > 0 then
           if string.len(self.content_bytes) > read_limit then
             return
           end

commit 201e9d3d678bc8669cead70a8c52df2da7aa0585
Author: Ryan Phillips <ryan.phillips at rackspace.com>
Date:   Thu Oct 27 13:02:39 2011 -0500

    use 100k for the limit

diff --git a/src/modules-lua/noit/module/http.lua b/src/modules-lua/noit/module/http.lua
index 6e9a807..47d5168 100644
--- a/src/modules-lua/noit/module/http.lua
+++ b/src/modules-lua/noit/module/http.lua
@@ -273,7 +273,7 @@ function initiate(module, check)
     local max_len = 80
     local pcre_match_limit = check.config.pcre_match_limit or 10000
     local redirects = check.config.redirects or 0
-    local read_limit = check.config.read_limit or 10
+    local read_limit = check.config.read_limit or 102400
 
     -- expect the worst
     check.bad()

commit 09fef40cac145611c80b1048abd0fbcbd2a6cd70
Author: Ryan Phillips <ryan.phillips at rackspace.com>
Date:   Thu Oct 27 13:00:16 2011 -0500

    add support for check config option read_limit

diff --git a/src/modules-lua/noit/HttpClient.lua b/src/modules-lua/noit/HttpClient.lua
index 937bcc5..12debab 100644
--- a/src/modules-lua/noit/HttpClient.lua
+++ b/src/modules-lua/noit/HttpClient.lua
@@ -129,9 +129,11 @@ function te_close(self, content_enc_func)
     until str == nil or string.len(str) ~= len
 end
 
-function te_length(self, content_enc_func)
+function te_length(self, content_enc_func, read_limit)
     local len = tonumber(self.headers["content-length"])
-    len = len > 102400 and 102400 or len
+    if read_limit > 0 and len > read_limit then
+      len = read_limit
+    end
     repeat
         local str = self.e:read(len)
         if str ~= nil then
@@ -144,7 +146,7 @@ function te_length(self, content_enc_func)
     until str == nil or len == 0
 end
 
-function te_chunked(self, content_enc_func)
+function te_chunked(self, content_enc_func, read_limit)
     while true do
         local str = self.e:read("\n")
         if str == nil then error("bad chunk transfer") end
@@ -161,12 +163,14 @@ function te_chunked(self, content_enc_func)
         local decoded = content_enc_func(str)
         self.content_bytes = self.content_bytes + string.len(decoded)
         if self.hooks.consume ~= nil then self.hooks.consume(decoded) end
+        if read_limit > 0 then
+          if string.len(self.content_bytes) > read_limit then
+            return
+          end
+        end
         -- each chunk ('cept a 0 size one) is followed by a \r\n
         str = self.e:read("\n")
         if str ~= "\r\n" and str ~= "\n" then error("short chunked boundary read") end
-        if string.len(self.content_bytes) > 102400 then
-          break
-        end
     end
     -- read trailers
     while true do
@@ -176,7 +180,7 @@ function te_chunked(self, content_enc_func)
     end
 end
 
-function HttpClient:get_body()
+function HttpClient:get_body(read_limit)
     local cefunc = ce_passthru
     local ce = self.headers["content-encoding"]
     if ce ~= nil then
@@ -191,16 +195,16 @@ function HttpClient:get_body()
     local te = self.headers["transfer-encoding"]
     local cl = self.headers["content-length"]
     if te ~= nil and te == "chunked" then
-        return te_chunked(self, cefunc)
+        return te_chunked(self, cefunc, read_limit)
     elseif cl ~= nil and tonumber(cl) ~= nil then
-        return te_length(self, cefunc)
+        return te_length(self, cefunc, read_limit)
     end
     return te_close(self, cefunc)
 end
 
-function HttpClient:get_response()
+function HttpClient:get_response(read_limit)
     self:get_headers()
-    return self:get_body()
+    return self:get_body(read_limit)
 end
 
 return HttpClient
diff --git a/src/modules-lua/noit/module/http.lua b/src/modules-lua/noit/module/http.lua
index 4754fa3..6e9a807 100644
--- a/src/modules-lua/noit/module/http.lua
+++ b/src/modules-lua/noit/module/http.lua
@@ -91,6 +91,10 @@ function onload(image)
                required="optional"
                default="10000"
                allowed="\d+">This sets the PCRE internal match limit (see pcreapi documentation).</parameter>
+    <parameter name="read_limit"
+               required="optional"
+               default="102400"
+               allowed="\d+">Sets the limit on the data read.</parameter>
   </checkconfig>
   <examples>
     <example>
@@ -269,6 +273,7 @@ function initiate(module, check)
     local max_len = 80
     local pcre_match_limit = check.config.pcre_match_limit or 10000
     local redirects = check.config.redirects or 0
+    local read_limit = check.config.read_limit or 10
 
     -- expect the worst
     check.bad()
@@ -353,7 +358,7 @@ function initiate(module, check)
             headers_firstpass[k] = v
         end
         client:do_request(method, uri, headers_firstpass)
-        client:get_response()
+        client:get_response(read_limit)
         if client.code ~= 401 or
            client.headers["www-authenticate"] == nil then
             check.status("expected digest challenge, got " .. client.code)
@@ -396,7 +401,7 @@ function initiate(module, check)
             return
         end
         optclient:do_request(method, uri, headers, payload)
-        optclient:get_response()
+        optclient:get_response(read_limit)
 
         redirects = redirects - 1
         client = optclient

commit f5982323e614fec6b6ca1900cedb708f89020355
Author: Ryan Phillips <ryan.phillips at rackspace.com>
Date:   Thu Oct 27 09:50:21 2011 -0500

    add 100k limit to http read lengths

diff --git a/src/modules-lua/noit/HttpClient.lua b/src/modules-lua/noit/HttpClient.lua
index eb76e3d..937bcc5 100644
--- a/src/modules-lua/noit/HttpClient.lua
+++ b/src/modules-lua/noit/HttpClient.lua
@@ -131,6 +131,7 @@ end
 
 function te_length(self, content_enc_func)
     local len = tonumber(self.headers["content-length"])
+    len = len > 102400 and 102400 or len
     repeat
         local str = self.e:read(len)
         if str ~= nil then
@@ -163,6 +164,9 @@ function te_chunked(self, content_enc_func)
         -- each chunk ('cept a 0 size one) is followed by a \r\n
         str = self.e:read("\n")
         if str ~= "\r\n" and str ~= "\n" then error("short chunked boundary read") end
+        if string.len(self.content_bytes) > 102400 then
+          break
+        end
     end
     -- read trailers
     while true do




hooks/post-receive
-- 
Reconnoiter


More information about the Reconnoiter-devel mailing list