root/test/t/testconfig.pm

Revision 5f816fc68a68eaeb73588d831044fdaadbafe5e9, 17.3 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 3 years ago)

fixes #358

  • Property mode set to 100644
Line 
1 package testconfig;
2 use Test::More;
3 use Fcntl;
4 use DBI;
5 use Cwd;
6 use Exporter 'import';
7 use Data::Dumper;
8 use IO::File;
9 use strict;
10 use vars qw/@EXPORT/;
11
12 my $noit_pid = 0;
13 my $noit_log = undef;
14 my $stratcon_pid = 0;
15 my $stratcon_log = undef;
16
17
18 @EXPORT = qw($NOIT_TEST_DB $NOIT_TEST_DB_PORT
19              $NOIT_API_PORT $NOIT_CLI_PORT
20              $STRATCON_API_PORT $STRATCON_CLI_PORT
21              $STRATCON_WEB_PORT
22              pg make_noit_config start_noit stop_noit get_noit_log
23              make_stratcon_config start_stratcon stop_stratcon get_stratcon_log
24              $MODULES_DIR $LUA_DIR $all_noit_modules $all_stratcon_modules);
25
26 our $default_filterset = {
27   allowall => [ { type => "allow" } ],
28 };
29 our $all_noit_modules = {
30   'selfcheck' => { 'image' => 'selfcheck' },
31   'ping_icmp' => { 'image' => 'ping_icmp' },
32   'snmp' => { 'image' => 'snmp' },
33   'ssh2' => { 'image' => 'ssh2' },
34   'mysql' => { 'image' => 'mysql' },
35   'postgres' => { 'image' => 'postgres' },
36   'test_abort' => { 'image' => 'test_abort' },
37   'varnish' => { 'loader' => 'lua', 'object' => 'noit.module.varnish' },
38   'http' => { 'loader' => 'lua', 'object' => 'noit.module.http' },
39   'resmon' => { 'loader' => 'lua', 'object' => 'noit.module.resmon' },
40   'smtp' => { 'loader' => 'lua', 'object' => 'noit.module.smtp' },
41   'tcp' => { 'loader' => 'lua', 'object' => 'noit.module.tcp' },
42 };
43
44 our $NOIT_TEST_DB = "/tmp/noit-test-db";
45 our $NOIT_TEST_DB_PORT = 23816;
46 our $NOIT_API_PORT = 42364;
47 our $NOIT_CLI_PORT = 42365;
48 our $STRATCON_API_PORT = 42366;
49 our $STRATCON_CLI_PORT = 42367;
50 our $STRATCON_WEB_PORT = 42368;
51
52 our ($MODULES_DIR, $LUA_DIR);
53
54 sub pg {
55   my $db = shift || 'postgres';
56   my $user = shift || $ENV{USER};
57   return DBI->connect(
58     "dbi:Pg:host=localhost;port=$NOIT_TEST_DB_PORT;database=$db", $user, '',
59     { 'PrintError' => 0 }
60   );
61 }
62
63 sub make_eventer_config {
64   my ($o, $opts) = @_;
65   my $cwd = $opts->{cwd};
66   $opts->{eventer_config}->{default_queue_threads} ||= 10;
67   $opts->{eventer_config}->{default_ca_chain} ||= "$cwd/../test-ca.crt";
68   print $o qq{
69   <eventer>
70     <config>
71       <default_queue_threads>$opts->{eventer_config}->{default_queue_threads}</default_queue_threads>
72       <default_ca_chain>$opts->{eventer_config}->{default_ca_chain}</default_ca_chain>
73     </config>
74   </eventer>
75 };
76 }
77 sub make_rest_acls {
78   my ($o, $opts) = @_;
79   my $acls = $opts->{rest_acls};
80   print $o qq{  <rest>\n};
81   foreach my $acl (@$acls) {
82     print $o qq^    <acl^;
83     print $o qq^ type="$acl->{type}"^ if exists($acl->{type});
84     print $o qq^ cn="$acl->{cn}"^ if exists($acl->{cn});
85     print $o qq^ url="$acl->{url}"^ if exists($acl->{url});
86     print $o qq^>\n^;
87     my $rules = $acl->{rules};
88     foreach my $rule (@$rules) {
89       print $o qq^      <rule^;
90       print $o qq^ type="$rule->{type}"^ if exists($rule->{type});
91       print $o qq^ cn="$rule->{cn}"^ if exists($rule->{cn});
92       print $o qq^ url="$rule->{url}"^ if exists($rule->{url});
93       print $o qq^/>\n^;
94     }
95     print $o qq^    </acl>\n^;
96   }
97   print $o qq{  </rest>\n};
98 }
99 sub make_log_section {
100   my ($o, $type, $dis) = @_;
101   print $o qq{      <$type>
102         <outlet name="$type"/>
103 };
104   while (my ($t, $d) = each %$dis) {
105     next unless length($t);
106     print $o qq{        <log name="$type/$t" disabled="$d"/>\n};
107   }
108   print $o qq{      </$type>\n};
109 }
110 sub make_logs_config {
111   my ($o, $opts) = @_;
112   my $cwd = $opts->{cwd};
113   my @logtypes = qw/collectd dns eventer external lua mysql ping_icmp postgres
114                     selfcheck snmp ssh2/;
115   # These are disabled attrs, so they look backwards
116   if(!exists($opts->{logs_error})) {
117     $opts->{logs_error}->{''} ||= 'false';
118   }
119   if(!exists($opts->{logs_debug})) {
120     $opts->{logs_debug}->{''} ||= 'true';
121   }
122   foreach(@logtypes) {
123     $opts->{logs_error}->{$_} ||= 'false';
124     $opts->{logs_debug}->{$_} ||= 'false';
125   }
126  
127   print $o qq{
128   <logs>
129     <console_output>
130       <outlet name="stderr"/>
131       <log name="error" disabled="$opts->{logs_error}->{''}" timestamps="true"/>
132       <log name="debug" disabled="$opts->{logs_debug}->{''}" timestamps="true"/>
133     </console_output>
134     <feeds>
135       <log name="feed" type="jlog" path="$cwd/logs/$opts->{name}.feed(stratcon)"/>
136     </feeds>
137     <components>
138 };
139   make_log_section($o, 'error', $opts->{logs_error});
140   make_log_section($o, 'debug', $opts->{logs_debug});
141   print $o qq{
142     </components>
143     <feeds>
144       <config><extended_id>on</extended_id></config>
145       <outlet name="feed"/>
146       <log name="check">
147         <outlet name="error"/>
148       </log>
149       <log name="status"/>
150       <log name="metrics"/>
151       <log name="config"/>
152     </feeds>
153   </logs>
154 };
155 }
156 sub make_modules_config {
157   my ($o, $opts) = @_;
158   my $cwd = $opts->{cwd};
159   print $o qq{
160   <modules directory="$cwd/../../src/modules">
161     <loader image="lua" name="lua">
162       <config><directory>$cwd/../../src/modules-lua/?.lua</directory></config>
163     </loader>
164 };
165   foreach(keys %{$opts->{generics}}) {
166     print $o qq{    <generic };
167     print $o qq{ image="$opts->{generics}->{$_}->{image}"}
168       if(exists($opts->{generics}->{$_}->{image}));
169     print $o qq{ name="$_"/>\n};
170   }
171   foreach(keys %{$opts->{modules}}) {
172     print $o qq{    <module };
173     print $o qq{ image="$opts->{modules}->{$_}->{image}"}
174       if(exists($opts->{modules}->{$_}->{image}));
175     print $o qq{ loader="$opts->{modules}->{$_}->{loader}"}
176       if(exists($opts->{modules}->{$_}->{loader}));
177     print $o qq{ object="$opts->{modules}->{$_}->{object}"}
178       if(exists($opts->{modules}->{$_}->{object}));
179     print $o qq{ name="$_"/>\n};
180   }
181   print $o qq{</modules>\n};
182 }
183 sub make_noit_listeners_config {
184   my ($o, $opts) = @_;
185   my $cwd = $opts->{cwd};
186   $opts->{noit_api_port} ||= $NOIT_API_PORT;
187   $opts->{noit_cli_port} ||= $NOIT_CLI_PORT;
188   print $o qq{
189   <listeners>
190     <sslconfig>
191       <optional_no_ca>false</optional_no_ca>
192       <certificate_file>$cwd/../test-noit.crt</certificate_file>
193       <key_file>$cwd/../test-noit.key</key_file>
194       <ca_chain>$cwd/../test-ca.crt</ca_chain>
195       <crl>$cwd/../test-ca.crl</crl>
196     </sslconfig>
197     <consoles type="noit_console">
198       <listener address="*" port="$opts->{noit_cli_port}">
199         <config>
200           <line_protocol>telnet</line_protocol>
201         </config>
202       </listener>
203     </consoles>
204     <listener type="control_dispatch" address="*" port="$opts->{noit_api_port}" ssl="on">
205       <config>
206         <log_transit_feed_name>feed</log_transit_feed_name>
207       </config>
208     </listener>
209   </listeners>
210 };
211 }
212 sub do_check_print {
213   my $o = shift;
214   my $list = shift;
215   return unless $list;
216   foreach my $node (@$list) {
217     print $o qq{<$node->[0]};
218     while(my ($k, $v) = each %{$node->[1]}) {
219       print $o qq{ $k="$v"};
220     }
221     if($node->[2]) {
222       print $o qq{>\n};
223       do_check_print($o, $node->[2]);
224       print $o qq{</check>\n};
225     }
226     else {
227       print $o qq{/>\n};
228     }
229   }
230 }
231 sub make_checks_config {
232   my ($o, $opts) = @_;
233   my $cwd = $opts->{cwd};
234   print $o qq{  <checks max_initial_stutter="10" filterset="default">\n};
235   do_check_print($o, $opts->{checks});
236   print $o qq{  </checks>\n};
237 }
238 sub make_filtersets_config {
239   my ($o, $opts) = @_;
240   my $cwd = $opts->{cwd};
241   print $o qq{<filtersets>\n};
242   while (my ($name, $set) = each %{$opts->{filtersets}}) {
243     print $o qq{  <filterset name="$name">\n};
244     foreach my $rule (@$set) {
245       print $o qq{    <rule };
246       while(my ($k,$v) = each %$rule) {
247         print $o qq{ $k="$v"};
248       }
249       print $o qq{/>\n};
250     }
251     print $o qq{  </filterset>\n};
252   }
253   print $o qq{</filtersets>\n};
254 }
255
256 sub make_noit_config {
257   my $name = shift;
258   my $options = shift;
259   $options->{cwd} ||= cwd();
260   $options->{modules} = $all_noit_modules unless exists($options->{modules});
261   $options->{filtersets} = $default_filterset unless exists($options->{filtersets});
262   $options->{rest_acls} ||= [ { type => 'deny', rules => [ { type => 'allow' } ] } ];
263   my $cwd = $options->{cwd};
264   my $file = "$cwd/logs/${name}_noit.conf";
265   open (my $o, ">$file") || BAIL_OUT("can't write config: $file");
266   print $o qq{<?xml version="1.0" encoding="utf8" standalone="yes"?>\n};
267   print $o qq{<noit>};
268   make_eventer_config($o, $options);
269   make_rest_acls($o, $options);
270   make_logs_config($o, $options);
271   make_modules_config($o, $options);
272   make_noit_listeners_config($o, $options);
273   make_checks_config($o, $options);
274   make_filtersets_config($o, $options);
275   print $o qq{</noit>\n};
276   close($o);
277   return $file;
278 }
279
280 sub make_stratcon_noits_config {
281   my ($o, $opts) = @_;
282   my $cwd = $opts->{cwd};
283   $opts->{noit_api_port} ||= $NOIT_API_PORT;
284   print $o qq{
285   <noits>
286     <sslconfig>
287       <certificate_file>$cwd/../test-stratcon.crt</certificate_file>
288       <key_file>$cwd/../test-stratcon.key</key_file>
289       <ca_chain>$cwd/../test-ca.crt</ca_chain>
290     </sslconfig>
291     <config>
292       <reconnect_initial_interval>1000</reconnect_initial_interval>
293       <reconnect_maximum_interval>15000</reconnect_maximum_interval>
294     </config>
295 };
296   foreach my $n (@{$opts->{noits}}) {
297     print $o qq{    <noit};
298     while (my ($k,$v) = each %$n) {
299       print $o qq{ $k=\"$v\"};
300     }
301     print $o qq{/>\n};
302   }
303   print $o qq{</noits>\n};
304 }
305
306 sub make_stratcon_listeners_config {
307   my ($o, $opts) = @_;
308   my $cwd = $opts->{cwd};
309   $opts->{stratcon_api_port} ||= $STRATCON_API_PORT;
310   $opts->{stratcon_web_port} ||= $STRATCON_WEB_PORT;
311   print $o qq{
312   <listeners>
313     <sslconfig>
314       <certificate_file>$cwd/../test-stratcon.crt</certificate_file>
315       <key_file>$cwd/../test-stratcon.key</key_file>
316       <ca_chain>$cwd/../test-ca.crt</ca_chain>
317     </sslconfig>
318     <realtime type="http_rest_api">
319       <listener address="*" port="$opts->{stratcon_web_port}">
320         <config>
321           <hostname>stratcon.noit.example.com</hostname>
322           <document_domain>noit.example.com</document_domain>
323         </config>
324       </listener>
325     </realtime>
326     <listener type="control_dispatch" address="*" port="$opts->{stratcon_api_port}" ssl="on" />
327   </listeners>
328 };
329 }
330
331 sub make_iep_config {
332   my ($o, $opts) = @_;
333   my $cwd = $opts->{cwd};
334   $opts->{iep}->{disabled} ||= 'false';
335   mkdir("$cwd/logs/$opts->{name}_iep_root");
336   open(my $run, "<$cwd/../../src/java/run-iep.sh") ||
337     BAIL_OUT("cannot open source run-iep.sh");
338   sysopen(my $newrun, "$cwd/logs/$opts->{name}_iep_root/run-iep.sh", O_WRONLY|O_CREAT, 0755) ||
339     BAIL_OUT("cannot open target run-iep.sh");
340   while(<$run>) {
341     s%^DIRS="%DIRS="$cwd/../../src/java $cwd/../../src/java/lib %;
342     print $newrun $_;
343   }
344   close($run);
345   close($newrun);
346   print $o qq{
347   <iep disabled="$opts->{iep}->{disabled}">
348     <start directory="$cwd/logs/$opts->{name}_iep_root"
349            command="$cwd/logs/$opts->{name}_iep_root/run-iep.sh" />
350 };
351   foreach my $mqt (keys %{$opts->{iep}->{mq}}) {
352     print $o qq{    <mq type="$mqt">\n};
353     while (my ($k,$v) = each %{$opts->{iep}->{mq}->{mqt}}) {
354       print $o qq{      <$k>$v</$k>\n};
355     }
356     print $o qq{    </mq>\n};
357   }
358   foreach my $bt (keys %{$opts->{iep}->{broker}}) {
359     print $o qq{    <broker adapter="$bt">\n};
360     while (my ($k,$v) = each %{$opts->{iep}->{broker}->{bt}}) {
361       print $o qq{      <$k>$v</$k>\n};
362     }
363     print $o qq{    </broker>\n};
364   }
365   print $o qq{    <queries master="iep">\n};
366   foreach my $s (@{$opts->{iep}->{statements}}) {
367     print $o qq{        <statement id="$s->{id}" provides="$s->{id}">\n};
368     print $o qq{            <requires>$s->{requires}</requires>\n} if $s->{requires};
369     print $o qq{            <epl><![CDATA[$s->{epl}]]></epl>\n};
370     print $o qq{        </statement>\n};
371   }
372   foreach my $s (@{$opts->{iep}->{queries}}) {
373     print $o qq{        <query id="$s->{id}" topic="$s->{topic}">\n};
374     print $o qq{            <epl><![CDATA[$s->{epl}]]></epl>\n};
375     print $o qq{        </query>\n};
376   }
377   print $o qq{    </queries>\n};
378   print $o qq{</iep>\n};
379 }
380 sub make_database_config {
381   my ($o, $opts) = @_;
382   my $cwd = $opts->{cwd};
383   print $o qq{
384   <database>
385     <journal>
386       <path>$cwd/logs/$opts->{name}_stratcon.persist</path>
387     </journal>
388     <dbconfig>
389       <host>localhost</host>
390       <port>$NOIT_TEST_DB_PORT</port>
391       <dbname>reconnoiter</dbname>
392       <user>stratcon</user>
393       <password>stratcon</password>
394     </dbconfig>
395     <statements>
396       <allchecks><![CDATA[
397         SELECT remote_address, id, target, module, name
398           FROM check_currently
399       ]]></allchecks>
400       <findcheck><![CDATA[
401         SELECT remote_address, id, target, module, name
402           FROM check_currently
403          WHERE sid = \$1
404       ]]></findcheck>
405       <allstoragenodes><![CDATA[
406         SELECT storage_node_id, fqdn, dsn
407           FROM stratcon.storage_node
408       ]]></allstoragenodes>
409       <findstoragenode><![CDATA[
410         SELECT fqdn, dsn
411           FROM stratcon.storage_node
412          WHERE storage_node_id = \$1
413       ]]></findstoragenode>
414       <mapallchecks><![CDATA[
415         SELECT id, sid, noit as remote_cn, storage_node_id, fqdn, dsn
416           FROM stratcon.map_uuid_to_sid LEFT JOIN stratcon.storage_node USING (storage_node_id)
417       ]]></mapallchecks>
418       <mapchecktostoragenode><![CDATA[
419         SELECT o_storage_node_id as storage_node_id, o_sid as sid,
420                o_fqdn as fqdn, o_dsn as dsn
421           FROM stratcon.map_uuid_to_sid(\$1,\$2)
422       ]]></mapchecktostoragenode>
423       <check><![CDATA[
424         INSERT INTO check_archive_%Y%m%d
425                     (remote_address, whence, sid, id, target, module, name)
426              VALUES (\$1, 'epoch'::timestamptz + (\$2 || ' seconds')::interval,
427                      \$3, \$4, \$5, \$6, \$7)
428       ]]></check>
429       <status><![CDATA[
430         INSERT INTO check_status_archive_%Y%m%d
431                     (whence, sid, state, availability, duration, status)
432              VALUES ('epoch'::timestamptz + (\$1 || ' seconds')::interval,
433                      \$2, \$3, \$4, \$5, \$6)
434       ]]></status>
435       <metric_numeric><![CDATA[
436         INSERT INTO metric_numeric_archive_%Y%m%d
437                     (whence, sid, name, value)
438              VALUES ('epoch'::timestamptz + (\$1 || ' seconds')::interval,
439                      \$2, \$3, \$4)
440       ]]></metric_numeric>
441       <metric_text><![CDATA[
442         INSERT INTO metric_text_archive_%Y%m%d
443                     ( whence, sid, name,value)
444              VALUES ('epoch'::timestamptz + (\$1 || ' seconds')::interval,
445                      \$2, \$3, \$4)
446       ]]></metric_text>
447       <config><![CDATA[
448         SELECT stratcon.update_config
449                (\$1, \$2, \$3,
450                 'epoch'::timestamptz + (\$4 || ' seconds')::interval,
451                 \$5)
452       ]]></config>
453       <findconfig><![CDATA[
454         SELECT config FROM stratcon.current_node_config WHERE remote_cn = \$1
455       ]]></findconfig>
456     </statements>
457   </database>
458 };
459 }
460
461 sub make_stratcon_config {
462   my $name = shift;
463   my $options = shift;
464   $options->{cwd} ||= cwd();
465   $options->{generics} ||= { 'stomp_driver' => { image => 'stomp_driver' },
466                              'postgres_ingestor' => { image => 'postgres_ingestor' } };
467   $options->{rest_acls} ||= [ { type => 'deny', rules => [ { type => 'allow' } ] } ];
468   $options->{iep}->{mq} ||= { 'stomp' => {} };
469   my $cwd = $options->{cwd};
470   my $file = "$cwd/logs/${name}_stratcon.conf";
471   open (my $o, ">$file") || BAIL_OUT("can't write config: $file");
472   print $o qq{<?xml version="1.0" encoding="utf8" standalone="yes"?>\n};
473   print $o qq{<stratcon>};
474   make_eventer_config($o, $options);
475   make_rest_acls($o, $options);
476   make_stratcon_noits_config($o, $options);
477   make_logs_config($o, $options);
478   make_modules_config($o, $options);
479   make_stratcon_listeners_config($o, $options);
480   make_database_config($o, $options);
481   make_iep_config($o, $options);
482   print $o qq{</stratcon>\n};
483   close($o);
484   return $file;
485 }
486
487 $SIG{CHLD} = sub {
488   my $pid = wait;
489   $noit_pid = 0 if($pid == $noit_pid);
490   $stratcon_pid = 0 if($pid == $stratcon_pid);
491 };
492
493 sub start_noit {
494   my $name = shift;
495   my $options = shift;
496   $options->{name} = $name;
497   return 0 if $noit_pid;
498   my $conf = make_noit_config($name, $options);
499   $noit_pid = fork();
500   mkdir "logs";
501   $noit_log = "logs/${name}_noit.log";
502   if($noit_pid == 0) {
503     $noit_pid = $$;
504     $noit_log = "logs/${name}_noit.log";
505     close(STDIN);
506     open(STDIN, "</dev/null");
507     close(STDOUT);
508     open(STDOUT, ">/dev/null");
509     close(STDERR);
510     open(STDERR, ">$noit_log");
511     my @args = ( 'noitd', '-D', '-c', $conf );
512     exec { '../../src/noitd' } @args;
513     exit(-1);
514   }
515   return $noit_pid;
516 }
517 sub get_noit_log {
518   return IO::File->new("<$noit_log");
519 }
520 sub stop_noit {
521   return 0 unless ($noit_pid && kill 0, $noit_pid);
522   kill 9, $noit_pid;
523   $noit_pid = 0;
524   return 1;
525 }
526
527 sub start_stratcon {
528   my $name = shift;
529   my $options = shift;
530   $options->{name} = $name;
531   return 0 if $stratcon_pid;
532   my $conf = make_stratcon_config($name, $options);
533   $stratcon_pid = fork();
534   mkdir "logs";
535   $stratcon_log = "logs/${name}_stratcon.log";
536   if($stratcon_pid == 0) {
537     $stratcon_pid = $$;
538     $stratcon_log = "logs/${name}_stratcon.log";
539     close(STDIN);
540     open(STDIN, "</dev/null");
541     close(STDOUT);
542     open(STDOUT, ">/dev/null");
543     close(STDERR);
544     open(STDERR, ">$stratcon_log");
545     my @args = ( 'stratcond', '-D', '-c', $conf );
546     exec { '../../src/stratcond' } @args;
547     exit(-1);
548   }
549   return $stratcon_pid;
550 }
551 sub get_stratcon_log {
552   return IO::File->new("<$stratcon_log");
553 }
554 sub stop_stratcon {
555   return 0 unless ($stratcon_pid && kill 0, $stratcon_pid);
556   kill 9, $stratcon_pid;
557   $stratcon_pid = 0;
558   return 1;
559 }
560
561 END {
562   kill 9, $noit_pid if($noit_pid && kill 1, $noit_pid);
563   kill 9, $stratcon_pid if($stratcon_pid && kill 1, $stratcon_pid);
564 }
565 1;
Note: See TracBrowser for help on using the browser.