Changeset 7 for trunk

Show
Ignore:
Timestamp:
11/13/00 01:43:19 (13 years ago)
Author:
jesus
Message:

All of George's <george@lethargy.org> vhost additions.
New time rewriting so that timestamps are rewritten to reflect the time on the local machine running spreadlogd -- me.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • trunk/config.c

    r6 r7  
    11/* ====================================================================== 
    2  * Copyright (c) 2000 Theo Schlossnagle 
    3  * All rights reserved. 
    4  * The following code was written by Theo Schlossnagle <jesus@omniti.com> 
    5  * This code was written to facilitate clustered logging via Spread. 
    6  * More information on Spread can be found at http://www.spread.org/ 
    7  * Please refer to the LICENSE file before using this software. 
    8  * ====================================================================== 
     2* Copyright (c) 2000 Theo Schlossnagle 
     3* All rights reserved. 
     4* The following code was written by Theo Schlossnagle <jesus@omniti.com> 
     5* This code was written to facilitate clustered logging via Spread. 
     6* More information on Spread can be found at http://www.spread.org/ 
     7* Please refer to the LICENSE file before using this software. 
     8* ====================================================================== 
    99*/ 
     10 
     11#include <linux/limits.h> 
    1012 
    1113#include <stdio.h> 
     
    1517#include <fcntl.h> 
    1618#include <sys/file.h> 
     19#include <sys/param.h> 
    1720#include <unistd.h> 
    1821 
    1922#include "config.h" 
    2023#include "skiplist.h" 
     24#include "timefuncs.h" 
    2125 
    2226extern FILE *yyin; 
     27extern int buffsize; 
     28 
    2329int yyparse (void); 
    24  
    2530int line_num, semantic_errors; 
    2631static Skiplist logfiles; 
    2732static Skiplist spreaddaemons; 
    28  
    2933extern int verbose; 
    3034extern int skiplocking; 
    31  
    3235int logfile_compare(void *a, void *b) { 
    3336  LogFile *ar = (LogFile *)a; 
    3437  LogFile *br = (LogFile *)b; 
    3538  return strcmp(ar->filename, br->filename); 
    36 } 
     39}  
    3740int logfile_compare_key(void *a, void *b) { 
    3841  LogFile *br = (LogFile *)b; 
    3942  return strcmp(a, br->filename); 
    40 
    41  
     43}   
    4244int spreadd_compare(void *a, void *b) { 
    4345  SpreadConfiguration *ar = (SpreadConfiguration *)a; 
     
    5052  if(!br->host) return 1; 
    5153  return strcmp(ar->host, br->host); 
    52 } 
     54}  
    5355int spreadd_compare_key(void *a, void *b) { 
    5456  SpreadConfiguration *br = (SpreadConfiguration *)b; 
     
    6062  if(!br->host) return 1; 
    6163  return strcmp(a, br->host); 
    62 
    63  
     64}   
    6465int facility_compare(void *a, void *b) { 
    6566  LogFacility *ar = (LogFacility *)a; 
    6667  LogFacility *br = (LogFacility *)b; 
    6768  return strcmp(ar->groupname, br->groupname); 
    68 } 
     69}  
    6970int facility_compare_key(void *a, void *b) { 
    7071  LogFacility *br = (LogFacility *)b; 
    7172  return strcmp(a, br->groupname); 
    72 
    73  
     73}   
    7474int config_init(char *filename) { 
    7575  int ret; 
    76  
    7776  sl_init(&logfiles); 
    7877  sl_set_compare(&logfiles, logfile_compare, logfile_compare_key); 
    7978  sl_init(&spreaddaemons); 
    8079  sl_set_compare(&spreaddaemons, spreadd_compare, spreadd_compare_key); 
    81    
    8280  /* 
    8381    sl_init(&logfacilities); 
    8482    sl_set_compare(&logfacilities, facility_compare, facility_compare_key); 
    8583  */ 
    86    
    8784  yyin = fopen(filename, "r"); 
    8885  if (!yyin) { 
     
    9390  fclose(yyin); 
    9491  return ret; 
    95 
    96  
     92}   
    9793char *config_get_spreaddaemon(SpreadConfiguration *sc) { 
    9894  static char buffer[1024]; 
    9995  if(sc->host) { 
    10096    snprintf(buffer, 1024, "%s@%s", sc->port, sc->host); 
    101   } else { 
     97  }  
     98  else { 
    10299    strncpy(buffer, sc->port, 1024); 
    103100  } 
    104101  return buffer; 
    105 
     102}  
     103 
    106104void config_set_spread_port(SpreadConfiguration *sc, char *newport) { 
    107105  if(sc->port) free(sc->port); 
    108106  sc->port = newport; 
    109 } 
     107}  
    110108void config_set_spread_host(SpreadConfiguration *sc, char *newhost) { 
    111109  if(sc->host) free(sc->host); 
    112110  sc->host = newhost; 
    113 } 
     111}  
    114112void config_add_spreadconf(SpreadConfiguration *sc) { 
    115113  sl_insert(&spreaddaemons, sc); 
    116 } 
     114}  
    117115SpreadConfiguration *config_new_spread_conf(void) { 
    118116  SpreadConfiguration *newsc; 
     
    126124                 facility_compare, facility_compare_key);   
    127125  return newsc; 
    128 
    129  
     126}   
    130127LogFacility *config_new_logfacility(void) { 
    131128  LogFacility *newlf; 
     
    134131  newlf->logfile=NULL; 
    135132  newlf->nmatches=0; 
     133  newlf->rewritetimes=NO_REWRITE_TIMES; 
     134  newlf->rewritetimesformat=NULL; 
    136135  return newlf; 
    137 } 
     136}  
    138137void config_add_logfacility(SpreadConfiguration *sc, LogFacility *lf) { 
    139138  sl_insert(sc->logfacilities, lf); 
    140 } 
     139}  
    141140void config_set_logfacility_group(LogFacility *lf, char *ng) { 
    142141  if(lf->groupname) free(lf->groupname); 
    143142  lf->groupname = ng; 
    144 } 
     143}  
    145144void config_set_logfacility_filename(LogFacility *lf, char *nf) { 
    146145  LogFile *logf; 
     
    153152    sl_insert(&logfiles, logf); 
    154153    lf->logfile = logf; 
    155   } else { 
     154  }  
     155  else { 
    156156    free(nf); 
    157157  } 
     158}  
     159void config_set_logfacility_vhostdir(LogFacility *lf, char *vhd) { 
     160  int i; 
     161  lf->vhostdir = vhd; 
     162  lf->hash = (hash_element *) malloc (NR_OPEN * sizeof(hash_element)); 
     163  fprintf( stderr, "\nZeroing vhost hash for usage!\n"); 
     164  for(i=0; i<NR_OPEN; i++) { 
     165    lf->hash[i].fd = -1; 
     166    lf->hash[i].hostheader = NULL; 
     167  } 
     168 
     169} 
     170void config_set_logfaclity_rewritetimes_clf(LogFacility *lf) { 
     171  lf->rewritetimes = REWRITE_TIMES_IN_CLF; 
     172} 
     173void config_set_logfaclity_rewritetimes_user(LogFacility *lf, char *format) { 
     174  lf->rewritetimes = REWRITE_TIMES_FORMAT; 
     175  if(lf->rewritetimesformat) { 
     176    free(lf->rewritetimesformat); 
     177  } 
     178  lf->rewritetimesformat = format; 
    158179} 
    159180void config_add_logfacility_match(LogFacility *lf, char *nm) { 
    160181#ifdef RE_SYNTAX_EGREP 
    161182  const char *ret; 
    162 #else 
     183#else   
    163184  int ret; 
    164 #endif 
     185#endif   
    165186  if(lf->nmatches>=10) { 
    166187    fprintf(stderr, "Already 10 regex's on group\n"); 
     
    170191  re_set_syntax(RE_SYNTAX_EGREP); 
    171192  if((ret = re_compile_pattern(nm, strlen(nm), 
    172                               &lf->match_expression[lf->nmatches]))!=0) { 
     193                              &lf->match_expression[lf->nmatches]))!=0) { 
    173194    fprintf(stderr, ret); 
    174 #else 
    175  if((ret = regcomp(&lf->match_expression[lf->nmatches], nm, REG_EGREP))!=0) { 
    176    char errbuf[120]; 
    177    regerror(ret, &lf->match_expression[lf->nmatches], errbuf, sizeof errbuf); 
    178    fprintf(stderr, errbuf); 
    179 #endif 
    180   } else { 
     195#else   
     196if((ret = regcomp(&lf->match_expression[lf->nmatches], nm, REG_EGREP))!=0) { 
     197      char errbuf[120]; 
     198      regerror(ret, &lf->match_expression[lf->nmatches], errbuf, sizeof errbuf); 
     199      fprintf(stderr, errbuf); 
     200#endif   
     201  } else {  
    181202    lf->nmatches++; 
    182203  } 
    183204} 
     205 
    184206int config_foreach_spreadconf(int (*func)(SpreadConfiguration *, void *), 
    185207                              void *closure) { 
     
    187209  struct skiplistnode *iter; 
    188210  SpreadConfiguration *sc; 
    189  
    190211  iter = sl_getlist(&spreaddaemons); 
    191212  if(!iter) return i; 
    192  
    193213  sc = iter->data; 
    194214  do { 
     
    197217  return i;   
    198218} 
     219 
    199220int config_foreach_logfacility(SpreadConfiguration *sc,  
    200221                               int (*func)(LogFacility *, void *), 
     
    203224  struct skiplistnode *iter; 
    204225  LogFacility *lf; 
    205  
    206226  iter = sl_getlist(sc->logfacilities); 
    207227  if(!iter) return i; 
    208  
    209228  lf = iter->data; 
    210229  do { 
     
    213232  return i;   
    214233} 
     234 
     235char *config_process_message(SpreadConfiguration *sc, char *group, 
     236                             char *message, int *len) { 
     237  LogFacility *lf; 
     238  char *cp; 
     239  lf = sl_find(sc->logfacilities, group, NULL); 
     240  if(lf->rewritetimes) 
     241    force_local_time(message, len, buffsize, 
     242                     lf->rewritetimes, lf->rewritetimesformat); 
     243  if(lf->vhostdir) { 
     244    cp=message; 
     245    while(*cp != ' ') { 
     246        cp++; 
     247        --*len; 
     248    } 
     249    --*len; 
     250    return cp+1; 
     251  } 
     252  return message; 
     253}    
     254 
    215255void config_hup(void) { 
    216256  config_close(); 
    217257  config_start(); 
    218 } 
     258}   
    219259 
    220260int config_close(void) { 
     
    222262  SpreadConfiguration *sc; 
    223263  LogFacility *lf; 
    224  
     264                                int i; 
    225265  sciter = sl_getlist(&spreaddaemons); 
    226266  if(!sciter) return 0; 
    227    
    228267  sc = (SpreadConfiguration *)sciter->data; 
    229268  /* For each spread configuration: */ 
     
    231270    lfiter = sl_getlist(sc->logfacilities); 
    232271    if(!lfiter) return 0; 
    233      
    234272    lf = (LogFacility *)lfiter->data; 
    235273    /* For each log facility in that spread configuration: */ 
    236274    do { 
    237       if(lf->logfile->fd>0) { 
     275      if(lf->vhostdir) { 
     276        for (i=0;i<NR_OPEN;i++) { 
     277          if(lf->hash[i].fd>0) { 
     278            if(!skiplocking) flock(lf->hash[i].fd, LOCK_UN); 
     279            close(lf->hash[i].fd); 
     280            lf->hash[i].fd = -1; 
     281          } 
     282        } 
     283      } else if(lf->logfile->fd>0) { 
    238284        if(!skiplocking) flock(lf->logfile->fd, LOCK_UN); 
    239285        close(lf->logfile->fd); 
     
    243289  } while((sc = sl_next(&spreaddaemons, &sciter))!=NULL); 
    244290  return 0; 
    245 } 
     291}   
    246292 
    247293int config_start(void) { 
     
    249295  SpreadConfiguration *sc; 
    250296  LogFacility *lf; 
    251  
    252297  sciter = sl_getlist(&spreaddaemons); 
    253298  if(!sciter) return 0; 
    254    
    255299  sc = (SpreadConfiguration *)sciter->data; 
    256300  /* For each spread configuration: */ 
     
    258302    lfiter = sl_getlist(sc->logfacilities); 
    259303    if(!lfiter) return 0; 
    260  
    261304    lf = (LogFacility *)lfiter->data; 
    262305    /* For each log facility in that spread configuration: */ 
    263306    do { 
    264       if(lf->logfile->fd<0) 
     307      if(lf->vhostdir) continue; 
     308      else if(lf->logfile->fd<0) 
    265309        lf->logfile->fd = open(lf->logfile->filename, 
    266310#ifdef __USE_LARGEFILE64 
    267311                               O_CREAT|O_APPEND|O_WRONLY|O_LARGEFILE, 
    268 #else 
     312#else   
    269313                               O_CREAT|O_APPEND|O_WRONLY, 
    270 #endif 
     314#endif   
    271315                               00644); 
    272316      if(!skiplocking) { 
     
    285329  } while((sc = sl_next(&spreaddaemons, &sciter))!=NULL); 
    286330  return 0; 
    287 } 
     331}   
    288332 
    289333int config_get_fd(SpreadConfiguration *sc, char *group, char *message) { 
    290334  LogFacility *lf; 
    291   int i, ret, slen; 
     335  int i, ret, slen, fd; 
     336  hash_element temp; 
     337  char *cp; 
     338  char fullpath[MAXPATHLEN]; 
    292339  lf = sl_find(sc->logfacilities, group, NULL); 
    293340  if(!lf) return -1; 
     341  if(lf->vhostdir) { 
     342    cp = message; 
     343    while(*cp != ' '){ 
     344      cp++; 
     345    } 
     346    *cp = '\0'; 
     347    if((fd = gethash(message, lf->hash)) < 0) { 
     348      temp.hostheader = strdup(message); 
     349      *cp = ' '; 
     350      snprintf(fullpath, MAXPATHLEN, "%s/%s", lf->vhostdir,temp.hostheader); 
     351      temp.fd = open(fullpath, 
     352#ifdef __USE_LARGEFILE64 
     353                     O_CREAT|O_APPEND|O_WRONLY|O_LARGEFILE, 
     354#else   
     355                     O_CREAT|O_APPEND|O_WRONLY, 
     356#endif   
     357                     00644); 
     358      if(!skiplocking) { 
     359        if(flock(temp.fd, LOCK_NB|LOCK_EX)==-1) { 
     360          fprintf(stderr, "Cannot lock %s, is another spreadlogd running?\n", 
     361                  fullpath); 
     362          exit(1); 
     363        } 
     364      } 
     365      inshash(temp,lf->hash); 
     366      return temp.fd; 
     367    }                                            
     368    *cp = ' '; 
     369    return fd;  
     370  }      
    294371  if(!lf->nmatches) return lf->logfile->fd; 
    295372  slen = strlen(message); 
     
    303380    else if(ret==-1 && verbose) 
    304381      fprintf(stderr, "Failed match!\n"); 
    305 #else 
    306     if(!regexec(&lf->match_expression[i], message, 0, NULL, 0)) 
    307       return lf->logfile->fd; 
    308 #endif 
     382#else  if(!regexec(&lf->match_expression[i], message, 0, NULL, 0)) 
     383    return lf->logfile->fd; 
     384#endif   
    309385  } 
    310386  return -1; 
    311 } 
    312  
     387}   
     388 
  • trunk/config.h

    r2 r7  
    1515#include <sys/file.h> 
    1616#include <regex.h> 
    17  
    1817#include <sp.h> 
    1918 
     19#include "hash.h" 
    2020#include "skiplist.h" 
    2121 
     
    3333} LogFile; 
    3434 
     35 
    3536typedef struct { 
    3637  char *groupname; 
    3738  LogFile *logfile; 
    3839  int nmatches; 
     40  int rewritetimes; 
     41  char *rewritetimesformat; 
    3942  regex_t match_expression[10]; /* only up to ten */ 
     43  char *vhostdir; 
     44  hash_element *hash; 
    4045} LogFacility; 
    4146 
     
    5560void config_set_logfacility_filename(LogFacility *, char *); 
    5661void config_add_logfacility_match(LogFacility *, char *); 
     62void config_set_logfacility_vhostdir(LogFacility *lf, char *vhd); 
     63void config_set_logfaclity_rewritetimes_clf(LogFacility *lf); 
     64void config_set_logfaclity_rewritetimes_user(LogFacility *lf, char *format); 
    5765 
    5866int config_foreach_logfacility(SpreadConfiguration *, 
    5967                               int (*)(LogFacility *, void *), void *); 
    60  
     68char *config_process_message(SpreadConfiguration *sc, char *group, char *message, int *len); 
    6169void config_hup(void);  /* config_close(); config_start(); */ 
    6270int config_close(void); /* Close files */ 
  • trunk/config_gram.l

    r2 r7  
    3333Host                            { return HOST; } 
    3434Log                             { return LOG; } 
     35VhostGroup                      { return VHOSTGROUP; } 
    3536Group                           { return GROUP; } 
    3637File                            { return FILENAME; } 
     38VhostDir                        { return VHOSTDIR; } 
    3739Match                           { return MATCH; } 
     40RewriteTimestamp                { return REWRITETIMES; } 
     41CommonLogFormat                 { return CLF; } 
    3842{qstring}                       { int l = strlen(yytext); 
    3943                                  yytext[l-1] = 0; 
  • trunk/config_gram.y

    r2 r7  
    1919static LogFacility *current_lf = NULL; 
    2020 
     21int yyerror(char *str); 
     22 
    2123#define NEW_SC_IFNEEDED if(!current_sc) current_sc=config_new_spread_conf(); 
    2224 
     
    2527%} 
    2628%start Config 
    27 %token BUFFERSIZE SPREAD PORT HOST LOG GROUP FILENAME MATCH 
    28 %token OPENBRACE CLOSEBRACE EQUALS STRING 
     29%token BUFFERSIZE SPREAD PORT HOST LOG GROUP FILENAME MATCH VHOSTGROUP VHOSTDIR 
     30%token OPENBRACE CLOSEBRACE EQUALS STRING CLF REWRITETIMES 
    2931%% 
    3032Config          :       Globals SpreadConfs 
     
    8486                        { NEW_LF_IFNEEDED; 
    8587                          config_add_logfacility_match(current_lf, $3); } 
     88                |       VHOSTDIR EQUALS STRING 
     89                        { NEW_LF_IFNEEDED; 
     90                          config_set_logfacility_vhostdir(current_lf, $3); } 
     91                |       REWRITETIMES EQUALS CLF 
     92                        { NEW_LF_IFNEEDED; 
     93fprintf(stderr, "Setting logfacility to force local times in CLF\n"); 
     94                          config_set_logfaclity_rewritetimes_clf(current_lf); } 
     95                |       REWRITETIMES EQUALS STRING 
     96                        { NEW_LF_IFNEEDED; 
     97                          config_set_logfaclity_rewritetimes_user(current_lf, 
     98                                                                  $3); } 
    8699                ; 
    87100 
  • trunk/makefile

    r6 r7  
    11CC=gcc 
    2 CFLAGS=-g -D__USE_LARGEFILE64 
     2CFLAGS=-g -D__USE_LARGEFILE64 -Wall 
    33INCLUDES=-I/usr/local/include 
    44 
     
    99 
    1010# For Solaris 
    11 #LIBS=-lsp -lskiplist -lnsl -lsocket -lucb 
     11#LIBS=-lsp -lskiplist -lnsl -lsocket -lucb  
    1212#LDFLAGS=-L/usr/local/lib -L/usr/ucblib -R/usr/ucblib -L. 
    1313#BSDINCLUDES=-I/usr/ucbinclude 
     
    1717LEX=flex 
    1818 
    19 OBJS=spreadlogd.o lex.yy.o y.tab.o config.o 
     19OBJS=spreadlogd.o lex.yy.o y.tab.o config.o hash.o timefuncs.o 
    2020LSLOBJS=skiplist.o 
    2121 
     
    3333        $(CC) $(CFLAGS) $(INCLUDES) -c y.tab.c 
    3434 
     35test.o: test.c 
     36        $(CC) $(CFLAGS) $(BSDINCLUDES) $(INCLUDES) -c $< 
     37 
    3538config.o:       config.c 
    3639        $(CC) $(CFLAGS) $(BSDINCLUDES) $(INCLUDES) -c $< 
    37  
     40hash.o:         hash.c 
     41        $(CC) $(CFLAGS) $(BSDINCLUDES) $(INCLUDES) -c $< 
    3842.c.o:   $*.c 
    3943        $(CC) $(CFLAGS) $(INCLUDES) -c $< 
  • trunk/spreadlogd.c

    r6 r7  
    1414#include <unistd.h> 
    1515#include <sys/time.h> 
    16  
    1716#include <sp.h> 
    1817 
    1918#include "config.h" 
    2019 
    21 #define SPREADLOGD_VERSION "1.3
     20#define SPREADLOGD_VERSION "1.4
    2221 
    2322extern char *optarg; 
     
    3029int skiplocking = 0; 
    3130int buffsize = -1; 
    32  
    3331SpreadConfiguration **fds; 
    3432int fdsetsize; 
     
    202200    service service_type; 
    203201    char sender[MAX_GROUP_NAME]; 
     202    char *pmessage; 
    204203    int len, num_groups, endian, logfd; 
    205204    char groups[1][MAX_GROUP_NAME]; 
     
    252251              logfd = config_get_fd(fds[fd], groups[0], message); 
    253252              if(logfd<0) continue; 
    254               write(logfd, message, len); 
     253              pmessage = config_process_message(fds[fd],groups[0], message, &len); 
     254              write(logfd, pmessage, len); 
    255255            } 
    256256          } 
  • trunk/spreadlogd.conf

    r2 r7  
    1414        Host = excalibur 
    1515        Log { 
     16# Rewrite incoming log lines so that the timestamps will reflect the time 
     17# on the local machine.  This will allow you web server to be slightly out 
     18# sync with respect to time and still have monotonically time increasing logs 
     19                RewriteTimestamp = CommonLogFormat 
    1620                Group = omniti 
    1721                File = omniti.log 
     
    3943#       } 
    4044 
     45#       Log { 
     46#               Group = apache-0003 
     47# This directory will contain one file for each of the apache vitual servers  
     48# that gets hashed to '1' by mod_log_spread's $#vhost 'magic group'.   
     49# Across you cluster there should be one such statement for every integer  
     50# between 0 and VhostLoghashSize - 1.  Be careful not to run out of open  
     51# file descriptors (i.e., if you have a 100,000 virtual hosts, you'll probably 
     52# run out of fd's on a single box.) 
     53#               VhostDir = /vhostdata1 
     54#       } 
     55#  
    4156        Log { 
    4257                Group = "test"