root/trunk/tools/archive_logs.sh

Revision 56, 9.4 kB (checked in by depesz, 4 years ago)

if logdir is symlink, version without / at the end doesnt work

  • Property svn:executable set to *
Line 
1 #!/bin/bash
2
3 # Function definitions have to be here.
4 # For actual code ran when you run this script, search for "# MAIN PROGRAM #"
5
6 # Function    : show_help_and_exit
7 # Description : Like the name suggests, it prints help page, and exits script
8 #             : If given args, treats them as format and arguments for printf
9 #             : and prints before help page
10 show_help_and_exit () {
11     if (( $# > 0 ))
12     then
13         FORMAT="ERROR:\n$1\n\n"
14         printf "$FORMAT" "${@:2:$#}" >&2
15     fi
16     cat <<_EO_HELP_
17 Syntax:
18     $0 -d /some/directory [OPTIONS]
19
20 Arguments:
21     -d       : specifies directory in which logs will be searched for
22
23 Options:
24     -x       : extended debug. Use when debugging the script itself.
25     -v       : show information while processing log files
26     -a VALUE : name of directory to put archives in
27              : if name of directory starts with / it is treated as full path
28              : to archive directory. Otherwise it is treated as subdirectory
29              : of log directory (-d)
30     -k VALUE : how many days of logs keep uncompressed
31     -c VALUE : compression program to use. Supported programs are:
32              :   - gzip
33              :   - bzip2
34              :   - lzma
35              : VALUE can be full path to program, but it's name must be one of
36              : the supported programs.
37     -n       : use nice when compressing
38     -o       : overwrite pre-existing archive files (otherwise, log files which
39              : would archive to files that already exist will get skipped with
40              : warning)
41
42 Defaults:
43     -a archive -k 7 -c gzip
44
45 Description:
46
47 $0 finds all PostgreSQL log files, named: postgresql-YYYY-MM-DD.log or
48 postgresql-YYYY-MM-DD_HHMISS.log, skips files that are too new to new
49 compressed, compressed rest and moves to archive dir.
50
51 Archive dir is structured in a way to minimize number of files in single
52 directory. That is, in archive dir, there will be subdirectories named
53 "YYYY-MM", where YYYY-MM is year and month taken from log file name.
54 _EO_HELP_
55     exit
56 }
57
58 # Function    : get_compress_extension
59 # Description : prints name of extension to files compressed with given
60 #             : compressor (-c option)
61 get_compress_extension () {
62     REAL_NAME=$( basename "$COMPRESS" )
63     case "$REAL_NAME" in
64         gzip)
65             echo "gz"
66             ;;
67         bzip2)
68             echo "bz2"
69             ;;
70         lzma)
71             echo "lzma"
72             ;;
73     esac
74 }
75
76 # Function    : verbose_msg
77 # Description : Calls printf on given args, but only if VERBOSE is on.
78 verbose_msg () {
79     if (( $VERBOSE == 1 ))
80     then
81         printf "$@"
82     fi
83 }
84
85 # Function    : read_arguments
86 # Description : Reads arguments from command line, and validates them
87 #             : default values are in "MAIN PROGRAM" to simplify finding them
88 read_arguments () {
89     while getopts ':d:xva:k:c:noh' opt "$@"
90     do
91         case "$opt" in
92             d)
93                 LOG_DIRECTORY="$OPTARG"
94                 ;;
95             x)
96                 EXTENDED_DEBUG=1
97                 ;;
98             v)
99                 VERBOSE=1
100                 ;;
101             a)
102                 ARCHIVE_DIR="$OPTARG"
103                 ;;
104             k)
105                 KEEP_DAYS="$OPTARG"
106                 ;;
107             c)
108                 COMPRESS="$OPTARG"
109                 ;;
110             n)
111                 USE_NICE=1
112                 ;;
113             o)
114                 OVERWRITE_ARCHIVE=1
115                 ;;
116             h)
117                 show_help_and_exit
118                 ;;
119             :)
120                 show_help_and_exit "Option -%s requires argument" "$OPTARG"
121                 ;;
122             \?)
123                 if [[ "$OPTARG" == "?" ]]
124                 then
125                     show_help_and_exit
126                 fi
127                 show_help_and_exit "Unknown option -%s" "$OPTARG"
128                 ;;
129         esac
130     done
131
132     if [[ "$LOG_DIRECTORY" == "" ]]
133     then
134         show_help_and_exit "log_directory (-d) was not provided!"
135     fi
136     if ! [[ -d "$LOG_DIRECTORY" ]]
137     then
138         show_help_and_exit "Given log_directory (%s) does not exist or is not directory!" "$LOG_DIRECTORY"
139     fi
140     if ! [[ -r "$LOG_DIRECTORY" ]]
141     then
142         show_help_and_exit "Given log_directory (%s) is not readable!" "$LOG_DIRECTORY"
143     fi
144     if ! [[ -w "$LOG_DIRECTORY" ]]
145     then
146         show_help_and_exit "Given log_directory (%s) is not writable!" "$LOG_DIRECTORY"
147     fi
148     if ! [[ -x "$LOG_DIRECTORY" ]]
149     then
150         show_help_and_exit "Given log_directory (%s) is not usable (lack of x in mode)!" "$LOG_DIRECTORY"
151     fi
152
153     COMPRESS_EXTENSION=$( get_compress_extension )
154     if [[ "$COMPRESS_EXTENSION" == "" ]]
155     then
156         show_help_and_exit "Given compressor (%s) is not supported!" "$COMPRESS"
157     fi
158
159     if [[ ! "$KEEP_DAYS" =~ ^[0-9]+$ ]]
160     then
161         show_help_and_exit "Number of days to keep uncompressed (%s) is not a valid number (0+, integer)" "$KEEP_DAYS"
162     fi
163
164     if [[ "$ARCHIVE_DIR" == "" ]]
165     then
166         show_help_and_exit "Archive dir (-a) cannot be empty!"
167     fi
168
169     # Strip trailing / (normally, it would be s#/+$## or s#/\+$##, but
170     # it has to work on Solaris, so we have to use their approximation
171     # of regular expressions
172     LOG_DIRECTORY=$( echo "$LOG_DIRECTORY" | sed 's#/\{1,\}$##' )
173     ARCHIVE_DIR=$( echo "$ARCHIVE_DIR" | sed 's#/\{1,\}$##' )
174
175     # Make archive dir relative to LOG_DIRECTORY, if it's not absolute
176     if [[ ! ${ARCHIVE_DIR:0:1} == "/" ]]
177     then
178         ARCHIVE_DIR="${LOG_DIRECTORY}/${ARCHIVE_DIR}"
179     fi
180
181     if (( "$USE_NICE" == 0 ))
182     then
183         USE_NICE=""
184     else
185         USE_NICE="nice"
186     fi
187 }
188
189 # MAIN PROGAM #
190
191 # default values for arguments and options
192 LOG_DIRECTORY=""
193 EXTENDED_DEBUG=0
194 VERBOSE=0
195 ARCHIVE_DIR=archive
196 KEEP_DAYS=7
197 COMPRESS=gzip
198 USE_NICE=0
199 OVERWRITE_ARCHIVE=0
200
201 # Set locale to sane one, to speed up comparisons, and be sure that < and > on
202 # strings work ok.
203 export LC_ALL=C
204
205 # Read arguments from command line
206 read_arguments "$@"
207
208 # Print settings
209 verbose_msg "$0 Settings:
210   - ARCHIVE_DIR       : $ARCHIVE_DIR
211   - COMPRESS          : $COMPRESS
212   - EXTENDED_DEBUG    : $EXTENDED_DEBUG
213   - KEEP_DAYS         : $KEEP_DAYS
214   - LOG_DIRECTORY     : $LOG_DIRECTORY
215   - OVERWRITE_ARCHIVE : $OVERWRITE_ARCHIVE
216   - USE_NICE          : $USE_NICE
217   - VERBOSE           : $VERBOSE
218 "
219
220 # Make sure every error past this line is critical - this is to avoid having to
221 # check return codes from all calls to external programs
222 set -e
223
224 # Turn on extended debug
225 if (( $EXTENDED_DEBUG == 1 ))
226 then
227     set -x
228 fi
229
230 # Border date - any log from this date or newer has to stay uncompressed
231 # Have to use perl instead of more natural date --date="..." because
232 # Solaris date doesn't support --date="" option
233 export KEEP_DAYS
234 BORDER_DATE=$( perl -MPOSIX=strftime -e 'print strftime(q{%Y-%m-%d}, localtime( time() - $ENV{"KEEP_DAYS"} * 24 * 60 * 60 ))' )
235 verbose_msg "Border date: %s\n" "$BORDER_DATE"
236
237 # Find all log files, sort them (to work on them in order), and process
238 # Have to use grep, because Solaris find doesn't have -mindepth nor
239 # -maxdepth options
240 # Using ls could be an option, but it tends to write messages to STDERR if
241 # there are no matching files - not something that we would like.
242 find "$LOG_DIRECTORY"/ -type f \( -name 'postgresql-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].log' -o -name 'postgresql-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9][0-9][0-9][0-9][0-9].log' \) -print | \
243     egrep "^${LOG_DIRECTORY}/postgresql-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9](_[0-9][0-9][0-9][0-9][0-9][0-9])?.log" | \
244     sort | \
245     while read SOURCE_FILENAME
246     do
247         FILENAME=$( basename "$SOURCE_FILENAME" )
248         # skip files that are not older than $BORDER_DATE
249         if [[ ! "$FILENAME" < "postgresql-$BORDER_DATE" ]]
250         then
251             continue
252         fi
253
254         verbose_msg "Archiving file %-32s ... " "$FILENAME"
255
256         # Extract year and month from log filename - to be used in archive path
257         YEAR_MONTH=$( echo "$FILENAME" | cut -d- -f2,3 )
258         if [[ ! -d "$ARCHIVE_DIR/$YEAR_MONTH" ]]
259         then
260             mkdir -p "$ARCHIVE_DIR/$YEAR_MONTH"
261         fi
262
263         DESTINATION_FILENAME="${ARCHIVE_DIR}/${YEAR_MONTH}/${FILENAME}.$( get_compress_extension )"
264
265         # Handle overriding
266         if [[ -e "${DESTINATION_FILENAME}" ]]
267         then
268             if (( $OVERWRITE_ARCHIVE == 0 ))
269             then
270                 printf "Destination file %s already exists. Skipping source %s.\n" "${DESTINATION_FILENAME}" "${SOURCE_FILENAME}" >&2
271                 continue
272             fi
273             verbose_msg "Destination file %s already exists. Overwriting. ... " "${DESTINATION_FILENAME}"
274         fi
275
276         # Have to use perl, because on Solaris, there is no stat command
277         export SOURCE_FILENAME
278         SIZE_BEFORE=$( perl -e 'print( (stat( $ENV{"SOURCE_FILENAME"} ))[7] )' )
279
280         # Actual archiving - compress to destination file, and remove of source
281         # I could also do compress without redirection, (x.log to x.log.gz) and
282         # then rename of temporary file, but this requires additional checks
283         # for existence of temporary file. Not worth the trouble
284         $USE_NICE $COMPRESS -c "$SOURCE_FILENAME" > "$DESTINATION_FILENAME"
285         rm -f "$SOURCE_FILENAME"
286
287         # Have to use perl, because on Solaris, there is no stat command
288         export DESTINATION_FILENAME
289         SIZE_AFTER=$( perl -e 'print( (stat( $ENV{"DESTINATION_FILENAME"} ))[7] )' )
290
291         verbose_msg "done. Size changed from %u to %u.\n" "$SIZE_BEFORE" "$SIZE_AFTER"
292     done
293
294 verbose_msg "All done.\n"
Note: See TracBrowser for help on using the browser.