root/trunk/tools/archive_logs.sh

Revision 195, 10.2 kB (checked in by depesz, 3 years ago)

Make archive_logs work correctly with log prefixes that contain - character.

  • Property svn:executable set to *
Line 
1 #!/bin/bash
2
3 # Function definitions have to be here.
4 # For actual code ran when you run this script, search for "# MAIN PROGRAM #"
5
6 # Function    : show_help_and_exit
7 # Description : Like the name suggests, it prints help page, and exits script
8 #             : If given args, treats them as format and arguments for printf
9 #             : and prints before help page
10 show_help_and_exit () {
11     if (( $# > 0 ))
12     then
13         FORMAT="ERROR:\n$1\n\n"
14         printf "$FORMAT" "${@:2:$#}" >&2
15     fi
16     cat <<_EO_HELP_
17 Syntax:
18     $0 -d /some/directory [OPTIONS]
19
20 Arguments:
21     -d       : specifies directory in which logs will be searched for
22
23 Options:
24     -x       : extended debug. Use when debugging the script itself.
25     -v       : show information while processing log files
26     -a VALUE : name of directory to put archives in
27              : if name of directory starts with / it is treated as full path
28              : to archive directory. Otherwise it is treated as subdirectory
29              : of log directory (-d)
30     -k VALUE : how many days of logs keep uncompressed
31     -c VALUE : compression program to use. Supported programs are:
32              :   - gzip
33              :   - bzip2
34              :   - lzma
35              : VALUE can be full path to program, but it's name must be one of
36              : the supported programs.
37     -r       : remove the logfiles instead of archiving them
38     -n       : use nice when compressing
39     -o       : overwrite pre-existing archive files (otherwise, log files which
40              : would archive to files that already exist will get skipped with
41              : warning)
42     -p VALUE : prefix of log filename. Defaults to 'postgresql'
43
44 Defaults:
45     -a archive -k 7 -c gzip
46
47 Description:
48
49 $0 finds all PostgreSQL log files, named: "PREFIX"-YYYY-MM-DD.log or
50 "PREFIX"-YYYY-MM-DD_HHMISS.log, skips files that are too new to new
51 compressed, compressed rest and moves to archive dir.
52
53 Archive dir is structured in a way to minimize number of files in single
54 directory. That is, in archive dir, there will be subdirectories named
55 "YYYY-MM", where YYYY-MM is year and month taken from log file name.
56 _EO_HELP_
57     exit
58 }
59
60 # Function    : get_compress_extension
61 # Description : prints name of extension to files compressed with given
62 #             : compressor (-c option)
63 get_compress_extension () {
64     REAL_NAME=$( basename "$COMPRESS" )
65     case "$REAL_NAME" in
66         gzip)
67             echo "gz"
68             ;;
69         bzip2)
70             echo "bz2"
71             ;;
72         lzma)
73             echo "lzma"
74             ;;
75     esac
76 }
77
78 # Function    : verbose_msg
79 # Description : Calls printf on given args, but only if VERBOSE is on.
80 verbose_msg () {
81     if (( $VERBOSE == 1 ))
82     then
83         printf "$@"
84     fi
85 }
86
87 # Function    : read_arguments
88 # Description : Reads arguments from command line, and validates them
89 #             : default values are in "MAIN PROGRAM" to simplify finding them
90 read_arguments () {
91     while getopts ':d:xva:k:c:p:norh' opt "$@"
92     do
93         case "$opt" in
94             d)
95                 LOG_DIRECTORY="$OPTARG"
96                 ;;
97             x)
98                 EXTENDED_DEBUG=1
99                 ;;
100             v)
101                 VERBOSE=1
102                 ;;
103             a)
104                 ARCHIVE_DIR="$OPTARG"
105                 ;;
106             k)
107                 KEEP_DAYS="$OPTARG"
108                 ;;
109             c)
110                 COMPRESS="$OPTARG"
111                 ;;
112             p)
113                 PREFIX="$OPTARG"
114                 ;;
115             n)
116                 USE_NICE=1
117                 ;;
118             o)
119                 OVERWRITE_ARCHIVE=1
120                 ;;
121             r)
122                 REMOVE_LOGS=1
123                 ;;
124             h)
125                 show_help_and_exit
126                 ;;
127             :)
128                 show_help_and_exit "Option -%s requires argument" "$OPTARG"
129                 ;;
130             \?)
131                 if [[ "$OPTARG" == "?" ]]
132                 then
133                     show_help_and_exit
134                 fi
135                 show_help_and_exit "Unknown option -%s" "$OPTARG"
136                 ;;
137         esac
138     done
139
140     if [[ "$LOG_DIRECTORY" == "" ]]
141     then
142         show_help_and_exit "log_directory (-d) was not provided!"
143     fi
144     if ! [[ -d "$LOG_DIRECTORY" ]]
145     then
146         show_help_and_exit "Given log_directory (%s) does not exist or is not directory!" "$LOG_DIRECTORY"
147     fi
148     if ! [[ -r "$LOG_DIRECTORY" ]]
149     then
150         show_help_and_exit "Given log_directory (%s) is not readable!" "$LOG_DIRECTORY"
151     fi
152     if ! [[ -w "$LOG_DIRECTORY" ]]
153     then
154         show_help_and_exit "Given log_directory (%s) is not writable!" "$LOG_DIRECTORY"
155     fi
156     if ! [[ -x "$LOG_DIRECTORY" ]]
157     then
158         show_help_and_exit "Given log_directory (%s) is not usable (lack of x in mode)!" "$LOG_DIRECTORY"
159     fi
160
161     if [[ "$PREFIX" == "" ]]
162     then
163         PREFIX=postgresql
164     fi
165     COMPRESS_EXTENSION=$( get_compress_extension )
166     if [[ "$COMPRESS_EXTENSION" == "" ]]
167     then
168         show_help_and_exit "Given compressor (%s) is not supported!" "$COMPRESS"
169     fi
170
171     if [[ ! "$KEEP_DAYS" =~ ^[0-9]+$ ]]
172     then
173         show_help_and_exit "Number of days to keep uncompressed (%s) is not a valid number (0+, integer)" "$KEEP_DAYS"
174     fi
175
176     if [[ "$ARCHIVE_DIR" == "" ]]
177     then
178         show_help_and_exit "Archive dir (-a) cannot be empty!"
179     fi
180
181     # Strip trailing / (normally, it would be s#/+$## or s#/\+$##, but
182     # it has to work on Solaris, so we have to use their approximation
183     # of regular expressions
184     LOG_DIRECTORY=$( echo "$LOG_DIRECTORY" | sed 's#/\{1,\}$##' )
185     ARCHIVE_DIR=$( echo "$ARCHIVE_DIR" | sed 's#/\{1,\}$##' )
186
187     # Make archive dir relative to LOG_DIRECTORY, if it's not absolute
188     if [[ ! ${ARCHIVE_DIR:0:1} == "/" ]]
189     then
190         ARCHIVE_DIR="${LOG_DIRECTORY}/${ARCHIVE_DIR}"
191     fi
192
193     if (( "$USE_NICE" == 0 ))
194     then
195         USE_NICE=""
196     else
197         USE_NICE="nice"
198     fi
199 }
200
201 # MAIN PROGAM #
202
203 # default values for arguments and options
204 LOG_DIRECTORY=""
205 EXTENDED_DEBUG=0
206 VERBOSE=0
207 ARCHIVE_DIR=archive
208 KEEP_DAYS=7
209 COMPRESS=gzip
210 USE_NICE=0
211 OVERWRITE_ARCHIVE=0
212 REMOVE_LOGS=0
213
214 # Set locale to sane one, to speed up comparisons, and be sure that < and > on
215 # strings work ok.
216 export LC_ALL=C
217
218 # Read arguments from command line
219 read_arguments "$@"
220
221 # Print settings
222 verbose_msg "$0 Settings:
223   - ARCHIVE_DIR       : $ARCHIVE_DIR
224   - COMPRESS          : $COMPRESS
225   - EXTENDED_DEBUG    : $EXTENDED_DEBUG
226   - KEEP_DAYS         : $KEEP_DAYS
227   - LOG_DIRECTORY     : $LOG_DIRECTORY
228   - OVERWRITE_ARCHIVE : $OVERWRITE_ARCHIVE
229   - USE_NICE          : $USE_NICE
230   - VERBOSE           : $VERBOSE
231   - REMOVE_LOGS       : $REMOVE_LOGS
232 "
233
234 # Make sure every error past this line is critical - this is to avoid having to
235 # check return codes from all calls to external programs
236 set -e
237
238 # Turn on extended debug
239 if (( $EXTENDED_DEBUG == 1 ))
240 then
241     set -x
242 fi
243
244 # Border date - any log from this date or newer has to stay uncompressed
245 # Have to use perl instead of more natural date --date="..." because
246 # Solaris date doesn't support --date="" option
247 export KEEP_DAYS
248 BORDER_DATE=$( perl -MPOSIX=strftime -e 'print strftime(q{%Y-%m-%d}, localtime( time() - $ENV{"KEEP_DAYS"} * 24 * 60 * 60 ))' )
249 verbose_msg "Border date: %s\n" "$BORDER_DATE"
250
251 # Find all log files, sort them (to work on them in order), and process
252 # Have to use grep, because Solaris find doesn't have -mindepth nor
253 # -maxdepth options
254 # Using ls could be an option, but it tends to write messages to STDERR if
255 # there are no matching files - not something that we would like.
256 find "$LOG_DIRECTORY"/ -type f \( -name "${PREFIX}-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].log" -o -name "${PREFIX}-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9][0-9][0-9][0-9][0-9].log" \) -print | \
257     egrep "^${LOG_DIRECTORY}/${PREFIX}-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9](_[0-9][0-9][0-9][0-9][0-9][0-9])?.log" | \
258     sort | \
259     while read SOURCE_FILENAME
260     do
261         FILENAME=$( basename "$SOURCE_FILENAME" )
262         # skip files that are not older than $BORDER_DATE
263         if [[ ! "$FILENAME" < "${PREFIX}-$BORDER_DATE" ]]
264         then
265             continue
266         fi
267
268         verbose_msg "Archiving file %-32s ... " "$FILENAME"
269
270         if [[ "$REMOVE_LOGS" -eq "1" ]]
271         then
272             rm "$SOURCE_FILENAME"
273             verbose_msg "File removed.\n"
274             continue
275         fi
276
277         # Extract year and month from log filename - to be used in archive path
278         # ${FILENAME:${#PREFIX}} extracts part of filename after prefix.
279         # i.e. with prefix == "a-b", and filename being a-b-2010-10-11, it
280         # returns -1010-10-11
281         # This is done to allow - character in prefixes, yet to still make
282         # the cut below to extract what we need.
283         YEAR_MONTH=$( echo "${FILENAME:${#PREFIX}}" | cut -d- -f2,3 )
284         if [[ ! -d "$ARCHIVE_DIR/$YEAR_MONTH" ]]
285         then
286             mkdir -p "$ARCHIVE_DIR/$YEAR_MONTH"
287         fi
288
289         DESTINATION_FILENAME="${ARCHIVE_DIR}/${YEAR_MONTH}/${FILENAME}.$( get_compress_extension )"
290
291         # Handle overriding
292         if [[ -e "${DESTINATION_FILENAME}" ]]
293         then
294             if (( $OVERWRITE_ARCHIVE == 0 ))
295             then
296                 printf "Destination file %s already exists. Skipping source %s.\n" "${DESTINATION_FILENAME}" "${SOURCE_FILENAME}" >&2
297                 continue
298             fi
299             verbose_msg "Destination file %s already exists. Overwriting. ... " "${DESTINATION_FILENAME}"
300         fi
301
302         # Have to use perl, because on Solaris, there is no stat command
303         export SOURCE_FILENAME
304         SIZE_BEFORE=$( perl -e 'print( (stat( $ENV{"SOURCE_FILENAME"} ))[7] )' )
305
306         # Actual archiving - compress to destination file, and remove of source
307         # I could also do compress without redirection, (x.log to x.log.gz) and
308         # then rename of temporary file, but this requires additional checks
309         # for existence of temporary file. Not worth the trouble
310         $USE_NICE $COMPRESS -c "$SOURCE_FILENAME" > "$DESTINATION_FILENAME"
311         rm -f "$SOURCE_FILENAME"
312
313         # Have to use perl, because on Solaris, there is no stat command
314         export DESTINATION_FILENAME
315         SIZE_AFTER=$( perl -e 'print( (stat( $ENV{"DESTINATION_FILENAME"} ))[7] )' )
316
317         verbose_msg "done. Size changed from %u to %u.\n" "$SIZE_BEFORE" "$SIZE_AFTER"
318     done
319
320 verbose_msg "All done.\n"
Note: See TracBrowser for help on using the browser.