root/trunk/tools/archive_logs.sh

Revision 139, 9.7 kB (checked in by depesz, 4 years ago)

add -r option, to remove old logs instead of archiving them

  • Property svn:executable set to *
Line 
1 #!/bin/bash
2
3 # Function definitions have to be here.
4 # For actual code ran when you run this script, search for "# MAIN PROGRAM #"
5
6 # Function    : show_help_and_exit
7 # Description : Like the name suggests, it prints help page, and exits script
8 #             : If given args, treats them as format and arguments for printf
9 #             : and prints before help page
10 show_help_and_exit () {
11     if (( $# > 0 ))
12     then
13         FORMAT="ERROR:\n$1\n\n"
14         printf "$FORMAT" "${@:2:$#}" >&2
15     fi
16     cat <<_EO_HELP_
17 Syntax:
18     $0 -d /some/directory [OPTIONS]
19
20 Arguments:
21     -d       : specifies directory in which logs will be searched for
22
23 Options:
24     -x       : extended debug. Use when debugging the script itself.
25     -v       : show information while processing log files
26     -a VALUE : name of directory to put archives in
27              : if name of directory starts with / it is treated as full path
28              : to archive directory. Otherwise it is treated as subdirectory
29              : of log directory (-d)
30     -k VALUE : how many days of logs keep uncompressed
31     -c VALUE : compression program to use. Supported programs are:
32              :   - gzip
33              :   - bzip2
34              :   - lzma
35              : VALUE can be full path to program, but it's name must be one of
36              : the supported programs.
37     -r       : remove the logfiles instead of archiving them
38     -n       : use nice when compressing
39     -o       : overwrite pre-existing archive files (otherwise, log files which
40              : would archive to files that already exist will get skipped with
41              : warning)
42
43 Defaults:
44     -a archive -k 7 -c gzip
45
46 Description:
47
48 $0 finds all PostgreSQL log files, named: postgresql-YYYY-MM-DD.log or
49 postgresql-YYYY-MM-DD_HHMISS.log, skips files that are too new to new
50 compressed, compressed rest and moves to archive dir.
51
52 Archive dir is structured in a way to minimize number of files in single
53 directory. That is, in archive dir, there will be subdirectories named
54 "YYYY-MM", where YYYY-MM is year and month taken from log file name.
55 _EO_HELP_
56     exit
57 }
58
59 # Function    : get_compress_extension
60 # Description : prints name of extension to files compressed with given
61 #             : compressor (-c option)
62 get_compress_extension () {
63     REAL_NAME=$( basename "$COMPRESS" )
64     case "$REAL_NAME" in
65         gzip)
66             echo "gz"
67             ;;
68         bzip2)
69             echo "bz2"
70             ;;
71         lzma)
72             echo "lzma"
73             ;;
74     esac
75 }
76
77 # Function    : verbose_msg
78 # Description : Calls printf on given args, but only if VERBOSE is on.
79 verbose_msg () {
80     if (( $VERBOSE == 1 ))
81     then
82         printf "$@"
83     fi
84 }
85
86 # Function    : read_arguments
87 # Description : Reads arguments from command line, and validates them
88 #             : default values are in "MAIN PROGRAM" to simplify finding them
89 read_arguments () {
90     while getopts ':d:xva:k:c:norh' opt "$@"
91     do
92         case "$opt" in
93             d)
94                 LOG_DIRECTORY="$OPTARG"
95                 ;;
96             x)
97                 EXTENDED_DEBUG=1
98                 ;;
99             v)
100                 VERBOSE=1
101                 ;;
102             a)
103                 ARCHIVE_DIR="$OPTARG"
104                 ;;
105             k)
106                 KEEP_DAYS="$OPTARG"
107                 ;;
108             c)
109                 COMPRESS="$OPTARG"
110                 ;;
111             n)
112                 USE_NICE=1
113                 ;;
114             o)
115                 OVERWRITE_ARCHIVE=1
116                 ;;
117             r)
118                 REMOVE_LOGS=1
119                 ;;
120             h)
121                 show_help_and_exit
122                 ;;
123             :)
124                 show_help_and_exit "Option -%s requires argument" "$OPTARG"
125                 ;;
126             \?)
127                 if [[ "$OPTARG" == "?" ]]
128                 then
129                     show_help_and_exit
130                 fi
131                 show_help_and_exit "Unknown option -%s" "$OPTARG"
132                 ;;
133         esac
134     done
135
136     if [[ "$LOG_DIRECTORY" == "" ]]
137     then
138         show_help_and_exit "log_directory (-d) was not provided!"
139     fi
140     if ! [[ -d "$LOG_DIRECTORY" ]]
141     then
142         show_help_and_exit "Given log_directory (%s) does not exist or is not directory!" "$LOG_DIRECTORY"
143     fi
144     if ! [[ -r "$LOG_DIRECTORY" ]]
145     then
146         show_help_and_exit "Given log_directory (%s) is not readable!" "$LOG_DIRECTORY"
147     fi
148     if ! [[ -w "$LOG_DIRECTORY" ]]
149     then
150         show_help_and_exit "Given log_directory (%s) is not writable!" "$LOG_DIRECTORY"
151     fi
152     if ! [[ -x "$LOG_DIRECTORY" ]]
153     then
154         show_help_and_exit "Given log_directory (%s) is not usable (lack of x in mode)!" "$LOG_DIRECTORY"
155     fi
156
157     COMPRESS_EXTENSION=$( get_compress_extension )
158     if [[ "$COMPRESS_EXTENSION" == "" ]]
159     then
160         show_help_and_exit "Given compressor (%s) is not supported!" "$COMPRESS"
161     fi
162
163     if [[ ! "$KEEP_DAYS" =~ ^[0-9]+$ ]]
164     then
165         show_help_and_exit "Number of days to keep uncompressed (%s) is not a valid number (0+, integer)" "$KEEP_DAYS"
166     fi
167
168     if [[ "$ARCHIVE_DIR" == "" ]]
169     then
170         show_help_and_exit "Archive dir (-a) cannot be empty!"
171     fi
172
173     # Strip trailing / (normally, it would be s#/+$## or s#/\+$##, but
174     # it has to work on Solaris, so we have to use their approximation
175     # of regular expressions
176     LOG_DIRECTORY=$( echo "$LOG_DIRECTORY" | sed 's#/\{1,\}$##' )
177     ARCHIVE_DIR=$( echo "$ARCHIVE_DIR" | sed 's#/\{1,\}$##' )
178
179     # Make archive dir relative to LOG_DIRECTORY, if it's not absolute
180     if [[ ! ${ARCHIVE_DIR:0:1} == "/" ]]
181     then
182         ARCHIVE_DIR="${LOG_DIRECTORY}/${ARCHIVE_DIR}"
183     fi
184
185     if (( "$USE_NICE" == 0 ))
186     then
187         USE_NICE=""
188     else
189         USE_NICE="nice"
190     fi
191 }
192
193 # MAIN PROGAM #
194
195 # default values for arguments and options
196 LOG_DIRECTORY=""
197 EXTENDED_DEBUG=0
198 VERBOSE=0
199 ARCHIVE_DIR=archive
200 KEEP_DAYS=7
201 COMPRESS=gzip
202 USE_NICE=0
203 OVERWRITE_ARCHIVE=0
204 REMOVE_LOGS=0
205
206 # Set locale to sane one, to speed up comparisons, and be sure that < and > on
207 # strings work ok.
208 export LC_ALL=C
209
210 # Read arguments from command line
211 read_arguments "$@"
212
213 # Print settings
214 verbose_msg "$0 Settings:
215   - ARCHIVE_DIR       : $ARCHIVE_DIR
216   - COMPRESS          : $COMPRESS
217   - EXTENDED_DEBUG    : $EXTENDED_DEBUG
218   - KEEP_DAYS         : $KEEP_DAYS
219   - LOG_DIRECTORY     : $LOG_DIRECTORY
220   - OVERWRITE_ARCHIVE : $OVERWRITE_ARCHIVE
221   - USE_NICE          : $USE_NICE
222   - VERBOSE           : $VERBOSE
223   - REMOVE_LOGS       : $REMOVE_LOGS
224 "
225
226 # Make sure every error past this line is critical - this is to avoid having to
227 # check return codes from all calls to external programs
228 set -e
229
230 # Turn on extended debug
231 if (( $EXTENDED_DEBUG == 1 ))
232 then
233     set -x
234 fi
235
236 # Border date - any log from this date or newer has to stay uncompressed
237 # Have to use perl instead of more natural date --date="..." because
238 # Solaris date doesn't support --date="" option
239 export KEEP_DAYS
240 BORDER_DATE=$( perl -MPOSIX=strftime -e 'print strftime(q{%Y-%m-%d}, localtime( time() - $ENV{"KEEP_DAYS"} * 24 * 60 * 60 ))' )
241 verbose_msg "Border date: %s\n" "$BORDER_DATE"
242
243 # Find all log files, sort them (to work on them in order), and process
244 # Have to use grep, because Solaris find doesn't have -mindepth nor
245 # -maxdepth options
246 # Using ls could be an option, but it tends to write messages to STDERR if
247 # there are no matching files - not something that we would like.
248 find "$LOG_DIRECTORY"/ -type f \( -name 'postgresql-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].log' -o -name 'postgresql-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_[0-9][0-9][0-9][0-9][0-9][0-9].log' \) -print | \
249     egrep "^${LOG_DIRECTORY}/postgresql-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9](_[0-9][0-9][0-9][0-9][0-9][0-9])?.log" | \
250     sort | \
251     while read SOURCE_FILENAME
252     do
253         FILENAME=$( basename "$SOURCE_FILENAME" )
254         # skip files that are not older than $BORDER_DATE
255         if [[ ! "$FILENAME" < "postgresql-$BORDER_DATE" ]]
256         then
257             continue
258         fi
259
260         verbose_msg "Archiving file %-32s ... " "$FILENAME"
261
262         if [[ "$REMOVE_LOGS" -eq "1" ]]
263         then
264             rm "$SOURCE_FILENAME"
265             verbose_msg "File removed.\n"
266             continue
267         fi
268
269         # Extract year and month from log filename - to be used in archive path
270         YEAR_MONTH=$( echo "$FILENAME" | cut -d- -f2,3 )
271         if [[ ! -d "$ARCHIVE_DIR/$YEAR_MONTH" ]]
272         then
273             mkdir -p "$ARCHIVE_DIR/$YEAR_MONTH"
274         fi
275
276         DESTINATION_FILENAME="${ARCHIVE_DIR}/${YEAR_MONTH}/${FILENAME}.$( get_compress_extension )"
277
278         # Handle overriding
279         if [[ -e "${DESTINATION_FILENAME}" ]]
280         then
281             if (( $OVERWRITE_ARCHIVE == 0 ))
282             then
283                 printf "Destination file %s already exists. Skipping source %s.\n" "${DESTINATION_FILENAME}" "${SOURCE_FILENAME}" >&2
284                 continue
285             fi
286             verbose_msg "Destination file %s already exists. Overwriting. ... " "${DESTINATION_FILENAME}"
287         fi
288
289         # Have to use perl, because on Solaris, there is no stat command
290         export SOURCE_FILENAME
291         SIZE_BEFORE=$( perl -e 'print( (stat( $ENV{"SOURCE_FILENAME"} ))[7] )' )
292
293         # Actual archiving - compress to destination file, and remove of source
294         # I could also do compress without redirection, (x.log to x.log.gz) and
295         # then rename of temporary file, but this requires additional checks
296         # for existence of temporary file. Not worth the trouble
297         $USE_NICE $COMPRESS -c "$SOURCE_FILENAME" > "$DESTINATION_FILENAME"
298         rm -f "$SOURCE_FILENAME"
299
300         # Have to use perl, because on Solaris, there is no stat command
301         export DESTINATION_FILENAME
302         SIZE_AFTER=$( perl -e 'print( (stat( $ENV{"DESTINATION_FILENAME"} ))[7] )' )
303
304         verbose_msg "done. Size changed from %u to %u.\n" "$SIZE_BEFORE" "$SIZE_AFTER"
305     done
306
307 verbose_msg "All done.\n"
Note: See TracBrowser for help on using the browser.