From 508e1d6585a37483cc5166824840fd6be776869d Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sat, 26 Mar 2011 04:57:45 +0100 Subject: abrt-action-trim-files: new tool for trimming old debuginfo and debug dumps Signed-off-by: Denys Vlasenko --- abrt.spec | 3 +- src/daemon/abrt-server.c | 3 +- src/hooks/abrt-hook-ccpp.c | 20 ++-- src/include/abrtlib.h | 1 + src/lib/hooklib.c | 38 +++++-- src/lib/hooklib.h | 4 +- src/plugins/CCpp.conf | 7 -- src/plugins/Makefile.am | 20 ++++ src/plugins/abrt-action-kerneloops.c | 4 +- src/plugins/abrt-action-trim-files.c | 188 +++++++++++++++++++++++++++++++++++ src/plugins/ccpp_events.conf | 28 +++--- 11 files changed, 265 insertions(+), 51 deletions(-) create mode 100644 src/plugins/abrt-action-trim-files.c diff --git a/abrt.spec b/abrt.spec index 3a6c470c..ab6610cb 100644 --- a/abrt.spec +++ b/abrt.spec @@ -480,10 +480,11 @@ fi %defattr(-,root,root,-) %config(noreplace) %{_sysconfdir}/%{name}/plugins/CCpp.conf %dir %attr(0775, abrt, abrt) %{_localstatedir}/cache/abrt-di -%attr(2755, abrt, abrt) %{_bindir}/abrt-action-install-debuginfo %{_initrddir}/abrt-ccpp %{_libexecdir}/abrt-hook-ccpp %{_bindir}/abrt-action-analyze-c +%{_bindir}/abrt-action-trim-files +%attr(2755, abrt, abrt) %{_bindir}/abrt-action-install-debuginfo %{_bindir}/abrt-action-install-debuginfo.py* %{_bindir}/abrt-action-generate-backtrace %{_bindir}/abrt-action-list-dsos.py* diff --git a/src/daemon/abrt-server.c b/src/daemon/abrt-server.c index 9d4a2310..c9da64d6 100644 --- a/src/daemon/abrt-server.c +++ b/src/daemon/abrt-server.c @@ -16,7 +16,6 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include "abrtlib.h" -#include "hooklib.h" #include "parse_options.h" #define PROGNAME "abrt-server" @@ -153,7 +152,7 @@ static void create_debug_dump() if (maxCrashReportsSize > 0) { check_free_space(maxCrashReportsSize); - trim_debug_dumps(maxCrashReportsSize, path); + trim_debug_dumps(DEBUG_DUMPS_DIR, maxCrashReportsSize * (double)(1024*1024), path); } free(path); diff --git a/src/hooks/abrt-hook-ccpp.c b/src/hooks/abrt-hook-ccpp.c index ddb5920a..8be24b9b 100644 --- a/src/hooks/abrt-hook-ccpp.c +++ b/src/hooks/abrt-hook-ccpp.c @@ -19,7 +19,6 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include "abrtlib.h" -#include "hooklib.h" #include static char* malloc_readlink(const char *linkname) @@ -503,11 +502,11 @@ int main(int argc, char** argv) int base_name = sprintf(source_filename, "/proc/%lu/smaps", (long)pid); base_name -= strlen("smaps"); char *dest_filename = concat_path_file(dd->dd_dirname, FILENAME_SMAPS); - copy_file(source_filename, dest_filename, S_IRUSR | S_IRGRP | S_IWUSR); + copy_file(source_filename, dest_filename, 0640); chown(dest_filename, dd->dd_uid, dd->dd_gid); strcpy(source_filename + base_name, "maps"); strcpy(strrchr(dest_filename, '/') + 1, FILENAME_MAPS); - copy_file(source_filename, dest_filename, S_IRUSR | S_IRGRP | S_IWUSR); + copy_file(source_filename, dest_filename, 0640); chown(dest_filename, dd->dd_uid, dd->dd_gid); free(dest_filename); @@ -525,7 +524,8 @@ int main(int argc, char** argv) if (src_fd_binary > 0) { strcpy(path + path_len, "/"FILENAME_BINARY); - int dst_fd_binary = xopen3(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + int dst_fd_binary = xopen3(path, O_WRONLY | O_CREAT | O_TRUNC, 0640); + fchown(dst_fd_binary, dd->dd_uid, dd->dd_gid); off_t sz = copyfd_eof(src_fd_binary, dst_fd_binary, COPYFD_SPARSE); if (sz < 0 || fsync(dst_fd_binary) != 0) { @@ -536,13 +536,8 @@ int main(int argc, char** argv) close(src_fd_binary); } - /* We need coredumps to be readable by all, because - * when abrt daemon processes coredump, - * process producing backtrace is run under the same UID - * as the crashed process. - * Thus 644, not 600 */ strcpy(path + path_len, "/"FILENAME_COREDUMP); - int abrt_core_fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0644); + int abrt_core_fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0640); if (abrt_core_fd < 0) { int sv_errno = errno; @@ -555,6 +550,7 @@ int main(int argc, char** argv) errno = sv_errno; perror_msg_and_die("can't open '%s'", path); } + fchown(abrt_core_fd, dd->dd_uid, dd->dd_gid); /* We write both coredumps at once. * We can't write user coredump first, since it might be truncated @@ -608,13 +604,11 @@ int main(int argc, char** argv) /* rhbz#539551: "abrt going crazy when crashing process is respawned" */ if (setting_MaxCrashReportsSize > 0) { - trim_debug_dumps(setting_MaxCrashReportsSize, path); + trim_debug_dumps(DEBUG_DUMPS_DIR, setting_MaxCrashReportsSize * (double)(1024*1024), path); } return 0; } - else - xfunc_die(); /* We didn't create abrt dump, but may need to create compat coredump */ create_user_core: diff --git a/src/include/abrtlib.h b/src/include/abrtlib.h index f64046d9..d9364673 100644 --- a/src/include/abrtlib.h +++ b/src/include/abrtlib.h @@ -81,6 +81,7 @@ int vdprintf(int d, const char *format, va_list ap); #include "abrt_crash_data.h" #include "abrt_types.h" #include "dump_dir.h" +#include "hooklib.h" #include "run_event.h" #include "event_config.h" diff --git a/src/lib/hooklib.c b/src/lib/hooklib.c index a89ab008..fd7ecc9a 100644 --- a/src/lib/hooklib.c +++ b/src/lib/hooklib.c @@ -114,24 +114,40 @@ void check_free_space(unsigned setting_MaxCrashReportsSize) * Check total size of dump dir, if it overflows, * delete oldest/biggest dumps. */ -void trim_debug_dumps(unsigned setting_MaxCrashReportsSize, const char *exclude_path) +void trim_debug_dumps(const char *dirname, double cap_size, const char *exclude_path) { - int count = 10; + const char *excluded_basename = NULL; + if (exclude_path) + { + unsigned len_dirname = strlen(dirname); + /* Trim trailing '/'s, but dont trim name "/" to "" */ + while (len_dirname > 1 && dirname[len_dirname-1] == '/') + len_dirname--; + if (strncmp(dirname, exclude_path, len_dirname) == 0 + && exclude_path[len_dirname] == '/' + ) { + /* exclude_path is "dirname/something" */ + excluded_basename = exclude_path + len_dirname + 1; + } + } + VERB3 log("excluded_basename:'%s'", excluded_basename); + + int count = 20; while (--count >= 0) { - const char *base_dirname = strrchr(exclude_path, '/') + 1; /* never NULL */ /* We exclude our own dump from candidates for deletion (3rd param): */ - char *worst_dir = NULL; - double dirsize = get_dirsize_find_largest_dir(DEBUG_DUMPS_DIR, &worst_dir, base_dirname); - if (dirsize / (1024*1024) < setting_MaxCrashReportsSize || !worst_dir) + char *worst_basename = NULL; + double cur_size = get_dirsize_find_largest_dir(dirname, &worst_basename, excluded_basename); + if (cur_size <= cap_size || !worst_basename) { - free(worst_dir); + VERB2 log("cur_size:%f cap_size:%f, no (more) trimming", cur_size, cap_size); + free(worst_basename); break; } - log("size of '%s' >= %u MB, deleting '%s'", DEBUG_DUMPS_DIR, setting_MaxCrashReportsSize, worst_dir); - char *d = concat_path_file(DEBUG_DUMPS_DIR, worst_dir); - free(worst_dir); - worst_dir = NULL; + log("%s is %.0f bytes (more than %.0f MB), deleting '%s'", + dirname, cur_size, cap_size / (1024*1024), worst_basename); + char *d = concat_path_file(dirname, worst_basename); + free(worst_basename); delete_dump_dir(d); free(d); } diff --git a/src/lib/hooklib.h b/src/lib/hooklib.h index 84b31a5f..a7421e06 100644 --- a/src/lib/hooklib.h +++ b/src/lib/hooklib.h @@ -25,10 +25,12 @@ void parse_conf(const char *additional_conf, unsigned *setting_MaxCrashReportsSize, bool *setting_MakeCompatCore, bool *setting_SaveBinaryImage); + #define check_free_space abrt_check_free_space void check_free_space(unsigned setting_MaxCrashReportsSize); + #define trim_debug_dumps abrt_trim_debug_dumps -void trim_debug_dumps(unsigned setting_MaxCrashReportsSize, const char *exclude_path); +void trim_debug_dumps(const char *dirname, double cap_size, const char *exclude_path); #ifdef __cplusplus } diff --git a/src/plugins/CCpp.conf b/src/plugins/CCpp.conf index 4af91470..bf66d86f 100644 --- a/src/plugins/CCpp.conf +++ b/src/plugins/CCpp.conf @@ -21,9 +21,6 @@ GdbTimeoutSec = 120 # If you cannot ensure that, it's better to set this option to "no" BacktraceRemotes = no -# Generate memory map too (IGNORED FOR NOW) -MemoryMap = no - # How to get debuginfo: install, mount ## install - download and install debuginfo packages ## mount - mount fedora NFS with debug info @@ -38,7 +35,3 @@ InstallDebugInfo = yes # For example, you can list a network-mounted shared store # of all debuginfos here. # ReadonlyLocalDebugInfoDirs = /path1:/path2:... - -# Keep @@LOCALSTATEDIR@@/cache/abrt-di -# from growing out-of-bounds. -DebugInfoCacheMB = 4000 diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am index 959306ef..8dd618e6 100644 --- a/src/plugins/Makefile.am +++ b/src/plugins/Makefile.am @@ -9,6 +9,7 @@ bin_PROGRAMS = \ abrt-action-analyze-c \ abrt-action-analyze-python \ abrt-action-analyze-oops \ + abrt-action-trim-files \ abrt-action-generate-backtrace \ abrt-action-bugzilla \ abrt-action-rhtsupport \ @@ -142,6 +143,25 @@ abrt_action_analyze_oops_CPPFLAGS = \ abrt_action_analyze_oops_LDADD = \ ../lib/libreport.la +abrt_action_trim_files_SOURCES = \ + abrt-action-trim-files.c +abrt_action_trim_files_CPPFLAGS = \ + -I$(srcdir)/../include/report -I$(srcdir)/../include \ + -I$(srcdir)/../lib \ + -DBIN_DIR=\"$(bindir)\" \ + -DVAR_RUN=\"$(VAR_RUN)\" \ + -DCONF_DIR=\"$(CONF_DIR)\" \ + -DLOCALSTATEDIR='"$(localstatedir)"' \ + -DDEBUG_DUMPS_DIR=\"$(DEBUG_DUMPS_DIR)\" \ + -DDEBUG_INFO_DIR=\"$(DEBUG_INFO_DIR)\" \ + -DPLUGINS_LIB_DIR=\"$(PLUGINS_LIB_DIR)\" \ + -DPLUGINS_CONF_DIR=\"$(PLUGINS_CONF_DIR)\" \ + $(GLIB_CFLAGS) \ + -D_GNU_SOURCE \ + -Wall -Wwrite-strings -Werror +abrt_action_trim_files_LDADD = \ + ../lib/libreport.la + abrt_action_generate_backtrace_SOURCES = \ abrt-action-generate-backtrace.c abrt_action_generate_backtrace_CPPFLAGS = \ diff --git a/src/plugins/abrt-action-kerneloops.c b/src/plugins/abrt-action-kerneloops.c index 72ec80e3..5d46d157 100644 --- a/src/plugins/abrt-action-kerneloops.c +++ b/src/plugins/abrt-action-kerneloops.c @@ -130,7 +130,7 @@ int main(int argc, char **argv) /* Can't keep these strings/structs static: _() doesn't support that */ const char *program_usage_string = _( - PROGNAME" [-v] -c CONFFILE -d DIR\n" + PROGNAME" [-v] [-c CONFFILE]... -d DIR\n" "\n" "Reports kernel oops to kerneloops.org (or similar) site" ); @@ -143,7 +143,7 @@ int main(int argc, char **argv) struct options program_options[] = { OPT__VERBOSE(&g_verbose), OPT_STRING('d', NULL, &dump_dir_name, "DIR" , _("Dump directory")), - OPT_LIST( 'c', NULL, &conf_file , "FILE", _("Configuration file (may be given many times)")), + OPT_LIST( 'c', NULL, &conf_file , "FILE", _("Configuration file")), OPT_END() }; /*unsigned opts =*/ parse_opts(argc, argv, program_options, program_usage_string); diff --git a/src/plugins/abrt-action-trim-files.c b/src/plugins/abrt-action-trim-files.c new file mode 100644 index 00000000..f2bc280d --- /dev/null +++ b/src/plugins/abrt-action-trim-files.c @@ -0,0 +1,188 @@ +/* + Copyright (C) 2011 ABRT team + Copyright (C) 2011 RedHat Inc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ +#include "abrtlib.h" +#include "parse_options.h" + +#define PROGNAME "abrt-action-trim" + +static double get_dir_size(const char *dirname, + char **worst_file, + double *worst_file_size) +{ + DIR *dp = opendir(dirname); + if (!dp) + return 0; + + /* "now" is used only if caller wants to know worst_file */ + time_t now = worst_file ? time(NULL) : 0; + struct dirent *dent; + struct stat stats; + double size = 0; + while ((dent = readdir(dp)) != NULL) + { + if (dot_or_dotdot(dent->d_name)) + continue; + + char *fullname = concat_path_file(dirname, dent->d_name); + if (lstat(fullname, &stats) != 0) + { + free(fullname); + continue; + } + + if (S_ISDIR(stats.st_mode)) + { + double sz = get_dir_size(fullname, worst_file, worst_file_size); + size += sz; + } + else if (S_ISREG(stats.st_mode)) + { + double sz = stats.st_size; + size += sz; + + if (worst_file) + { + /* Calculate "weighted" size and age + * w = sz_kbytes * age_mins */ + sz /= 1024; + long age = (now - stats.st_mtime) / 60; + if (age > 0) + sz *= age; + + if (sz > *worst_file_size) + { + *worst_file_size = sz; + free(*worst_file); + *worst_file = fullname; + fullname = NULL; + } + } + } + free(fullname); + } + closedir(dp); + + return size; +} + +static const char *parse_size_pfx(double *size, const char *str) +{ + errno = (isdigit(str[0]) ? 0 : ERANGE); + char *end; + *size = strtoull(str, &end, 10); + + if (end != str) + { + char c = (*end | 0x20); /* lowercasing */ + if (c == 'k') + end++, *size *= 1024; + else if (c == 'm') + end++, *size *= 1024*1024; + else if (c == 'g') + end++, *size *= 1024*1024*1024; + else if (c == 't') + end++, *size *= 1024.0*1024*1024*1024; + } + + if (errno || end == str || *end != ':') + perror_msg_and_die("Bad size prefix in '%s'", str); + + return end + 1; +} + +static void delete_dirs(gpointer data, gpointer exclude_path) +{ + double cap_size; + const char *dir = parse_size_pfx(&cap_size, data); + + trim_debug_dumps(dir, cap_size, exclude_path); +} + +static void delete_files(gpointer data, gpointer user_data_unused) +{ + double cap_size; + const char *dir = parse_size_pfx(&cap_size, data); + + unsigned count = 1000; + while (--count != 0) + { + char *worst_file = NULL; + double worst_file_size = 0; + double cur_size = get_dir_size(dir, &worst_file, &worst_file_size); + if (cur_size <= cap_size) + { + VERB2 log("cur_size:%f cap_size:%f, no (more) trimming", cur_size, cap_size); + free(worst_file); + break; + } + log("%s is %.0f bytes (more than %.0f MB), deleting '%s'", + dir, cur_size, cap_size / (1024*1024), worst_file); + if (unlink(worst_file) != 0) + perror_msg("Can't unlink '%s'", worst_file); + free(worst_file); + } +} + +int main(int argc, char **argv) +{ + char *env_verbose = getenv("ABRT_VERBOSE"); + if (env_verbose) + g_verbose = atoi(env_verbose); + + GList *dir_list = NULL; + GList *file_list = NULL; + char *preserve = NULL; + + /* Can't keep these strings/structs static: _() doesn't support that */ + const char *program_usage_string = _( + PROGNAME" [-v] [-d SIZE:DIR]... [-f SIZE:DIR]... [-p DIR]\n" + "\n" + "Deletes dump dirs (-d) or files (-f) in DIRs until they are smaller than SIZE" + ); + enum { + OPT_v = 1 << 0, + OPT_d = 1 << 1, + OPT_f = 1 << 2, + OPT_p = 1 << 3, + }; + /* Keep enum above and order of options below in sync! */ + struct options program_options[] = { + OPT__VERBOSE(&g_verbose), + OPT_LIST( 'd', NULL, &dir_list , "SIZE:DIR", _("Delete dump dirs")), + OPT_LIST( 'f', NULL, &file_list, "SIZE:DIR", _("Delete files")), + OPT_STRING('p', NULL, &preserve , "DIR" , _("Preserve this dump dir")), + OPT_END() + }; + + /*unsigned opts =*/ parse_opts(argc, argv, program_options, program_usage_string); + argv += optind; + if (argv[0] || !(dir_list || file_list)) + show_usage_and_die(program_usage_string, program_options); + + putenv(xasprintf("ABRT_VERBOSE=%u", g_verbose)); + + char *pfx = getenv("ABRT_PROG_PREFIX"); + if (pfx && string_to_bool(pfx)) + msg_prefix = PROGNAME; + + g_list_foreach(dir_list, delete_dirs, preserve); + g_list_foreach(file_list, delete_files, NULL); + + return 0; +} diff --git a/src/plugins/ccpp_events.conf b/src/plugins/ccpp_events.conf index 5dee5727..0698e655 100644 --- a/src/plugins/ccpp_events.conf +++ b/src/plugins/ccpp_events.conf @@ -1,19 +1,19 @@ -EVENT=post-create analyzer=CCpp abrt-action-analyze-c -EVENT=post-create analyzer=CCpp abrt-action-list-dsos.py > dsos +EVENT=post-create analyzer=CCpp abrt-action-analyze-c +EVENT=post-create analyzer=CCpp abrt-action-list-dsos.py > dsos -#TODO: implement this (or add this functionality to abrt-action-install-debuginfo): -#EVENT=analyze analyzer=CCpp backtrace= trim-debuginfo-cache /var/cache/abrt-di 4096m - -#TODO: can we still specify additional directories to search for debuginfos, +# We run analyze_foo steps only if backtrace is empty (not yet generated): +EVENT=analyze_LocalGDB analyzer=CCpp backtrace= abrt-action-trim-files -f 4096m:/var/cache/abrt-di +# TODO: can we still specify additional directories to search for debuginfos, # or was this ability lost with move to python installer? -EVENT=analyze_LocalGDB analyzer=CCpp backtrace= abrt-action-install-debuginfo --core="$DUMP_DIR/coredump" -EVENT=analyze_LocalGDB analyzer=CCpp backtrace= abrt-action-generate-backtrace -EVENT=analyze_RetraceServer analyzer=CCpp backtrace= abrt-retrace-client batch -k --dir "$DUMP_DIR" +EVENT=analyze_LocalGDB analyzer=CCpp backtrace= abrt-action-install-debuginfo --core="$DUMP_DIR/coredump" +EVENT=analyze_LocalGDB analyzer=CCpp backtrace= abrt-action-generate-backtrace +EVENT=analyze_RetraceServer analyzer=CCpp backtrace= abrt-retrace-client batch -k --dir "$DUMP_DIR" # Same as "analyze", but executed when user requests "refresh" in GUI -#EVENT=reanalyze analyzer=CCpp trim-debuginfo-cache /var/cache/abrt-di 4096m -EVENT=reanalyze_LocalGDB analyzer=CCpp abrt-action-install-debuginfo --core="$DUMP_DIR/coredump" -EVENT=reanalyze_LocalGDB analyzer=CCpp abrt-action-generate-backtrace -EVENT=reanalyze_RetraceServer analyzer=CCpp abrt-retrace-client batch -k --dir "$DUMP_DIR" +# It doesn't check that backtrace is empty: +EVENT=reanalyze_LocalGDB analyzer=CCpp abrt-action-trim-files -f 4096m:/var/cache/abrt-di +EVENT=reanalyze_LocalGDB analyzer=CCpp abrt-action-install-debuginfo --core="$DUMP_DIR/coredump" +EVENT=reanalyze_LocalGDB analyzer=CCpp abrt-action-generate-backtrace +EVENT=reanalyze_RetraceServer analyzer=CCpp abrt-retrace-client batch -k --dir "$DUMP_DIR" -EVENT=report_Bugzilla analyzer=CCpp abrt-action-bugzilla -c /etc/abrt/plugins/Bugzilla.conf +EVENT=report_Bugzilla analyzer=CCpp abrt-action-bugzilla -c /etc/abrt/plugins/Bugzilla.conf -- cgit