summaryrefslogtreecommitdiffstats
path: root/src/plugins/KerneloopsSysLog.c
blob: 162fd9774e3cfcfa2bc546f067a6d7e0c262a006 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
/*
    Copyright (C) 2010  ABRT team
    Copyright (C) 2010  RedHat Inc

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    Authors:
       Anton Arapov <anton@redhat.com>
       Arjan van de Ven <arjan@linux.intel.com>
 */
#include "abrtlib.h"
#include "KerneloopsSysLog.h"
#include <glib.h>

static void queue_oops(GList **vec, const char *data, const char *version)
{
    char *ver_data = xasprintf("%s\n%s", version, data);
    *vec = g_list_append(*vec, ver_data);
}

/*
 * extract_version tries to find the kernel version in given data
 */
static char *extract_version(const char *linepointer)
{
    if (strstr(linepointer, "Pid")
     || strstr(linepointer, "comm")
     || strstr(linepointer, "CPU")
     || strstr(linepointer, "REGS")
     || strstr(linepointer, "EFLAGS")
    ) {
        char* start;
        char* end;

        start = strstr((char*)linepointer, "2.6.");
        if (start)
        {
            end = strchr(start, ')');
            if (!end)
                end = strchrnul(start, ' ');
            return xstrndup(start, end-start);
        }
    }

    return NULL;
}

/*
 * extract_oops tries to find oops signatures in a log
 */
struct line_info {
    char *ptr;
    char level;
};

static int record_oops(GList **oopses, struct line_info* lines_info, int oopsstart, int oopsend)
{
    int q;
    int len;
    char *oops;
    char *version;

    len = 2;
    for (q = oopsstart; q <= oopsend; q++)
            len += strlen(lines_info[q].ptr) + 1;

    oops = (char*)xzalloc(len);

    version = NULL;
    for (q = oopsstart; q <= oopsend; q++)
    {
        if (!version)
            version = extract_version(lines_info[q].ptr);

        if (lines_info[q].ptr[0])
        {
            strcat(oops, lines_info[q].ptr);
            strcat(oops, "\n");
        }
    }
    int rv = 1;
    /* too short oopses are invalid */
    if (strlen(oops) > 100)
        queue_oops(oopses, oops, version ? version : "undefined");
    else
    {
        VERB3 log("Dropped oops: too short");
        rv = 0;
    }
    free(oops);
    free(version);
    return rv;
}
#define REALLOC_CHUNK 1000
int extract_oopses(GList **oopses, char *buffer, size_t buflen)
{
    char *c;
    int linecount = 0;
    int lines_info_alloc = 0;
    struct line_info *lines_info = NULL;

    /* Split buffer into lines */

    if (buflen != 0)
            buffer[buflen - 1] = '\n';  /* the buffer usually ends with \n, but let's make sure */
    c = buffer;
    while (c < buffer + buflen)
    {
        char linelevel;
        char *c9;
        char *colon;

        c9 = (char*)memchr(c, '\n', buffer + buflen - c); /* a \n will always be found */
        assert(c9);
        *c9 = '\0'; /* turn the \n into a string termination */
        if (c9 == c)
            goto next_line;

        /* Is it a syslog file (/var/log/messages or similar)?
         * Even though _usually_ it looks like "Nov 19 12:34:38 localhost kernel: xxx",
         * some users run syslog in non-C locale:
         * "2010-02-22T09:24:08.156534-08:00 gnu-4 gnome-session[2048]: blah blah"
         *  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ !!!
         * We detect it by checking for N:NN:NN pattern in first 15 chars
         * (and this still is not good enough... false positive: "pci 0000:15:00.0: PME# disabled")
         */
        colon = strchr(c, ':');
        if (colon && colon > c && colon < c + 15
         && isdigit(colon[-1]) /* N:... */
         && isdigit(colon[1]) /* ...N:NN:... */
         && isdigit(colon[2])
         && colon[3] == ':'
         && isdigit(colon[4]) /* ...N:NN:NN... */
         && isdigit(colon[5])
        ) {
            /* It's syslog file, not a bare dmesg */

            /* Skip non-kernel lines */
            char *kernel_str = strstr(c, "kernel: ");
            if (kernel_str == NULL)
            {
                /* if we see our own marker:
                 * "hostname abrt: Kerneloops: Reported 1 kernel oopses to Abrt"
                 * we know we submitted everything upto here already */
                if (strstr(c, "abrt:") && strstr(c, "Abrt"))
                {
                    VERB3 log("Found our marker at line %d, restarting line count from 0", linecount);
                    linecount = 0;
                    lines_info_alloc = 0;
                    free(lines_info);
                    lines_info = NULL;
                }
                goto next_line;
            }
            c = kernel_str + sizeof("kernel: ")-1;
        }

        linelevel = 0;
        /* store and remove kernel log level */
        if (*c == '<' && c[1] && c[2] == '>')
        {
            linelevel = c[1];
            c += 3;
        }
        /* remove jiffies time stamp counter if present */
        if (*c == '[')
        {
            char *c2 = strchr(c, '.');
            char *c3 = strchr(c, ']');
            if (c2 && c3 && (c2 < c3) && (c3-c) < 14 && (c2-c) < 8)
            {
                c = c3 + 1;
                if (*c == ' ')
                        c++;
            }
        }
        if (linecount >= lines_info_alloc)
        {
            lines_info_alloc += REALLOC_CHUNK;
            lines_info = (struct line_info*)xrealloc(lines_info,
                            lines_info_alloc * sizeof(lines_info[0]));
        }
        lines_info[linecount].ptr = c;
        lines_info[linecount].level = linelevel;
        linecount++;
next_line:
        c = c9 + 1;
    }

    /* Analyze lines */

    int i;
    char prevlevel = 0;
    int oopsstart = -1;
    int inbacktrace = 0;
    int oopsesfound = 0;

    i = 0;
    while (i < linecount)
    {
        char *curline = lines_info[i].ptr;

        if (curline == NULL)
        {
            i++;
            continue;
        }
        while (*curline == ' ')
                curline++;

        if (oopsstart < 0)
        {
            /* Find start-of-oops markers */
            /* In some comparisons, we skip 1st letter, to avoid dealing with
             * changes in capitalization in kernel. For example, I see that
             * current kernel git (at 2011-01-01) has both "kernel BUG at ..."
             * and "Kernel BUG at ..." messages, and I don't want to change
             * the code below whenever kernel is changed to use "K" (or "k")
             * uniformly.
             */
            if (strstr(curline, /*g*/ "eneral protection fault:"))
                oopsstart = i;
            else if (strstr(curline, "BUG:"))
                oopsstart = i;
            else if (strstr(curline, /*k*/ "ernel BUG at"))
                oopsstart = i;
            else if (strstr(curline, "do_IRQ: stack overflow:"))
                oopsstart = i;
            else if (strstr(curline, "RTNL: assertion failed"))
                 oopsstart = i;
            else if (strstr(curline, /*e*/ "eek! page_mapcount(page) went negative!"))
                oopsstart = i;
            else if (strstr(curline, /*n*/ "ear stack overflow (cur:"))
                oopsstart = i;
            else if (strstr(curline, /*d*/ "ouble fault:"))
                oopsstart = i;
            else if (strstr(curline, /*b*/ "adness at"))
                oopsstart = i;
            else if (strstr(curline, "NETDEV WATCHDOG"))
                oopsstart = i;
            else if (strstr(curline, "WARNING: at ")) /* WARN_ON() generated message */
                oopsstart = i;
            else if (strstr(curline, /*u*/ "nable to handle kernel"))
                oopsstart = i;
            else if (strstr(curline, /*s*/ "ysctl table check failed"))
                oopsstart = i;
            else if (strstr(curline, "INFO: possible recursive locking detected"))
                oopsstart = i;
            // Not needed: "--[ cut here ]--" is always followed
            // by "Badness at", "kernel BUG at", or "WARNING: at" string
            //else if (strstr(curline, "------------[ cut here ]------------"))
            //	oopsstart = i;
            else if (strstr(curline, "list_del corruption"))
                oopsstart = i;
            else if (strstr(curline, "list_add corruption"))
                oopsstart = i;

            if (i >= 3 && strstr(curline, "Oops:"))
                oopsstart = i-3;

            if (oopsstart >= 0)
            {
                /* debug information */
                VERB3 {
                    log("Found oops at line %d: '%s'", oopsstart, lines_info[oopsstart].ptr);
                    if (oopsstart != i)
                            log("Trigger line is %d: '%s'", i, c);
                }
                /* try to find the end marker */
                int i2 = i + 1;
                while (i2 < linecount && i2 < (i+50))
                {
                    if (strstr(lines_info[i2].ptr, "---[ end trace"))
                    {
                        inbacktrace = 1;
                        i = i2;
                        break;
                    }
                    i2++;
                }
            }
        }

        /* Are we entering a call trace part? */
        /* a call trace starts with "Call Trace:" or with the " [<.......>] function+0xFF/0xAA" pattern */
        if (oopsstart >= 0 && !inbacktrace)
        {
            if (strstr(curline, "Call Trace:"))
                inbacktrace = 1;
            else
            if (strnlen(curline, 9) > 8
             && curline[0] == '[' && curline[1] == '<'
             && strstr(curline, ">]")
             && strstr(curline, "+0x")
             && strstr(curline, "/0x")
            ) {
                inbacktrace = 1;
            }
        }

        /* Are we at the end of an oops? */
        else if (oopsstart >= 0 && inbacktrace)
        {
            int oopsend = INT_MAX;

            /* line needs to start with " [" or have "] [" if it is still a call trace */
            /* example: "[<ffffffffa006c156>] radeon_get_ring_head+0x16/0x41 [radeon]" */
            if (curline[0] != '['
             && !strstr(curline, "] [")
             && !strstr(curline, "--- Exception")
             && !strstr(curline, "LR =")
             && !strstr(curline, "<#DF>")
             && !strstr(curline, "<IRQ>")
             && !strstr(curline, "<EOI>")
             && !strstr(curline, "<<EOE>>")
             && strncmp(curline, "Code: ", 6) != 0
             && strncmp(curline, "RIP ", 4) != 0
             && strncmp(curline, "RSP ", 4) != 0
            ) {
                oopsend = i-1; /* not a call trace line */
            }
            /* oops lines are always more than 8 chars long */
            else if (strnlen(curline, 8) < 8)
                oopsend = i-1;
            /* single oopses are of the same loglevel */
            else if (lines_info[i].level != prevlevel)
                oopsend = i-1;
            else if (strstr(curline, "Instruction dump:"))
                oopsend = i;
            /* if a new oops starts, this one has ended */
            else if (strstr(curline, "WARNING: at ") && oopsstart != i) /* WARN_ON() generated message */
                oopsend = i-1;
            else if (strstr(curline, "Unable to handle") && oopsstart != i)
                oopsend = i-1;
            /* kernel end-of-oops marker (not including marker itself) */
            else if (strstr(curline, "---[ end trace"))
                oopsend = i-1;

            if (oopsend <= i)
            {
                VERB3 log("End of oops at line %d (%d): '%s'", oopsend, i, lines_info[oopsend].ptr);
                if (record_oops(oopses, lines_info, oopsstart, oopsend))
                    oopsesfound++;
                oopsstart = -1;
                inbacktrace = 0;
            }
        }

        prevlevel = lines_info[i].level;
        i++;

        if (oopsstart >= 0)
        {
            /* Do we have a suspiciously long oops? Cancel it */
            if (i-oopsstart > 60)
            {
                inbacktrace = 0;
                oopsstart = -1;
                VERB3 log("Dropped oops, too long");
                continue;
            }
            if (!inbacktrace && i-oopsstart > 40)
            {
                /*inbacktrace = 0; - already is */
                oopsstart = -1;
                VERB3 log("Dropped oops, too long");
                continue;
            }
        }
    } /* while (i < linecount) */

    /* process last oops if we have one */
    if (oopsstart >= 0 && inbacktrace)
    {
        int oopsend = i-1;
        VERB3 log("End of oops at line %d (end of file): '%s'", oopsend, lines_info[oopsend].ptr);
        if (record_oops(oopses, lines_info, oopsstart, oopsend))
                oopsesfound++;
    }

    free(lines_info);
    return oopsesfound;
}