Georg/OOMAnalyser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Restructure regex for better error messages

Browse Source
This commit is contained in:
Carsten Grohmann 2021-09-14 20:18:37 +02:00
parent dd8b806bd3
commit 6528b96adb
2 changed files with 117 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
OOMAnalyser.py
View File

@ -353,115 +353,115 @@ class OOMEntity(object):
class OOMAnalyser(object): class OOMAnalyser(object):
"""Analyse an OOM object and calculate additional values""" """Analyse an OOM object and calculate additional values"""
REC_INVOKED_OOMKILLER = re.compile( EXTRACT_PATTERN = {
r'^(?P<trigger_proc_name>[\S ]+) invoked oom-killer: ' 'invoked oom-killer': (
r'gfp_mask=(?P<trigger_proc_gfp_mask>0x[a-z0-9]+)(\((?P<trigger_proc_gfp_flags>[A-Z_|]+)\))?, ' r'^(?P<trigger_proc_name>[\S ]+) invoked oom-killer: '
r'(nodemask=(?P<trigger_proc_nodemask>([\d,-]+|\(null\))), )?' r'gfp_mask=(?P<trigger_proc_gfp_mask>0x[a-z0-9]+)(\((?P<trigger_proc_gfp_flags>[A-Z_|]+)\))?, '
r'order=(?P<trigger_proc_order>\d+), ' r'(nodemask=(?P<trigger_proc_nodemask>([\d,-]+|\(null\))), )?'
r'oom_score_adj=(?P<trigger_proc_oomscore>\d+)', r'order=(?P<trigger_proc_order>\d+), '
re.MULTILINE) r'oom_score_adj=(?P<trigger_proc_oomscore>\d+)',
True,
),
'Trigger process and kernel version': (
r'^CPU: \d+ PID: (?P<trigger_proc_pid>\d+) '
r'Comm: .* (Not tainted|Tainted:.*) '
r'(?P<kernel_version>\d[\w.-]+) #\d',
True,
),
REC_PID_KERNELVERSION = re.compile( # split caused by a limited number of iterations during converting PY regex into JS regex
r'^CPU: \d+ PID: (?P<trigger_proc_pid>\d+) ' 'Mem-Info (part 1)': (
r'Comm: .* (Not tainted|Tainted:.*) ' r'^Mem-Info:.*'
r'(?P<kernel_version>\d[\w.-]+) #\d', r'(?:\n)'
re.MULTILINE
)
# split caused by a limited number of iterations during converting PY regex into JS regex # first line (starting w/o a space)
REC_MEMINFO_1 = re.compile( r'^active_anon:(?P<active_anon_pages>\d+) inactive_anon:(?P<inactive_anon_pages>\d+) '
# head line r'isolated_anon:(?P<isolated_anon_pages>\d+)'
r'^Mem-Info:.*' r'(?:\n)'
# first line break # remaining lines (w/ leading space)
r'(?:\n)' r'^ active_file:(?P<active_file_pages>\d+) inactive_file:(?P<inactive_file_pages>\d+) '
r'isolated_file:(?P<isolated_file_pages>\d+)'
r'(?:\n)'
# first line (starting with a space) r'^ unevictable:(?P<unevictable_pages>\d+) dirty:(?P<dirty_pages>\d+) writeback:(?P<writeback_pages>\d+) '
r'^active_anon:(?P<active_anon_pages>\d+) inactive_anon:(?P<inactive_anon_pages>\d+) ' r'unstable:(?P<unstable_pages>\d+)',
r'isolated_anon:(?P<isolated_anon_pages>\d+)' True,
),
# next line break 'Mem-Info (part 2)': (
r'(?:\n)' r'^ slab_reclaimable:(?P<slab_reclaimable_pages>\d+) slab_unreclaimable:(?P<slab_unreclaimable_pages>\d+)'
r'(?:\n)'
# remaining lines (with leading space) r'^ mapped:(?P<mapped_pages>\d+) shmem:(?P<shmem_pages>\d+) pagetables:(?P<pagetables_pages>\d+) '
r'^ active_file:(?P<active_file_pages>\d+) inactive_file:(?P<inactive_file_pages>\d+) ' r'bounce:(?P<bounce_pages>\d+)'
r'isolated_file:(?P<isolated_file_pages>\d+)' r'(?:\n)'
r'^ free:(?P<free_pages>\d+) free_pcp:(?P<free_pcp_pages>\d+) free_cma:(?P<free_cma_pages>\d+)',
# next line break True,
r'(?:\n)' ),
'Memory node information': (
r'^ unevictable:(?P<unevictable_pages>\d+) dirty:(?P<dirty_pages>\d+) writeback:(?P<writeback_pages>\d+) ' r'(^Node \d+ (DMA|Normal|hugepages).*(:?\n))+',
r'unstable:(?P<unstable_pages>\d+)' False,
),
# # next line break 'Page cache': (
# r'(?:\n)' r'^(?P<pagecache_total_pages>\d+) total pagecache pages.*$',
# True,
, re.MULTILINE ),
) 'Swap usage information': (
r'^(?P<swap_cache_pages>\d+) pages in swap cache'
REC_MEMINFO_2 = re.compile( r'(?:\n)'
r'^ slab_reclaimable:(?P<slab_reclaimable_pages>\d+) slab_unreclaimable:(?P<slab_unreclaimable_pages>\d+)' r'^Swap cache stats: add \d+, delete \d+, find \d+\/\d+'
r'(?:\n)' r'(?:\n)'
r'^ mapped:(?P<mapped_pages>\d+) shmem:(?P<shmem_pages>\d+) pagetables:(?P<pagetables_pages>\d+) ' r'^Free swap = (?P<swap_free_kb>\d+)kB'
r'bounce:(?P<bounce_pages>\d+)' r'(?:\n)'
r'(?:\n)' r'^Total swap = (?P<swap_total_kb>\d+)kB',
r'^ free:(?P<free_pages>\d+) free_pcp:(?P<free_pcp_pages>\d+) free_cma:(?P<free_cma_pages>\d+)', True,
re.MULTILINE ),
) 'Page information': (
r'^(?P<ram_pages>\d+) pages RAM'
REC_MEM_NODEINFO = re.compile(r'(^Node \d+ (DMA|Normal|hugepages).*(:?\n))+', re.MULTILINE) r'('
r'(?:\n)'
REC_PAGECACHE = re.compile(r'^(?P<pagecache_total_pages>\d+) total pagecache pages.*$', re.MULTILINE) r'^(?P<highmem_pages>\d+) pages HighMem/MovableOnly'
r')?'
REC_SWAP = re.compile( r'(?:\n)'
r'^(?P<swap_cache_pages>\d+) pages in swap cache' r'^(?P<reserved_pages>\d+) pages reserved'
r'(?:\n)' r'('
r'^Swap cache stats: add \d+, delete \d+, find \d+\/\d+' r'(?:\n)'
r'(?:\n)' r'^(?P<cma_pages>\d+) pages cma reserved'
r'^Free swap = (?P<swap_free_kb>\d+)kB' r')?'
r'(?:\n)' r'('
r'^Total swap = (?P<swap_total_kb>\d+)kB', r'(?:\n)'
re.MULTILINE) r'^(?P<pagetablecache_pages>\d+) pages in pagetable cache'
r')?'
REC_PAGEINFO = re.compile( r'('
r'^(?P<ram_pages>\d+) pages RAM' r'(?:\n)'
r'(' r'^(?P<hwpoisoned_pages>\d+) pages hwpoisoned'
r'(?:\n)' r')?',
r'^(?P<highmem_pages>\d+) pages HighMem/MovableOnly' True,
r')?' ),
r'(?:\n)' 'Process killed by OOM': (
r'^(?P<reserved_pages>\d+) pages reserved' r'^Out of memory: Kill process (?P<killed_proc_pid>\d+) \((?P<killed_proc_name>[\S ]+)\) '
r'(' r'score (?P<killed_proc_score>\d+) or sacrifice child',
r'(?:\n)' True,
r'^(?P<cma_pages>\d+) pages cma reserved' ),
r')?' 'Details of process killed by OOM': (
r'(' r'^Killed process \d+ \(.*\)'
r'(?:\n)' r'(, UID \d+,)?'
r'^(?P<pagetablecache_pages>\d+) pages in pagetable cache' r' total-vm:(?P<killed_proc_total_vm_kb>\d+)kB, anon-rss:(?P<killed_proc_anon_rss_kb>\d+)kB, '
r')?' r'file-rss:(?P<killed_proc_file_rss_kb>\d+)kB, shmem-rss:(?P<killed_proc_shmem_rss_kb>\d+)kB.*',
r'(' True,
r'(?:\n)' ),
r'^(?P<hwpoisoned_pages>\d+) pages hwpoisoned' }
r')?', """
re.MULTILINE) RE pattern to extract information from OOM.
The first item is the RE pattern and the second is whether it is mandatory to find this pattern.
:type: dict(tuple(str, bool))
"""
REC_PROCESS_LINE = re.compile( REC_PROCESS_LINE = re.compile(
r'^\[(?P<pid>[ \d]+)\]\s+(?P<uid>\d+)\s+(?P<tgid>\d+)\s+(?P<total_vm_pages>\d+)\s+(?P<rss_pages>\d+)\s+' r'^\[(?P<pid>[ \d]+)\]\s+(?P<uid>\d+)\s+(?P<tgid>\d+)\s+(?P<total_vm_pages>\d+)\s+(?P<rss_pages>\d+)\s+'
r'(?P<nr_ptes_pages>\d+)\s+(?P<swapents_pages>\d+)\s+(?P<oom_score_adj>-?\d+)\s+(?P<name>.+)\s*') r'(?P<nr_ptes_pages>\d+)\s+(?P<swapents_pages>\d+)\s+(?P<oom_score_adj>-?\d+)\s+(?P<name>.+)\s*')
REC_OOM_KILL_PROCESS = re.compile(
r'^Out of memory: Kill process (?P<killed_proc_pid>\d+) \((?P<killed_proc_name>[\S ]+)\) '
r'score (?P<killed_proc_score>\d+) or sacrifice child',
re.MULTILINE
)
REC_KILLED_PROCESS = re.compile(
r'^Killed process \d+ \(.*\)'
r'(, UID \d+,)?'
r' total-vm:(?P<killed_proc_total_vm_kb>\d+)kB, anon-rss:(?P<killed_proc_anon_rss_kb>\d+)kB, '
r'file-rss:(?P<killed_proc_file_rss_kb>\d+)kB, shmem-rss:(?P<killed_proc_shmem_rss_kb>\d+)kB.*',
re.MULTILINE)
lines = [] lines = []
"""All lines of an OOM without leading timestamps""" """All lines of an OOM without leading timestamps"""
@ -500,26 +500,17 @@ class OOMAnalyser(object):
"""Extract details from OOM message text""" """Extract details from OOM message text"""
self.results = {} self.results = {}
# __pragma__ ('jsiter')
for rec in [self.REC_INVOKED_OOMKILLER, for k in self.EXTRACT_PATTERN:
self.REC_KILLED_PROCESS, pattern, is_mandatory = self.EXTRACT_PATTERN[k]
self.REC_MEMINFO_1, rec = re.compile(pattern, re.MULTILINE)
self.REC_MEMINFO_2,
self.REC_OOM_KILL_PROCESS,
self.REC_PAGECACHE,
self.REC_PAGEINFO,
self.REC_PID_KERNELVERSION,
self.REC_SWAP,
]:
match = rec.search(self.oom_entity.text) match = rec.search(self.oom_entity.text)
if match: if match:
self.results.update(match.groupdict()) self.results.update(match.groupdict())
else: elif is_mandatory:
warning('No match for regex: "{}"'.format(rec.pattern)) error('Failed to extract information from OOM text. The regular expression "{}" (pattern "{}") '
'does not find anything. This will cause subsequent errors.'.format(k, pattern))
match = self.REC_MEM_NODEINFO.search(self.oom_entity.text) # __pragma__ ('nojsiter')
if match:
self.results['mem_node_info'] = match.group()
self.results['hardware_info'] = self._extract_block_from_next_pos('Hardware name:') self.results['hardware_info'] = self._extract_block_from_next_pos('Hardware name:')

19
test.py
View File

@ -20,6 +20,7 @@
import http.server import http.server
import logging import logging
import os import os
import re
import socketserver import socketserver
import threading import threading
import unittest import unittest
@ -286,28 +287,26 @@ class TestPython(TestBase):
def test_001_trigger_proc_space(self): def test_001_trigger_proc_space(self):
"""Test RE to find name of trigger process""" """Test RE to find name of trigger process"""
first = self.get_first_line(OOMAnalyser.OOMDisplay.example) first = self.get_first_line(OOMAnalyser.OOMDisplay.example)
rec = OOMAnalyser.OOMAnalyser.REC_INVOKED_OOMKILLER pattern = OOMAnalyser.OOMAnalyser.EXTRACT_PATTERN['invoked oom-killer'][0]
rec = re.compile(pattern, re.MULTILINE)
match = rec.search(first) match = rec.search(first)
self.assertTrue(match, 'Error: re.search(REC_INVOKED_OOMKILLER) failed for simple ' self.assertTrue(match, "Error: re.search('invoked oom-killer') failed for simple process name")
'process name')
first = first.replace('sed', 'VM Monitoring Task') first = first.replace('sed', 'VM Monitoring Task')
match = rec.search(first) match = rec.search(first)
self.assertTrue(match, 'Error: re.search(REC_INVOKED_OOMKILLER) failed for process name ' self.assertTrue(match, "Error: re.search('invoked oom-killer') failed for process name with space")
'with space')
def test_002_killed_proc_space(self): def test_002_killed_proc_space(self):
"""Test RE to find name of killed process""" """Test RE to find name of killed process"""
last = self.get_last_line(OOMAnalyser.OOMDisplay.example) last = self.get_last_line(OOMAnalyser.OOMDisplay.example)
rec = OOMAnalyser.OOMAnalyser.REC_OOM_KILL_PROCESS pattern = OOMAnalyser.OOMAnalyser.EXTRACT_PATTERN['Process killed by OOM'][0]
rec = re.compile(pattern, re.MULTILINE)
match = rec.search(last) match = rec.search(last)
self.assertTrue(match, 'Error: re.search(REC_OOM_KILL_PROCESS) failed for simple ' self.assertTrue(match, "Error: re.search('Process killed by OOM') failed for simple process name")
'process name')
last = last.replace('sed', 'VM Monitoring Task') last = last.replace('sed', 'VM Monitoring Task')
match = rec.search(last) match = rec.search(last)
self.assertTrue(match, 'Error: re.search(REC_OOM_KILL_PROCESS) failed for process name ' self.assertTrue(match, "Error: re.search('Process killed by OOM') failed for process name with space")
'with space')
def test_003_OOMEntity_number_of_columns_to_strip(self): def test_003_OOMEntity_number_of_columns_to_strip(self):
"""Test stripping useless / leading columns""" """Test stripping useless / leading columns"""