Georg/OOMAnalyser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Restructure code to analyse OOM

Browse Source 
This code changes should simplify future extension of the analysis and
calculation code.
This commit is contained in:
Carsten Grohmann 2021-09-27 21:06:47 +02:00
parent f0b4ac895c
commit 7af0c1d7e8
2 changed files with 362 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

456
OOMAnalyser.py
View File

@ -54,8 +54,9 @@ class element():
# __pragma__ ('noskip') # __pragma__ ('noskip')
class OOMEntityState(object): class OOMEntityState:
"""Simple enum to track the completeness of an OOM block""" """Enum for completeness of the OOM block"""
unknown = 0
empty = 1 empty = 1
invalid = 2 invalid = 2
started = 3 started = 3
@ -127,8 +128,11 @@ def show_notifybox(prefix, msg):
notify_box.appendChild(notification) notify_box.appendChild(notification)
class KernelConfig(object): class BaseKernelConfig:
"""Kernel configuration""" """Base class for all kernel specific configuration"""
name = 'Base configuration for all kernels'
"""Name/description of this kernel configuration"""
GFP_FLAGS = { GFP_FLAGS = {
'GFP_ATOMIC': {'value': '__GFP_HIGH | __GFP_ATOMIC | __GFP_KSWAPD_RECLAIM'}, 'GFP_ATOMIC': {'value': '__GFP_HIGH | __GFP_ATOMIC | __GFP_KSWAPD_RECLAIM'},
@ -189,8 +193,38 @@ class KernelConfig(object):
'oom_score_adj'] 'oom_score_adj']
"""Elements of the process table""" """Elements of the process table"""
rec_version4kconfig = re.compile('.+')
"""RE to match kernel version to kernel configuration"""
class OOMEntity(object): rec_oom_begin = re.compile('invoked oom-killer:', re.MULTILINE)
"""RE to match the first line of an OOM block"""
rec_oom_end = re.compile('^Killed process \d+', re.MULTILINE)
"""RE to match the last line of an OOM block"""
class KernelConfigRhel7(BaseKernelConfig):
"""RHEL7 / CentOS7 specific configuration"""
name = 'RHEL7 / CentOS7 specific kernel configuration'
rec_version4kconfig = re.compile('^3\..+')
AllKernelConfigs = [
KernelConfigRhel7(),
BaseKernelConfig(),
]
"""
Instances of all available kernel configurations.
The last entry in this list is the base configuration as a fallback.
@type: List(BaseKernelConfig)
"""
class OOMEntity:
"""Hold whole OOM message block and provide access""" """Hold whole OOM message block and provide access"""
current_line = 0 current_line = 0
@ -199,7 +233,7 @@ class OOMEntity(object):
lines = [] lines = []
"""OOM text as list of lines""" """OOM text as list of lines"""
state = "unknown" state = OOMEntityState.unknown
"""State of the OOM after initial parsing""" """State of the OOM after initial parsing"""
text = "" text = ""
@ -370,7 +404,45 @@ class OOMEntity(object):
return self.next() return self.next()
class OOMAnalyser(object): class OOMResult:
"""Results of an OOM analysis"""
kconfig = BaseKernelConfig()
"""Kernel configuration"""
details = {}
"""Extracted result"""
oom_entity = None
"""
State of this OOM (unknown, incomplete, ...)
:type: OOMEntityState
"""
error_msg = ""
"""
Error message
@type: str
"""
kversion = None
"""
Kernel version
@type: str
"""
oom_text = None
"""
OOM text
@type: str
"""
class OOMAnalyser:
"""Analyse an OOM object and calculate additional values""" """Analyse an OOM object and calculate additional values"""
EXTRACT_PATTERN = { EXTRACT_PATTERN = {
@ -482,21 +554,91 @@ class OOMAnalyser(object):
r'^\[(?P<pid>[ \d]+)\]\s+(?P<uid>\d+)\s+(?P<tgid>\d+)\s+(?P<total_vm_pages>\d+)\s+(?P<rss_pages>\d+)\s+' r'^\[(?P<pid>[ \d]+)\]\s+(?P<uid>\d+)\s+(?P<tgid>\d+)\s+(?P<total_vm_pages>\d+)\s+(?P<rss_pages>\d+)\s+'
r'(?P<nr_ptes_pages>\d+)\s+(?P<swapents_pages>\d+)\s+(?P<oom_score_adj>-?\d+)\s+(?P<name>.+)\s*') r'(?P<nr_ptes_pages>\d+)\s+(?P<swapents_pages>\d+)\s+(?P<oom_score_adj>-?\d+)\s+(?P<name>.+)\s*')
lines = []
"""All lines of an OOM without leading timestamps"""
kernel_cfg = KernelConfig()
"""Kernel configuration"""
results = {}
"""Extracted result"""
oom_entity = None oom_entity = None
"""Reference to the OOMEntity object""" """
State of this OOM (unknown, incomplete, ...)
:type: OOMEntityState
"""
oom_result = None
"""
Store details of OOM analysis
:type: OOMResult
"""
def __init__(self, oom): def __init__(self, oom):
self.results = {}
self.oom_entity = oom self.oom_entity = oom
self.oom_result = OOMResult()
def _identify_kernel_version(self):
"""
Identify the used kernel version and
@rtype: bool
"""
pattern = r'CPU: \d+ PID: \d+ Comm: .* (Not tainted|Tainted: [A-Z ]+) (?P<kernel_version>\d[\w.-]+) #.+'
rec = re.compile(pattern, re.MULTILINE)
match = rec.search(self.oom_entity.text)
if not match:
self.oom_result.error_msg = 'Failed to extract kernel version from OOM text'
return False
self.oom_result.kversion = match.group('kernel_version')
return True
def _choose_kernel_config(self):
"""
Select proper kernel configuration
@rtype: bool
"""
for kcfg in AllKernelConfigs:
match = kcfg.rec_version4kconfig.match(self.oom_result.kversion)
if match:
self.oom_result.kconfig = kcfg
break
if not self.oom_result.kconfig:
warning('Failed to find a proper configuration for kernel "{}"'.format(self.oom_result.kversion))
self.oom_result.kconfig = BaseKernelConfig()
return True
def _check_for_empty_oom(self):
"""
Check for an empty OOM text
@rtype: bool
"""
if not self.oom_entity.text:
self.state = OOMEntityState.empty
self.oom_result.error_msg = 'Empty OOM text. Please insert an OOM message block.'
return False
return True
def _check_for_complete_oom(self):
"""
Check if the OOM in self.oom_entity is complete and update self.oom_state accordingly
@rtype: bool
"""
self.oom_state = OOMEntityState.unknown
self.oom_result.error_msg = 'Unknown OOM format'
if not self.oom_result.kconfig.rec_oom_begin.search(self.oom_entity.text):
self.state = OOMEntityState.invalid
self.oom_result.error_msg = 'The inserted text is not a valid OOM block! The initial pattern was not found!'
return False
if not self.oom_result.kconfig.rec_oom_end.search(self.oom_entity.text):
self.state = OOMEntityState.started
self.oom_result.error_msg = 'The inserted OOM is incomplete! The initial pattern was found ' \
'but not the final. The result may be incomplete!'
return False
self.state = OOMEntityState.complete
self.oom_result.error_msg = None
return True
def _extract_block_from_next_pos(self, marker): def _extract_block_from_next_pos(self, marker):
""" """
@ -519,20 +661,20 @@ class OOMAnalyser(object):
def _extract_from_oom_text(self): def _extract_from_oom_text(self):
"""Extract details from OOM message text""" """Extract details from OOM message text"""
self.results = {} self.oom_result.details = {}
# __pragma__ ('jsiter') # __pragma__ ('jsiter')
for k in self.EXTRACT_PATTERN: for k in self.EXTRACT_PATTERN:
pattern, is_mandatory = self.EXTRACT_PATTERN[k] pattern, is_mandatory = self.EXTRACT_PATTERN[k]
rec = re.compile(pattern, re.MULTILINE) rec = re.compile(pattern, re.MULTILINE)
match = rec.search(self.oom_entity.text) match = rec.search(self.oom_entity.text)
if match: if match:
self.results.update(match.groupdict()) self.oom_result.details.update(match.groupdict())
elif is_mandatory: elif is_mandatory:
error('Failed to extract information from OOM text. The regular expression "{}" (pattern "{}") ' error('Failed to extract information from OOM text. The regular expression "{}" (pattern "{}") '
'does not find anything. This will cause subsequent errors.'.format(k, pattern)) 'does not find anything. This will cause subsequent errors.'.format(k, pattern))
# __pragma__ ('nojsiter') # __pragma__ ('nojsiter')
self.results['hardware_info'] = self._extract_block_from_next_pos('Hardware name:') self.oom_result.details['hardware_info'] = self._extract_block_from_next_pos('Hardware name:')
# strip "Call Trace" line at beginning and remove leading spaces # strip "Call Trace" line at beginning and remove leading spaces
call_trace = '' call_trace = ''
@ -541,10 +683,10 @@ class OOMAnalyser(object):
if line.startswith('Call Trace'): if line.startswith('Call Trace'):
continue continue
call_trace += "{}\n".format(line.strip()) call_trace += "{}\n".format(line.strip())
self.results['call_trace'] = call_trace self.oom_result.details['call_trace'] = call_trace
# extract process table # extract process table
self.results['_ps'] = {} self.oom_result.details['_ps'] = {}
self.oom_entity.find_text('[ pid ]') self.oom_entity.find_text('[ pid ]')
for line in self.oom_entity: for line in self.oom_entity:
if not line.startswith('['): if not line.startswith('['):
@ -556,8 +698,8 @@ class OOMAnalyser(object):
details = match.groupdict() details = match.groupdict()
details['notes'] = '' details['notes'] = ''
pid = details.pop('pid') pid = details.pop('pid')
self.results['_ps'][pid] = {} self.oom_result.details['_ps'][pid] = {}
self.results['_ps'][pid].update(details) self.oom_result.details['_ps'][pid].update(details)
def _hex2flags(self, hexvalue, flag_definition): def _hex2flags(self, hexvalue, flag_definition):
"""\ """\
@ -633,22 +775,22 @@ class OOMAnalyser(object):
def _convert_numeric_results_to_integer(self): def _convert_numeric_results_to_integer(self):
"""Convert all *_pages and *_kb to integer""" """Convert all *_pages and *_kb to integer"""
# __pragma__ ('jsiter') # __pragma__ ('jsiter')
for item in self.results: for item in self.oom_result.details:
if self.results[item] is None: if self.oom_result.details[item] is None:
self.results[item] = '<not found>' self.oom_result.details[item] = '<not found>'
continue continue
if item.endswith('_kb') or item.endswith('_pages') or item.endswith('_pid') or \ if item.endswith('_kb') or item.endswith('_pages') or item.endswith('_pid') or \
item in ['killed_proc_score', 'trigger_proc_order', 'trigger_proc_oomscore']: item in ['killed_proc_score', 'trigger_proc_order', 'trigger_proc_oomscore']:
try: try:
self.results[item] = int(self.results[item]) self.oom_result.details[item] = int(self.oom_result.details[item])
except: except:
error('Converting item "{}={}" to integer failed'.format(item, self.results[item])) error('Converting item "{}={}" to integer failed'.format(item, self.oom_result.details[item]))
# __pragma__ ('nojsiter') # __pragma__ ('nojsiter')
def _convert_numeric_process_values_to_integer(self): def _convert_numeric_process_values_to_integer(self):
"""Convert numeric values in process table to integer values""" """Convert numeric values in process table to integer values"""
ps = self.results['_ps'] ps = self.oom_result.details['_ps']
ps_index = [] ps_index = []
# TODO Check if transcrypt issue: pragma jsiter for the whole block "for pid_str in ps: ..." # TODO Check if transcrypt issue: pragma jsiter for the whole block "for pid_str in ps: ..."
# sets item in "for item in ['uid',..." to 0 instead of 'uid' # sets item in "for item in ['uid',..." to 0 instead of 'uid'
@ -656,7 +798,7 @@ class OOMAnalyser(object):
for pid_str in ps.keys(): for pid_str in ps.keys():
converted = {} converted = {}
process = ps[pid_str] process = ps[pid_str]
for item in self.kernel_cfg.ps_table_items: for item in self.oom_result.kconfig.ps_table_items:
if item == 'pid': if item == 'pid':
continue continue
try: try:
@ -672,87 +814,87 @@ class OOMAnalyser(object):
ps_index.append(pid_int) ps_index.append(pid_int)
ps_index.sort(key=int) ps_index.sort(key=int)
self.results['_ps_index'] = ps_index self.oom_result.details['_ps_index'] = ps_index
def _calc_pstable_values(self): def _calc_pstable_values(self):
"""Set additional notes to processes listed in the process table""" """Set additional notes to processes listed in the process table"""
tpid = self.results['trigger_proc_pid'] tpid = self.oom_result.details['trigger_proc_pid']
kpid = self.results['killed_proc_pid'] kpid = self.oom_result.details['killed_proc_pid']
# sometimes the trigger process isn't part of the process table # sometimes the trigger process isn't part of the process table
if tpid in self.results['_ps']: if tpid in self.oom_result.details['_ps']:
self.results['_ps'][tpid]['notes'] = 'trigger process' self.oom_result.details['_ps'][tpid]['notes'] = 'trigger process'
# assume the killed process may also not part of the process table # assume the killed process may also not part of the process table
if kpid in self.results['_ps']: if kpid in self.oom_result.details['_ps']:
self.results['_ps'][kpid]['notes'] = 'killed process' self.oom_result.details['_ps'][kpid]['notes'] = 'killed process'
def _calc_trigger_process_values(self): def _calc_trigger_process_values(self):
"""Calculate all values related with the trigger process""" """Calculate all values related with the trigger process"""
self.results['trigger_proc_requested_memory_pages'] = 2 ** self.results['trigger_proc_order'] self.oom_result.details['trigger_proc_requested_memory_pages'] = 2 ** self.oom_result.details['trigger_proc_order']
self.results['trigger_proc_requested_memory_pages_kb'] = self.results['trigger_proc_requested_memory_pages'] * \ self.oom_result.details['trigger_proc_requested_memory_pages_kb'] = self.oom_result.details['trigger_proc_requested_memory_pages'] * \
self.results['page_size_kb'] self.oom_result.details['page_size_kb']
# process gfp_mask # process gfp_mask
if self.results['trigger_proc_gfp_flags'] != '<not found>': # None has been is converted to '<not found>' if self.oom_result.details['trigger_proc_gfp_flags'] != '<not found>': # None has been is converted to '<not found>'
flags = self.results['trigger_proc_gfp_flags'] flags = self.oom_result.details['trigger_proc_gfp_flags']
del self.results['trigger_proc_gfp_flags'] del self.oom_result.details['trigger_proc_gfp_flags']
else: else:
flags, unknown = self._hex2flags(self.results['trigger_proc_gfp_mask'], self.kernel_cfg.GFP_FLAGS) flags, unknown = self._hex2flags(self.oom_result.details['trigger_proc_gfp_mask'], self.oom_result.kconfig.GFP_FLAGS)
if unknown: if unknown:
flags.append('0x{0:x}'.format(unknown)) flags.append('0x{0:x}'.format(unknown))
flags = ' | '.join(flags) flags = ' | '.join(flags)
self.results['trigger_proc_gfp_mask'] = '{} ({})'.format(self.results['trigger_proc_gfp_mask'], flags) self.oom_result.details['trigger_proc_gfp_mask'] = '{} ({})'.format(self.oom_result.details['trigger_proc_gfp_mask'], flags)
# already fully processed and no own element to display -> delete otherwise an error msg will be shown # already fully processed and no own element to display -> delete otherwise an error msg will be shown
del self.results['trigger_proc_gfp_flags'] del self.oom_result.details['trigger_proc_gfp_flags']
def _calc_killed_process_values(self): def _calc_killed_process_values(self):
"""Calculate all values related with the killed process""" """Calculate all values related with the killed process"""
self.results['killed_proc_total_rss_kb'] = self.results['killed_proc_anon_rss_kb'] + \ self.oom_result.details['killed_proc_total_rss_kb'] = self.oom_result.details['killed_proc_anon_rss_kb'] + \
self.results['killed_proc_file_rss_kb'] + \ self.oom_result.details['killed_proc_file_rss_kb'] + \
self.results['killed_proc_shmem_rss_kb'] self.oom_result.details['killed_proc_shmem_rss_kb']
self.results['killed_proc_rss_percent'] = int(100 * self.oom_result.details['killed_proc_rss_percent'] = int(100 *
self.results['killed_proc_total_rss_kb'] / self.oom_result.details['killed_proc_total_rss_kb'] /
int(self.results['system_total_ram_kb'])) int(self.oom_result.details['system_total_ram_kb']))
def _calc_swap_values(self): def _calc_swap_values(self):
"""Calculate all swap related values""" """Calculate all swap related values"""
self.results['swap_cache_kb'] = self.results['swap_cache_pages'] * self.results['page_size_kb'] self.oom_result.details['swap_cache_kb'] = self.oom_result.details['swap_cache_pages'] * self.oom_result.details['page_size_kb']
del self.results['swap_cache_pages'] del self.oom_result.details['swap_cache_pages']
# SwapUsed = SwapTotal - SwapFree - SwapCache # SwapUsed = SwapTotal - SwapFree - SwapCache
self.results['swap_used_kb'] = self.results['swap_total_kb'] - self.results['swap_free_kb'] - \ self.oom_result.details['swap_used_kb'] = self.oom_result.details['swap_total_kb'] - self.oom_result.details['swap_free_kb'] - \
self.results['swap_cache_kb'] self.oom_result.details['swap_cache_kb']
self.results['system_swap_used_percent'] = int(100 * self.oom_result.details['system_swap_used_percent'] = int(100 *
self.results['swap_total_kb'] / self.oom_result.details['swap_total_kb'] /
self.results['swap_used_kb']) self.oom_result.details['swap_used_kb'])
def _calc_system_values(self): def _calc_system_values(self):
"""Calculate system memory""" """Calculate system memory"""
# educated guess # educated guess
self.results['page_size_kb'] = 4 self.oom_result.details['page_size_kb'] = 4
# calculate remaining explanation values # calculate remaining explanation values
self.results['system_total_ram_kb'] = self.results['ram_pages'] * self.results['page_size_kb'] self.oom_result.details['system_total_ram_kb'] = self.oom_result.details['ram_pages'] * self.oom_result.details['page_size_kb']
self.results['system_total_ramswap_kb'] = self.results['system_total_ram_kb'] + self.results['swap_total_kb'] self.oom_result.details['system_total_ramswap_kb'] = self.oom_result.details['system_total_ram_kb'] + self.oom_result.details['swap_total_kb']
total_rss_pages = 0 total_rss_pages = 0
for pid in self.results['_ps'].keys(): for pid in self.oom_result.details['_ps'].keys():
total_rss_pages += self.results['_ps'][pid]['rss_pages'] total_rss_pages += self.oom_result.details['_ps'][pid]['rss_pages']
self.results['system_total_ram_used_kb'] = total_rss_pages * self.results['page_size_kb'] self.oom_result.details['system_total_ram_used_kb'] = total_rss_pages * self.oom_result.details['page_size_kb']
self.results['system_total_used_percent'] = int(100 * self.oom_result.details['system_total_used_percent'] = int(100 *
self.results['system_total_ram_used_kb'] / self.oom_result.details['system_total_ram_used_kb'] /
self.results['system_total_ram_kb']) self.oom_result.details['system_total_ram_kb'])
def _determinate_platform_and_distribution(self): def _determinate_platform_and_distribution(self):
"""Determinate platform and distribution""" """Determinate platform and distribution"""
kernel_version = self.results.get('kernel_version', '') kernel_version = self.oom_result.details.get('kernel_version', '')
if 'x86_64' in kernel_version: if 'x86_64' in kernel_version:
self.results['platform'] = 'x86 64bit' self.oom_result.details['platform'] = 'x86 64bit'
else: else:
self.results['platform'] = 'unknown' self.oom_result.details['platform'] = 'unknown'
dist = 'unknown' dist = 'unknown'
if '.el7uek' in kernel_version: if '.el7uek' in kernel_version:
@ -767,13 +909,13 @@ class OOMAnalyser(object):
dist = 'Arch Linux' dist = 'Arch Linux'
elif '_generic' in kernel_version: elif '_generic' in kernel_version:
dist = 'Ubuntu' dist = 'Ubuntu'
self.results['dist'] = dist self.oom_result.details['dist'] = dist
def _calc_from_oom_details(self): def _calc_from_oom_details(self):
""" """
Calculate values from already extracted details Calculate values from already extracted details
@see: self.results @see: self.details
""" """
self._convert_numeric_results_to_integer() self._convert_numeric_results_to_integer()
self._convert_numeric_process_values_to_integer() self._convert_numeric_process_values_to_integer()
@ -786,20 +928,52 @@ class OOMAnalyser(object):
self._calc_swap_values() self._calc_swap_values()
def analyse(self): def analyse(self):
"""Extract and calculate values from the given OOM object""" """
Extract and calculate values from the given OOM object
If the return value is False, the OOM is too incomplete to perform an analysis.
@rtype: bool
"""
if not self._check_for_empty_oom():
error(self.oom_result.error_msg)
return False
if not self._identify_kernel_version():
error(self.oom_result.error_msg)
return False
if not self._choose_kernel_config():
error(self.oom_result.error_msg)
return False
if not self._check_for_complete_oom():
error(self.oom_result.error_msg)
return False
# 4. extract values
self._extract_from_oom_text() self._extract_from_oom_text()
# 5. calculate details
self._calc_from_oom_details() self._calc_from_oom_details()
return self.results
# 6. store results
self.oom_result.oom_text = self.oom_entity.text
# 7. return results
return True
class OOMDisplay(object): class OOMDisplay:
"""Display the OOM analysis""" """Display the OOM analysis"""
kernel_cfg = KernelConfig() # result ergibt an manchen stellen self.result.result :-/
"""Kernel configuration""" oom_result = OOMResult()
"""
OOM analysis details
oom_details = {} @rtype: OOMResult
"""Extracted result""" """
example = u'''\ example = u'''\
sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0 sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
@ -1022,7 +1196,7 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
""" """
elements = document.getElementsByClassName(item) elements = document.getElementsByClassName(item)
for element in elements: for element in elements:
content = self.oom_details.get(item, '') content = self.oom_result.details.get(item, '')
if isinstance(content, str): if isinstance(content, str):
content = content.strip() content = content.strip()
@ -1077,14 +1251,14 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
new_table = '' new_table = ''
table_content = document.getElementById('process_table') table_content = document.getElementById('process_table')
for pid in self.oom_details['_ps_index']: for pid in self.oom_result.details['_ps_index']:
if pid == self.oom_details['trigger_proc_pid']: if pid == self.oom_result.details['trigger_proc_pid']:
css_class = 'class="js-pstable__triggerproc--bgcolor"' css_class = 'class="js-pstable__triggerproc--bgcolor"'
elif pid == self.oom_details['killed_proc_pid']: elif pid == self.oom_result.details['killed_proc_pid']:
css_class = 'class="js-pstable__killedproc--bgcolor"' css_class = 'class="js-pstable__killedproc--bgcolor"'
else: else:
css_class = '' css_class = ''
process = self.oom_details['_ps'][pid] process = self.oom_result.details['_ps'][pid]
line = """ line = """
<tr {}> <tr {}>
<td>{}</td> <td>{}</td>
@ -1111,10 +1285,10 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
"""Set the sorting symbols for all columns in the process table""" """Set the sorting symbols for all columns in the process table"""
# TODO Check operator overloading # TODO Check operator overloading
# Operator overloading (Pragma opov) does not work in this context. # Operator overloading (Pragma opov) does not work in this context.
# self.kernel_cfg.ps_table_items + ['notes'] will compile to a string # self.oom_result.kconfig.ps_table_items + ['notes'] will compile to a string
# "pid,uid,tgid,total_vm_pages,rss_pages,nr_ptes_pages,swapents_pages,oom_score_adjNotes" and not to an # "pid,uid,tgid,total_vm_pages,rss_pages,nr_ptes_pages,swapents_pages,oom_score_adjNotes" and not to an
# array # array
ps_table_and_notes = self.kernel_cfg.ps_table_items[:] ps_table_and_notes = self.oom_result.kconfig.ps_table_items[:]
ps_table_and_notes.append('notes') ps_table_and_notes.append('notes')
for column_name in ps_table_and_notes: for column_name in ps_table_and_notes:
element_id = "pstable_sort_{}".format(column_name) element_id = "pstable_sort_{}".format(column_name)
@ -1283,59 +1457,41 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
def analyse_and_show(self): def analyse_and_show(self):
"""Analyse the OOM text inserted into the form and show the results""" """Analyse the OOM text inserted into the form and show the results"""
self.oom = OOMEntity(self.load_from_form()) self.oom = OOMEntity(self.load_from_form())
if not self.is_valid(self.oom):
self.oom = None
return
# set defaults and clear notifications # set defaults and clear notifications
self.oom_details.clear()
self.set_HTML_defaults() self.set_HTML_defaults()
# analyse
analyser = OOMAnalyser(self.oom) analyser = OOMAnalyser(self.oom)
self.oom_details = analyser.analyse() success = analyser.analyse()
# Update kernel configuration if success:
self.kernel_cfg = analyser.kernel_cfg self.oom_result = analyser.oom_result
self.show_oom_details()
# display results self.update_toc()
self.show() else:
self.update_toc() # don't show results - just return
return
def load_from_form(self): def load_from_form(self):
"""
Return the OOM text from textarea element
@rtype: str
"""
element = document.getElementById('textarea_oom') element = document.getElementById('textarea_oom')
oom_text = element.value oom_text = element.value
return oom_text return oom_text
def is_valid(self, oom): def show_oom_details(self):
"""
Return True for a complete OOM otherwise False and a warning msg for a incomplete or an error msg
if the start sequence was not found.
"""
if oom.state == OOMEntityState.complete:
return True
elif oom.state == OOMEntityState.started:
warning('The inserted OOM is incomplete!')
warning('The initial pattern was found but not the final. The result may be incomplete!')
elif oom.state == OOMEntityState.invalid:
error('The inserted text is not a valid OOM block!')
error('The initial pattern was not found!')
elif oom.state == OOMEntityState.empty:
error('The inserted text is empty! Please insert an OOM message block.')
else:
error('Invalid state "{}" after the OOM has formally checked!'.format(self.oom.state))
return False
def show(self):
""" """
Show all extracted details as well as additionally generated information Show all extracted details as well as additionally generated information
""" """
if DEBUG: if DEBUG:
print(self.oom_details) print(self.oom_result.details)
hide_element('input') hide_element('input')
show_element('analysis') show_element('analysis')
for item in self.oom_details.keys(): for item in self.oom_result.details.keys():
# ignore internal items # ignore internal items
if item.startswith('_'): if item.startswith('_'):
continue continue
@ -1347,9 +1503,9 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
# generate swap usage diagram # generate swap usage diagram
svg_swap = self.svg_generate_bar_chart( svg_swap = self.svg_generate_bar_chart(
self.svg_colors_swap, self.svg_colors_swap,
('Swap Used', self.oom_details['swap_used_kb']), ('Swap Used', self.oom_result.details['swap_used_kb']),
('Swap Free', self.oom_details['swap_free_kb']), ('Swap Free', self.oom_result.details['swap_free_kb']),
('Swap Cached', self.oom_details['swap_cache_kb']), ('Swap Cached', self.oom_result.details['swap_cache_kb']),
) )
elem_svg_swap = document.getElementById('svg_swap') elem_svg_swap = document.getElementById('svg_swap')
elem_svg_swap.appendChild(svg_swap) elem_svg_swap.appendChild(svg_swap)
@ -1357,41 +1513,41 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
# generate RAM usage diagram # generate RAM usage diagram
svg_ram = self.svg_generate_bar_chart( svg_ram = self.svg_generate_bar_chart(
self.svg_colors_mem, self.svg_colors_mem,
('Active mem', self.oom_details['active_anon_pages']), ('Active mem', self.oom_result.details['active_anon_pages']),
('Inactive mem', self.oom_details['inactive_anon_pages']), ('Inactive mem', self.oom_result.details['inactive_anon_pages']),
('Isolated mem', self.oom_details['isolated_anon_pages']), ('Isolated mem', self.oom_result.details['isolated_anon_pages']),
('Active PC', self.oom_details['active_file_pages']), ('Active PC', self.oom_result.details['active_file_pages']),
('Inactive PC', self.oom_details['inactive_file_pages']), ('Inactive PC', self.oom_result.details['inactive_file_pages']),
('Isolated PC', self.oom_details['isolated_file_pages']), ('Isolated PC', self.oom_result.details['isolated_file_pages']),
('Unevictable', self.oom_details['unevictable_pages']), ('Unevictable', self.oom_result.details['unevictable_pages']),
('Dirty', self.oom_details['dirty_pages']), ('Dirty', self.oom_result.details['dirty_pages']),
('Writeback', self.oom_details['writeback_pages']), ('Writeback', self.oom_result.details['writeback_pages']),
('Unstable', self.oom_details['unstable_pages']), ('Unstable', self.oom_result.details['unstable_pages']),
('Slab reclaimable', self.oom_details['slab_reclaimable_pages']), ('Slab reclaimable', self.oom_result.details['slab_reclaimable_pages']),
('Slab unreclaimable', self.oom_details['slab_unreclaimable_pages']), ('Slab unreclaimable', self.oom_result.details['slab_unreclaimable_pages']),
('Mapped', self.oom_details['mapped_pages']), ('Mapped', self.oom_result.details['mapped_pages']),
('Shared', self.oom_details['shmem_pages']), ('Shared', self.oom_result.details['shmem_pages']),
('Pagetable', self.oom_details['pagetables_pages']), ('Pagetable', self.oom_result.details['pagetables_pages']),
('Bounce', self.oom_details['bounce_pages']), ('Bounce', self.oom_result.details['bounce_pages']),
('Free', self.oom_details['free_pages']), ('Free', self.oom_result.details['free_pages']),
('Free PCP', self.oom_details['free_pcp_pages']), ('Free PCP', self.oom_result.details['free_pcp_pages']),
('Free CMA', self.oom_details['free_cma_pages']), ('Free CMA', self.oom_result.details['free_cma_pages']),
) )
elem_svg_ram = document.getElementById('svg_ram') elem_svg_ram = document.getElementById('svg_ram')
elem_svg_ram.appendChild(svg_ram) elem_svg_ram.appendChild(svg_ram)
element = document.getElementById('oom') element = document.getElementById('oom')
element.textContent = self.oom.text element.textContent = self.oom_result.oom_text
self.toggle_oom(show=False) self.toggle_oom(show=False)
def sort_pstable(self, column_name): def sort_pstable(self, column_name):
"""Sort process table by the values in the given column""" """Sort process table by the values in the given column"""
# TODO Check operator overloading # TODO Check operator overloading
# Operator overloading (Pragma opov) does not work in this context. # Operator overloading (Pragma opov) does not work in this context.
# self.kernel_cfg.ps_table_items + ['notes'] will compile to a string # self.oom_result.kconfig.ps_table_items + ['notes'] will compile to a string
# "pid,uid,tgid,total_vm_pages,rss_pages,nr_ptes_pages,swapents_pages,oom_score_adjNotes" and not to an # "pid,uid,tgid,total_vm_pages,rss_pages,nr_ptes_pages,swapents_pages,oom_score_adjNotes" and not to an
# array # array
ps_table_and_notes = self.kernel_cfg.ps_table_items[:] ps_table_and_notes = self.oom_result.kconfig.ps_table_items[:]
ps_table_and_notes.append('notes') ps_table_and_notes.append('notes')
if column_name not in ps_table_and_notes: if column_name not in ps_table_and_notes:
internal_error('Can not sort process table with an unknown column name "{}"'.format(column_name)) internal_error('Can not sort process table with an unknown column name "{}"'.format(column_name))
@ -1420,8 +1576,8 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
Is uses bubble sort with all disadvantages but just a few lines of code Is uses bubble sort with all disadvantages but just a few lines of code
""" """
ps = self.oom_details['_ps'] ps = self.oom_result.details['_ps']
ps_index = self.oom_details['_ps_index'] ps_index = self.oom_result.details['_ps_index']
def getvalue(column, pos): def getvalue(column, pos):
if column == 'pid': if column == 'pid':

87
test.py
View File

@ -18,7 +18,6 @@
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import http.server import http.server
import logging
import os import os
import re import re
import socketserver import socketserver
@ -111,13 +110,9 @@ class TestInBrowser(TestBase):
self.fail('Unexpected warning message: "%s"' % warning.text) self.fail('Unexpected warning message: "%s"' % warning.text)
def assert_on_error(self): def assert_on_error(self):
notify_box = self.driver.find_element_by_id('notify_box') error = self.get_error_text()
try: if error:
error = notify_box.find_element_by_class_name('js-notify_box__msg--error') self.fail('Unexpected error message: "%s"' % error)
except NoSuchElementException:
pass
else:
self.fail('Unexpected error message: "%s"' % error.text)
for event in self.driver.get_log('browser'): for event in self.driver.get_log('browser'):
# ignore favicon.ico errors # ignore favicon.ico errors
@ -133,14 +128,25 @@ class TestInBrowser(TestBase):
analyse = self.driver.find_element_by_xpath('//button[text()="Analyse"]') analyse = self.driver.find_element_by_xpath('//button[text()="Analyse"]')
analyse.click() analyse.click()
def get_error_text(self):
"""
Return text from error notification box or an empty string if no error message exists
:@rtype: str
"""
notify_box = self.driver.find_element_by_id('notify_box')
try:
notify_box.find_element_by_class_name('js-notify_box__msg--error')
except NoSuchElementException:
return ""
return notify_box.text
def click_reset(self): def click_reset(self):
# OOMAnalyser.OOMDisplayInstance.reset_form()
reset = self.driver.find_element_by_xpath('//button[text()="Reset"]') reset = self.driver.find_element_by_xpath('//button[text()="Reset"]')
if reset.is_displayed(): if reset.is_displayed():
reset.click() reset.click()
else: else:
new_analysis = self.driver.find_element_by_xpath('//a[contains(text(), "Step 1 - Enter your OOM message")]') new_analysis = self.driver.find_element_by_xpath('//a[contains(text(), "Step 1 - Enter your OOM message")]')
# new_analysis = self.driver.find_element_by_link_text('Run a new analysis')
new_analysis.click() new_analysis.click()
self.assert_on_warn_error() self.assert_on_warn_error()
@ -184,16 +190,16 @@ class TestInBrowser(TestBase):
swap_total_kb = self.driver.find_element_by_class_name('swap_total_kb') swap_total_kb = self.driver.find_element_by_class_name('swap_total_kb')
self.assertEqual(swap_total_kb.text, '8388604 kBytes') self.assertEqual(swap_total_kb.text, '8388604 kBytes')
def test_001_load_page(self): def test_010_load_page(self):
"""Test if the page is loading""" """Test if the page is loading"""
assert "OOM Analyser" in self.driver.title assert "OOM Analyser" in self.driver.title
def test_002_load_js(self): def test_020_load_js(self):
"""Test if JS is loaded""" """Test if JS is loaded"""
elem = self.driver.find_element_by_id("version") elem = self.driver.find_element_by_id("version")
self.assertIsNotNone(elem.text, "Version statement not set - JS not loaded") self.assertIsNotNone(elem.text, "Version statement not set - JS not loaded")
def test_003_insert_and_analyse_example(self): def test_030_insert_and_analyse_example(self):
"""Test loading and analysing example""" """Test loading and analysing example"""
textarea = self.driver.find_element_by_id('textarea_oom') textarea = self.driver.find_element_by_id('textarea_oom')
self.assertEqual(textarea.get_attribute('value'), '', 'Empty textarea expected') self.assertEqual(textarea.get_attribute('value'), '', 'Empty textarea expected')
@ -207,7 +213,22 @@ class TestInBrowser(TestBase):
self.click_analyse() self.click_analyse()
self.check_results() self.check_results()
def test_004_begin_but_no_end(self): def test_031_empty_textarea(self):
"""Test "Analyse" with empty textarea"""
textarea = self.driver.find_element_by_id('textarea_oom')
self.assertEqual(textarea.get_attribute('value'), '', 'Empty textarea expected')
# textarea.send_keys(text)
self.assertEqual(textarea.get_attribute('value'), '', 'Expected empty text area, but text found')
h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]')
self.assertFalse(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be not displayed")
self.click_analyse()
self.assertEqual(self.get_error_text(), "ERROR: Empty OOM text. Please insert an OOM message block.")
self.click_reset()
def test_032_begin_but_no_end(self):
"""Test incomplete OOM text - just the beginning""" """Test incomplete OOM text - just the beginning"""
example = """\ example = """\
sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0 sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
@ -215,52 +236,41 @@ sed cpuset=/ mems_allowed=0-1
CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1 CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
""" """
self.analyse_oom(example) self.analyse_oom(example)
self.assertEqual(self.get_error_text(), "ERROR: The inserted OOM is incomplete! The initial pattern was "
notify_box = self.driver.find_element_by_id('notify_box') "found but not the final. The result may be incomplete!")
notify_box.find_element_by_class_name('js-notify_box__msg--warning')
self.assertTrue(notify_box.text.startswith("WARNING: The inserted OOM is incomplete!"))
self.click_reset() self.click_reset()
def test_005_no_begin_but_end(self): def test_033_no_begin_but_end(self):
"""Test incomplete OOM text - just the end""" """Test incomplete OOM text - just the end"""
example = """\ example = """\
Out of memory: Kill process 6576 (java) score 651 or sacrifice child Out of memory: Kill process 6576 (java) score 651 or sacrifice child
Killed process 6576 (java) total-vm:33914892kB, anon-rss:20629004kB, file-rss:0kB, shmem-rss:0kB Killed process 6576 (java) total-vm:33914892kB, anon-rss:20629004kB, file-rss:0kB, shmem-rss:0kB
""" """
self.analyse_oom(example) self.analyse_oom(example)
self.assertEqual(self.get_error_text(), "ERROR: Failed to extract kernel version from OOM text")
notify_box = self.driver.find_element_by_id('notify_box')
notify_box.find_element_by_class_name('js-notify_box__msg--error')
self.assertTrue(notify_box.text.startswith("ERROR: The inserted text is not a valid OOM block!"))
self.click_reset() self.click_reset()
def test_006_trigger_proc_space(self): def test_040_trigger_proc_space(self):
"""Test trigger process name contains a space""" """Test trigger process name contains a space"""
example = OOMAnalyser.OOMDisplay.example example = OOMAnalyser.OOMDisplay.example
example = example.replace('sed', 'VM Monitoring Task') example = example.replace('sed', 'VM Monitoring Task')
self.analyse_oom(example) self.analyse_oom(example)
self.assert_on_warn_error() self.assert_on_warn_error()
h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]') h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]')
self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed") self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")
def test_007_kill_proc_space(self): def test_050_kill_proc_space(self):
"""Test killed process name contains a space""" """Test killed process name contains a space"""
example = OOMAnalyser.OOMDisplay.example example = OOMAnalyser.OOMDisplay.example
example = example.replace('mysqld', 'VM Monitoring Task') example = example.replace('mysqld', 'VM Monitoring Task')
self.analyse_oom(example) self.analyse_oom(example)
self.assert_on_warn_error() self.assert_on_warn_error()
h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]') h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]')
self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed") self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")
def test_008_removal_of_leading_but_useless_columns(self): def test_060_removal_of_leading_but_useless_columns(self):
"""Test removal of leading but useless columns""" """Test removal of leading but useless columns"""
self.analyse_oom(OOMAnalyser.OOMDisplay.example) self.analyse_oom(OOMAnalyser.OOMDisplay.example)
self.check_results() self.check_results()
@ -333,6 +343,19 @@ Hardware name: HP ProLiant DL385 G7, BIOS A18 12/08/2012
''' '''
self.assertEqual(text, expected) self.assertEqual(text, expected)
def test_005_extract_kernel_version(self):
"""Test extracting kernel version"""
oom = OOMAnalyser.OOMEntity(OOMAnalyser.OOMDisplay.example)
analyser = OOMAnalyser.OOMAnalyser(oom)
for text, kversion in [
('CPU: 0 PID: 19163 Comm: kworker/0:0 Tainted: G OE 5.4.0-80-lowlatency #90~18.04.1-Ubuntu', '5.4.0-80-lowlatency'),
('CPU: 4 PID: 1 Comm: systemd Not tainted 3.10.0-1062.9.1.el7.x86_64 #1', '3.10.0-1062.9.1.el7.x86_64'),
]:
analyser.oom_entity.text = text
success = analyser._identify_kernel_version()
self.assertTrue(analyser._identify_kernel_version(), analyser.oom_result.error_msg)
self.assertEqual(analyser.oom_result.kversion, kversion)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main(verbosity=2) unittest.main(verbosity=2)