Georg/OOMAnalyser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework removal of unused information

Browse Source 
The columns left to the oom are not used. With this change, the
algorithm becomes more robust and can better handle known special
cases like "kernel:<processname>".
This commit is contained in:
Carsten Grohmann 2021-07-24 00:28:38 +02:00
parent 1abf455711
commit 41a51a7e2a
3 changed files with 100 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
OOMAnalyser.html
View File

@ -818,6 +818,7 @@ function read_and_display_file(file) {
<li>Improve SVG chart colour palette</li>
<li>Add Selenium based unit tests</li>
<li>Fix to allow process names with spaces</li>
<li>Rework removal of unused information</li>
<li>...</li>
</ol>

38
OOMAnalyser.py
View File

@ -201,8 +201,9 @@ class OOMEntity(object):
return
oom_lines = self._remove_non_oom_lines(oom_lines)
oom_lines = self._strip_needless_columns(oom_lines)
oom_lines = self._remove_kernel_colon(oom_lines)
oom_lines = self._rsyslog_unescape_lf(oom_lines)
oom_lines = self._strip_needless_columns(oom_lines)
self.lines = oom_lines
self.text = '\n'.join(oom_lines)
@ -223,21 +224,13 @@ class OOMEntity(object):
columns = first_line.split(" ")
# Examples:
# [11686.888109] sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_adj=0, oom_score_adj=0
# Apr 01 14:13:32 mysrv kernel: sed invoked OOM-killer: gfp_mask=0x201da, order=0
# Apr 01 14:13:32 mysrv kernel: [11686.888109] sed invoked oom-killer: gfp_mask=0x84d0, order=0, oom_adj=0, oom_score_adj=0
# [11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
# Apr 01 14:13:32 mysrv kernel: CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
# Apr 01 14:13:32 mysrv kernel: [11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
try:
# strip all incl. "kernel:"
if 'kernel:' in first_line:
to_strip = columns.index("kernel:")
# increase to include "kernel:"
to_strip += 1
# check if next column is a timestamp like "[11686.888109]" and remove it too
rec = re.compile('\[\d+\.\d+\]')
if rec.match(columns[to_strip]):
# increase to include timestamp
to_strip += 1
# strip all excl. "CPU:"
if 'CPU:' in first_line:
to_strip = columns.index("CPU:")
except ValueError:
pass
@ -283,6 +276,16 @@ class OOMEntity(object):
return lines
def _remove_kernel_colon(self, oom_lines):
"""
Remove the "kernel:" pattern w/o leading and tailing spaces.
Some OOM messages don't have a space between "kernel:" and the process name. _strip_needless_columns() will
fail in such cases. Therefore the pattern is removed.
"""
oom_lines = [i.replace('kernel:', '') for i in oom_lines]
return oom_lines
def _strip_needless_columns(self, oom_lines):
"""
Remove needless columns at the start of every line.
@ -291,7 +294,7 @@ class OOMEntity(object):
syslog priority/facility.
"""
stripped_lines = []
cols_to_strip = self._number_of_columns_to_strip(oom_lines[0])
cols_to_strip = self._number_of_columns_to_strip(oom_lines[2])
for line in oom_lines:
# remove empty lines
@ -412,7 +415,7 @@ class OOMAnalyser(object):
REC_SWAP = re.compile(
r'^(?P<swap_cache_pages>\d+) pages in swap cache'
r'(?:\n)'
r'^Swap cache stats: add \d+, delete \d+, find \d+/\d+'
r'^Swap cache stats: add \d+, delete \d+, find \d+\/\d+'
r'(?:\n)'
r'^Free swap = (?P<swap_free_kb>\d+)kB'
r'(?:\n)'
@ -989,6 +992,7 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
"""SVG graphics with one black triangle DOWN for sorting"""
def __init__(self):
self.oom = None
self.set_HTML_defaults()
self.update_toc()

109
test.py
View File

@ -18,6 +18,7 @@
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import http.server
import logging
import os
import socketserver
import threading
@ -101,13 +102,21 @@ class TestInBrowser(TestBase):
def assert_on_warn(self):
notify_box = self.driver.find_element_by_id('notify_box')
with self.assertRaises(NoSuchElementException):
notify_box.find_element_by_class_name('js-notify_box__msg--warning')
try:
warning = notify_box.find_element_by_class_name('js-notify_box__msg--warning')
except NoSuchElementException:
pass
else:
self.fail('Unexpected warning message: %s' % warning.text)
def assert_on_error(self):
notify_box = self.driver.find_element_by_id('notify_box')
with self.assertRaises(NoSuchElementException):
notify_box.find_element_by_class_name('js-notify_box__msg--error')
try:
error = notify_box.find_element_by_class_name('js-notify_box__msg--error')
except NoSuchElementException:
pass
else:
self.fail('Unexpected error message: %s' % error.text)
for event in self.driver.get_log('browser'):
# ignore favicon.ico errors
@ -115,13 +124,23 @@ class TestInBrowser(TestBase):
continue
self.fail('Error on browser console reported: %s' % event)
def assert_on_warn_error(self):
self.assert_on_warn()
self.assert_on_error()
def click_analyse(self):
analyse = self.driver.find_element_by_xpath('//button[text()="Analyse"]')
analyse.click()
def click_reset(self):
# OOMAnalyser.OOMDisplayInstance.reset_form()
reset = self.driver.find_element_by_xpath('//button[text()="Reset"]')
reset.click()
if reset.is_displayed():
reset.click()
else:
new_analysis = self.driver.find_element_by_xpath('//a[contains(text(), "Step 1 - Enter your OOM message")]')
# new_analysis = self.driver.find_element_by_link_text('Run a new analysis')
new_analysis.click()
self.assert_on_warn_error()
def analyse_oom(self, text):
@ -141,9 +160,28 @@ class TestInBrowser(TestBase):
self.click_analyse()
def assert_on_warn_error(self):
self.assert_on_warn()
self.assert_on_error()
def check_results(self):
"""Check the results of the analysis of the default example"""
self.assert_on_warn_error()
h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]')
self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")
trigger_proc_name = self.driver.find_element_by_class_name('trigger_proc_name')
self.assertEqual(trigger_proc_name.text, 'sed', 'Unexpected trigger process name')
trigger_proc_pid = self.driver.find_element_by_class_name('trigger_proc_pid')
self.assertEqual(trigger_proc_pid.text, '29481', 'Unexpected trigger process pid')
killed_proc_score = self.driver.find_element_by_class_name('killed_proc_score')
self.assertEqual(killed_proc_score.text, '651', 'Unexpected OOM score of killed process')
swap_cache_kb = self.driver.find_element_by_class_name('swap_cache_kb')
self.assertEqual(swap_cache_kb.text, '45368 kBytes')
swap_used_kb = self.driver.find_element_by_class_name('swap_used_kb')
self.assertEqual(swap_used_kb.text, '8343236 kBytes')
swap_free_kb = self.driver.find_element_by_class_name('swap_free_kb')
self.assertEqual(swap_free_kb.text, '0 kBytes')
swap_total_kb = self.driver.find_element_by_class_name('swap_total_kb')
self.assertEqual(swap_total_kb.text, '8388604 kBytes')
def test_001_load_page(self):
"""Test if the page is loading"""
@ -166,32 +204,14 @@ class TestInBrowser(TestBase):
self.assertFalse(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be not displayed")
self.click_analyse()
self.assert_on_warn_error()
self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")
trigger_proc_name = self.driver.find_element_by_class_name('trigger_proc_name')
self.assertEqual(trigger_proc_name.text, 'sed', 'Unexpected trigger process name')
trigger_proc_pid = self.driver.find_element_by_class_name('trigger_proc_pid')
self.assertEqual(trigger_proc_pid.text, '29481', 'Unexpected trigger process pid')
killed_proc_score = self.driver.find_element_by_class_name('killed_proc_score')
self.assertEqual(killed_proc_score.text, '651', 'Unexpected OOM score of killed process')
swap_cache_kb = self.driver.find_element_by_class_name('swap_cache_kb')
self.assertEqual(swap_cache_kb.text, '45368 kBytes')
swap_used_kb = self.driver.find_element_by_class_name('swap_used_kb')
self.assertEqual(swap_used_kb.text, '8343236 kBytes')
swap_free_kb = self.driver.find_element_by_class_name('swap_free_kb')
self.assertEqual(swap_free_kb.text, '0 kBytes')
swap_total_kb = self.driver.find_element_by_class_name('swap_total_kb')
self.assertEqual(swap_total_kb.text, '8388604 kBytes')
self.check_results()
def test_004_begin_but_no_end(self):
"""Test incomplete OOM text - just the beginning"""
example = """\
sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
sed cpuset=/ mems_allowed=0-1
CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
"""
self.analyse_oom(example)
@ -239,6 +259,33 @@ Killed process 6576 (java) total-vm:33914892kB, anon-rss:20629004kB, file-rss:0k
h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]')
self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")
def test_removal_of_leading_but_useless_columns(self):
"""Test removal of leading but useless columns"""
self.analyse_oom(OOMAnalyser.OOMDisplay.example)
self.check_results()
self.click_reset()
for prefix in ["[11686.888109] ",
"Apr 01 14:13:32 mysrv: ",
"Apr 01 14:13:32 mysrv kernel: ",
"Apr 01 14:13:32 mysrv <kern.warning> kernel: ",
"Apr 01 14:13:32 mysrv kernel: [11686.888109] ",
"kernel:",
"Apr 01 14:13:32 mysrv <kern.warning> kernel:",
]:
lines = OOMAnalyser.OOMDisplay.example.split('\n')
lines = ["{}{}".format(prefix, line) for line in lines]
oom_text = "\n".join(lines)
self.analyse_oom(oom_text)
try:
self.check_results()
except AssertionError:
logging.error('prefix %s', prefix)
import pdb; pdb.set_trace()
oom = OOMAnalyser.OOMEntity(oom_text)
print(oom.text)
self.click_reset()
class TestPython(TestBase):
@ -272,9 +319,9 @@ class TestPython(TestBase):
"""Test stripping useless / leading columns"""
oom_entity = OOMAnalyser.OOMEntity(OOMAnalyser.OOMDisplay.example)
for pos, line in [
(1, '[11686.888109] sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_adj=0, oom_score_adj=0'),
(5, 'Apr 01 14:13:32 mysrv kernel: sed invoked OOM-killer: gfp_mask=0x201da, order=0'),
(6, 'Apr 01 14:13:32 mysrv kernel: [11686.888109] sed invoked oom-killer: gfp_mask=0x84d0, order=0, oom_adj=0, oom_score_adj=0'),
(1, '[11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1'),
(5, 'Apr 01 14:13:32 mysrv kernel: CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1'),
(6, 'Apr 01 14:13:32 mysrv kernel: [11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1'),
]:
to_strip = oom_entity._number_of_columns_to_strip(line)
self.assertEqual(to_strip, pos, 'Calc wrong number of columns to strip for "%s": got: %d, expect: %d' % (