Mikaela/Limnoria
1
0
Fork 0
mirror of https://github.com/Mikaela/Limnoria.git synced 2024-11-23 11:09:23 +01:00
Code Issues Projects Releases Wiki Activity

plugins/Web: Swtich the title parser back to HTMLParser sing sgmllib's parser spins on invalid input.

Browse Source
This commit is contained in:
James Vega 2006-09-13 19:40:51 +00:00
parent deb6dbc1e1
commit ee9aaa89d6
2 changed files with 34 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
plugins/Web/plugin.py
View File

@ -28,7 +28,7 @@
### ###
import re import re
import sgmllib import HTMLParser
import htmlentitydefs import htmlentitydefs
import supybot.conf as conf import supybot.conf as conf
@ -38,32 +38,32 @@ import supybot.plugins as plugins
import supybot.ircutils as ircutils import supybot.ircutils as ircutils
import supybot.callbacks as callbacks import supybot.callbacks as callbacks
class Title(sgmllib.SGMLParser): class Title(HTMLParser.HTMLParser):
entitydefs = htmlentitydefs.entitydefs.copy() entitydefs = htmlentitydefs.entitydefs.copy()
entitydefs['nbsp'] = ' ' entitydefs['nbsp'] = ' '
entitydefs['apos'] = '\''
def __init__(self): def __init__(self):
self.inTitle = False self.inTitle = False
self.title = '' self.title = ''
sgmllib.SGMLParser.__init__(self) HTMLParser.HTMLParser.__init__(self)
def start_title(self, attrs): def handle_starttag(self, tag, attrs):
if tag == 'title':
self.inTitle = True self.inTitle = True
def end_title(self): def handle_endtag(self, tag):
if tag == 'title':
self.inTitle = False self.inTitle = False
def unknown_entityref(self, name):
if self.inTitle:
self.title += ' '
def unknown_charref(self, name):
if self.inTitle:
self.title += ' '
def handle_data(self, data): def handle_data(self, data):
if self.inTitle: if self.inTitle:
self.title += data self.title += data
def handle_entityref(self, name):
if self.inTitle:
if name in self.entitydefs:
self.title += self.entitydefs[name]
class Web(callbacks.PluginRegexp): class Web(callbacks.PluginRegexp):
"""Add the help for "@help Web" here.""" """Add the help for "@help Web" here."""
threaded = True threaded = True
@ -172,7 +172,7 @@ class Web(callbacks.PluginRegexp):
parser = Title() parser = Title()
try: try:
parser.feed(text) parser.feed(text)
except sgmllib.SGMLParseError: except HTMLParser.HTMLParseError:
self.log.debug('Encountered a problem parsing %u. Title may ' self.log.debug('Encountered a problem parsing %u. Title may '
'already be set, though', url) 'already be set, though', url)
if parser.title: if parser.title:

28
plugins/Web/test.py
View File

@ -53,16 +53,18 @@ class WebTestCase(ChannelPluginTestCase):
'Slashdot: News for nerds, stuff that matters') 'Slashdot: News for nerds, stuff that matters')
# Amazon add a bunch of scripting stuff to the top of their page, # Amazon add a bunch of scripting stuff to the top of their page,
# so we need to allow for a larger peekSize # so we need to allow for a larger peekSize
try: # Actually, screw Amazon. Even bumping this up to 10k doesn't give us enough
orig = conf.supybot.protocols.http.peekSize() # info.
conf.supybot.protocols.http.peekSize.setValue(8192) # try:
self.assertNotRegexp('title ' # orig = conf.supybot.protocols.http.peekSize()
'http://www.amazon.com/exec/obidos/tg/detail/-/' # conf.supybot.protocols.http.peekSize.setValue(8192)
'1884822312/qid=1063140754/sr=8-1/ref=sr_8_1/' # self.assertNotRegexp('title '
'002-9802970-2308826?v=glance&s=books&n=507846', # 'http://www.amazon.com/exec/obidos/tg/detail/-/'
'no HTML title') # '1884822312/qid=1063140754/sr=8-1/ref=sr_8_1/'
finally: # '002-9802970-2308826?v=glance&s=books&n=507846',
conf.supybot.protocols.http.peekSize.setValue(orig) # 'no HTML title')
# finally:
# conf.supybot.protocols.http.peekSize.setValue(orig)
# Checks the non-greediness of the regexp # Checks the non-greediness of the regexp
self.assertResponse('title ' self.assertResponse('title '
'http://www.space.com/scienceastronomy/' 'http://www.space.com/scienceastronomy/'
@ -82,6 +84,12 @@ class WebTestCase(ChannelPluginTestCase):
# Checks that title parser grabs the full title instead of just # Checks that title parser grabs the full title instead of just
# part of it. # part of it.
self.assertRegexp('title http://www.n-e-r-d.com/', 'N.*E.*R.*D') self.assertRegexp('title http://www.n-e-r-d.com/', 'N.*E.*R.*D')
# Checks that the parser doesn't hang on invalid tags
print
print "If we have not fixed a bug with the parser, the following",
print "test will hang the test-suite."
self.assertNotError(
'title http://www.youtube.com/watch?v=x4BtiqPN4u8')
def testNetcraft(self): def testNetcraft(self):
self.assertNotError('netcraft slashdot.org') self.assertNotError('netcraft slashdot.org')