Add configuration variable supybot.servers.http.robots.

src/conf.py

@@ -1080,9 +1080,13 @@ registerGlobalValue(supybot.servers.http, 'port',
     registry.Integer(8080, _("""Determines what port the HTTP server will
     bind.""")))
 registerGlobalValue(supybot.servers.http, 'keepAlive',
-    registry.Boolean(False, _("""Defines whether the server will stay alive if
-    no plugin is using it. This also means that the server will start even
-    if it is not used.""")))
+    registry.Boolean(False, _("""Determiness whether the server will stay
+    alive if no plugin is using it. This also means that the server will
+    start even if it is not used.""")))
+registerGlobalValue(supybot.servers.http, 'robots',
+    registry.String('', _("""Determines the content of the robots.txt file,
+    served on the server to search engine. All \\n will be converted to line
+    breaks.""")))
 
 
 ###

src/httpserver.py

@@ -84,6 +84,8 @@ class SupyHTTPRequestHandler(BaseHTTPRequestHandler):
     def do_X(self, callbackMethod, *args, **kwargs):
         if self.path == '/':
             callback = SupyIndex()
+        elif self.path == '/robots.txt':
+            callback = RobotsTxt()
         else:
             subdir = self.path.split('/')[1]
             try:
@@ -191,6 +193,18 @@ class SupyIndex(SupyHTTPServerCallback):
         self.end_headers()
         self.wfile.write(response)
 
+class RobotsTxt(SupyHTTPServerCallback):
+    """Serves the robot.txt file to robots."""
+    name = 'robotstxt'
+    defaultResponse = _('Request not handled')
+    def doGet(self, handler, path):
+        response = conf.supybot.servers.http.robots().replace('\\n', '\n')
+        handler.send_response(200)
+        self.send_header('Content_type', 'text/html')
+        self.send_header('Content-Length', len(response))
+        self.end_headers()
+        self.wfile.write(response)
+
 httpServer = None
 
 def startServer():

src/version.py

@@ -1,3 +1,3 @@
 """stick the various versioning attributes in here, so we only have to change
 them once."""
-version = '0.83.4.1+limnoria (2011-10-28T22:44:09+0200)'
+version = '0.83.4.1+limnoria (2011-10-29T12:13:09+0200)'