PyLink 3.1.0

Update README links and mark the project as discontinued.

.codeclimate.yml

@@ -1,13 +0,0 @@
-# This file is used for the configuration of https://codeclimate.com/github/GLolol/PyLink/
-# You needn't change this if you're running your own copy of PyLink.
-
-engines:
-  duplication:
-    enabled: true
-    config:
-      languages:
-      - python
-  pep8:
-    enabled: true
-  fixme:
-    enabled: true

.dockerignore

@@ -0,0 +1,25 @@
+*.yml
+*.yaml
+# Git, CI, etc. config files
+.*
+test/
+
+# Automatically generated by setup.py
+/__init__.py
+
+env/
+build/
+__pycache__/
+.idea/
+*.py[cod]
+*.bak
+*~
+*#
+*.save*
+*.db
+*.pid
+*.pem
+.eggs
+*.egg-info/
+dist/
+log/

.drone-write-tags.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Write Docker tags for Drone CI: version-YYMMDD, version, major version, latest
+
+VERSION="$1"
+
+if test -z "$VERSION"; then
+    echo "Reading version from VERSION file" >&2
+    VERSION=$(<VERSION)
+fi
+
+if [[ "$VERSION" == *"alpha"* || "$VERSION" == *"dev"* ]]; then
+    # This should never trigger if reference based tagging is enabled
+    echo "ERROR: Pushing alpha / dev tags is not supported"
+    exit 1
+fi
+
+major_version="$(printf '%s' "$VERSION" | cut -d . -f 1)"
+
+# Date based tag
+printf '%s' "$VERSION-$(date +%Y%m%d),"
+# Program version
+printf '%s' "$VERSION,"
+
+if [[ "$VERSION" == *"beta"* ]]; then
+    printf '%s' "$major_version-beta,"
+    printf '%s' "latest-beta"
+else  # Stable or rc build
+    printf '%s' "$major_version,"
+    printf '%s' "latest"
+fi

.drone.jsonnet

@@ -0,0 +1,94 @@
+local test(py_version) = {
+  "kind": "pipeline",
+  "type": "docker",
+  "name": "test-" + py_version,
+  "steps": [
+    {
+      "name": "test",
+      "image": "python:" + py_version,
+      "commands": [
+        "git submodule update --recursive --remote --init",
+        "pip install -r requirements-docker.txt",
+        "python3 setup.py install",
+        "python3 -m unittest discover test/ --verbose"
+      ]
+    }
+  ]
+};
+
+local build_docker(py_version) = {
+  "kind": "pipeline",
+  "type": "docker",
+  "name": "build_docker",
+  "steps": [
+    {
+      "name": "set Docker image tags",
+      "image": "bash",
+      "commands": [
+        "bash .drone-write-tags.sh $DRONE_TAG > .tags",
+        "# Will build the following tags:",
+        "cat .tags"
+      ]
+    },
+    {
+      "name": "build Docker image",
+      "image": "plugins/docker",
+      "settings": {
+        "repo": "jlu5/pylink",
+        "username": {
+          "from_secret": "docker_user"
+        },
+        "password": {
+          "from_secret": "docker_token"
+        }
+      }
+    }
+  ],
+  "trigger": {
+    "event": [
+      "push"
+    ],
+    "branch": ["release"],
+  },
+  "depends_on": ["test-" + py_version]
+};
+
+local deploy_pypi(py_version) = {
+  "kind": "pipeline",
+  "type": "docker",
+  "name": "deploy_pypi",
+  "steps": [
+    {
+      "name": "pypi_publish",
+      "image": "plugins/pypi",
+      "settings": {
+        "username": "__token__",
+        "password": {
+            "from_secret": "pypi_token"
+        }
+      }
+    }
+  ],
+  "trigger": {
+    "event": [
+      "tag"
+    ],
+    "ref": {
+      "exclude": [
+        "refs/tags/*alpha*",
+        "refs/tags/*beta*",
+        "refs/tags/*dev*"
+      ]
+    }
+  },
+  "depends_on": ["test-" + py_version]
+};
+
+[
+    test("3.7"),
+    test("3.8"),
+    test("3.9"),
+    test("3.10"),
+    deploy_pypi("3.10"),
+    build_docker("3.10"),
+]

.gitattributes

@@ -0,0 +1,4 @@
+* eol=lf
+
+*.png binary
+*.jpg binary

.gitignore

@@ -3,15 +3,20 @@
 !example-*.yml
 !.*.yml
 
+# Generated from .drone.jsonnet
+.drone.yml
+
 # Automatically generated by setup.py
 /__init__.py
 
 env/
 build/
 __pycache__/
+.idea/
 *.py[cod]
 *.bak
 *~
+*#
 *.save*
 *.db
 *.pid
@@ -19,3 +24,4 @@ __pycache__/
 .eggs
 *.egg-info/
 dist/
+log/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/parser_tests"]
+	path = test/parser-tests
+	url = https://github.com/ircdocs/parser-tests

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+line_length=100

.mailmap

@@ -1,2 +1,7 @@
-James Lu <GLolol@overdrivenetworks.com> <GLolol1@hotmail.com>
+James Lu <james@overdrivenetworks.com> <GLolol@overdrivenetworks.com>
-James Lu <GLolol@overdrivenetworks.com> <GLolol@overdrive.pw>
+James Lu <james@overdrivenetworks.com> <bitflip3+github@gmail.com>
+James Lu <james@overdrivenetworks.com> <GLolol1@hotmail.com>
+James Lu <james@overdrivenetworks.com> <GLolol@overdrive.pw>
+Ken Spencer <ken@electrocode.net> <kspencer@electrocode.net>
+Ken Spencer <ken@electrocode.net> <iota@electrocode.net>
+Ken Spencer <ken@electrocode.net> <iota@e-code.in>

.pylintrc

@@ -0,0 +1,4 @@
+[FORMAT]
+max-line-length=120
+
+good-names=ip,f,i

AUTHORS

@@ -1,4 +1,5 @@
 The following people have contributed substantially to PyLink:
 
-James Lu <glolol@overdrivenetworks.com>
+James Lu <james@overdrivenetworks.com>
 Daniel Oaks <daniel@danieloaks.net>
+Ken Spencer <iota@electrocode.net>

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3-alpine
+
+RUN adduser -D -H -u 10000 pylink
+
+VOLUME /pylink
+
+COPY . /pylink-src
+
+RUN cd /pylink-src && pip3 install --no-cache-dir -r requirements-docker.txt
+RUN cd /pylink-src && python3 setup.py install
+RUN rm -r /pylink-src
+
+USER pylink
+WORKDIR /pylink
+
+# Run in no-PID file mode by default
+CMD ["pylink", "-n"]

README.md

@@ -1,50 +1,83 @@
-# PyLink
+# PyLink IRC Services
+
+## END OF LIFE NOTICE: This project is no longer maintained. So long and thanks for all the fish.
+
+<!--
+[![Latest stable release](https://img.shields.io/github/v/tag/jlu5/pylink?label=stable&color=1a1)](https://github.com/PyLink/PyLink/tree/master)
+[![PyPI version](https://img.shields.io/pypi/v/pylinkirc.svg?maxAge=2592000)](https://pypi.python.org/pypi/pylinkirc/)
+[![Docker image version](https://img.shields.io/docker/v/jlu5/pylink/latest?label=docker)](https://hub.docker.com/r/jlu5/pylink)
+[![Supported Python versions](https://img.shields.io/badge/python-3.7%20and%20later-50e)](https://www.python.org/downloads/)
+-->
 
 PyLink is an extensible, plugin-based IRC services framework written in Python. It aims to be:
 
-1) a replacement for the now-defunct Janus.
+1) a transparent server-side relayer between IRC networks.
 
-2) a versatile framework and gateway to IRC.
+2) a versatile framework for developing IRC services.
 
-PyLink and any bundled software are licensed under the Mozilla Public License, version 2.0 ([LICENSE.MPL2](LICENSE.MPL2)). The corresponding documentation in the [docs/](docs/) folder is licensed under the Creative Attribution-ShareAlike 4.0 International License. ([LICENSE.CC-BY-SA-4.0](LICENSE.CC-BY-SA-4.0))
+PyLink is licensed under the Mozilla Public License, version 2.0 ([LICENSE.MPL2](LICENSE.MPL2)). The [corresponding documentation](docs/) is licensed under the Creative Attribution-ShareAlike 4.0 International License. ([LICENSE.CC-BY-SA-4.0](LICENSE.CC-BY-SA-4.0))
 
-## Support
+## Getting help
 
 **First, MAKE SURE you've read the [FAQ](docs/faq.md)!**
 
 **When upgrading between major versions, remember to read the [release notes](RELNOTES.md) for any breaking changes!**
 
-Please report any bugs you find to the [issue tracker](https://github.com/GLolol/PyLink/issues). Pull requests are open if you'd like to contribute, though new stuff generally goes to the **devel** branch.
+Please report any bugs you find to the [issue tracker](https://github.com/PyLink/PyLink/issues). Pull requests are likewise welcome.
-
-You can also find support via our IRC channels: `#PyLink @ irc.overdrivenetworks.com `([webchat](https://webchat.overdrivenetworks.com/?channels=PyLink,dev)) or `#PyLink @ chat.freenode.net`. Ask your questions and be patient for a response.
 
 ## Installation
 
-### Installing from source (recommended)
+### Pre-requisites
+* Python 3.7 or above - prefer the newest Python 3.x when available
+* A Unix-like operating system: PyLink is actively developed on Linux only, so we cannot guarantee that things will work properly on other systems.
 
-First, make sure the following dependencies are met:
+If you are a developer and want to help make PyLink more portable, patches are welcome.
 
-* Python 3.4+
+### Installing from source
-* Setuptools (`pip3 install setuptools`)
-* PyYAML (`pip3 install pyyaml`)
-* [ircmatch](https://github.com/mammon-ircd/ircmatch) (`pip3 install ircmatch`)
-* *For the servprotect plugin*: [expiringdict](https://github.com/mailgun/expiringdict) (install this from source; installation is broken in pip due to [mailgun/expiringdict#13](https://github.com/mailgun/expiringdict/issues/13))
 
-1) Clone the repository: `git clone https://github.com/GLolol/PyLink && cd PyLink`
+1) First, make sure the following dependencies are met:
 
-2) Install PyLink using `python3 setup.py install` (global install) or `python3 setup.py install --user` (local install)
+    * Setuptools (`pip3 install setuptools`)
-    - Note: `--user` is a *literal* string; *do not* replace it with your username.
+    * PyYAML (`pip3 install pyyaml`)
+    * cachetools (`pip3 install cachetools`)
+    * *For hashed password support*: Passlib >= 1.7.0 (`pip3 install passlib`)
+    * *For Unicode support in Relay*: unidecode (`pip3 install Unidecode`)
+    * *For extended PID file tracking (i.e. removing stale PID files after a crash)*: psutil (`pip3 install psutil`)
 
-### Installing via PyPI
+2) Clone the repository: `git clone https://github.com/PyLink/PyLink && cd PyLink`
-1) Make sure you're running the right pip command: on most distros, pip for Python3 uses the command `pip3`.
+    - Previously there was a *devel* branch for testing versions of PyLink - this practice has since been discontinued.
+
+3) Install PyLink using `python3 setup.py install` (global install) or `python3 setup.py install --user` (local install)
+    * Note: `--user` is a *literal* string; *do not* replace it with your username.
+    *  **Whenever you switch branches or update PyLink's sources via `git pull`, you will need to re-run this command for changes to apply!**
+
+### Installing via Docker
+
+As of PyLink 3.0 there is a Docker image available on Docker Hub: [jlu5/pylink](https://hub.docker.com/r/jlu5/pylink)
+
+It supports the following tags:
+
+- Rolling tags: **`latest`** (latest stable/RC release), **`latest-beta`** (latest beta snapshot)
+- Pinned to a major branch: e.g. **`3`** (latest 3.x stable release), **`3-beta`** (latest 3.x beta snapshot)
+- Pinned to a specific version: e.g. **`3.0.0`**
+
+To use this image you should mount your configuration/DB folder into `/pylink`. **Make sure this directory is writable by UID 10000.**
+
+```bash
+$ docker run -v $HOME/pylink:/pylink jlu5/pylink
+```
+
+### Installing via PyPI (stable branch only)
+
+1) Make sure you're running the right pip command: on most distros, pip for Python 3 uses the command `pip3`.
 
 2) Run `pip3 install pylinkirc` to download and install PyLink. pip will automatically resolve dependencies.
 
-3) Download or copy https://github.com/GLolol/PyLink/blob/master/example-conf.yml for an example configuration.
+3) Download or copy https://github.com/PyLink/PyLink/blob/master/example-conf.yml for an example configuration.
 
 ## Configuration
 
-1) Rename `example-conf.yml` to `pylink.yml` (or a similarly named `.yml` file) and configure your instance there. Note that the configuration format isn't finalized yet - this means that your configuration may break in an update!
+1) Rename `example-conf.yml` to `pylink.yml` (or a similarly named `.yml` file) and configure your instance there.
 
 2) Run `pylink` from the command line. PyLink will load its configuration from `pylink.yml` by default, but you can override this by running `pylink` with a config argument (e.g. `pylink mynet.yml`).
 
@@ -54,24 +87,55 @@ First, make sure the following dependencies are met:
 
 These IRCds (in alphabetical order) are frequently tested and well supported. If any issues occur, please file a bug on the issue tracker.
 
-* [charybdis](http://charybdis.io/) (3.5+ / git master) - module `ts6`
+* [InspIRCd](http://www.inspircd.org/) (2.0 - 3.x) - module `inspircd`
-* [InspIRCd](http://www.inspircd.org/) 2.0.x - module `inspircd`
+    - Set the `target_version` option to `insp3` to target InspIRCd 3.x (default), or `insp20` to target InspIRCd 2.0 (legacy).
-    - For vHost setting to work, `m_chghost.so` must be loaded.
+    - For vHost setting to work, `m_chghost.so` must be loaded. For ident and realname changing support, `m_chgident.so` and `m_chgname.so` must be loaded respectively.
+    - Supported channel, user, and prefix modes are negotiated on connect, but hotloading modules that change these is not supported. After changing module configuration, it is recommended to SQUIT PyLink to force a protocol renegotiation.
+* [Nefarious IRCu](https://github.com/evilnet/nefarious2) (2.0.0+) - module `p10`
+    - Note: Both account cloaks (user and oper) and hashed IP cloaks are optionally supported (`HOST_HIDING_STYLE` settings 0 to 3). Make sure you configure PyLink to match your IRCd settings.
+* [UnrealIRCd](https://www.unrealircd.org/) (4.2.x - 5.0.x) - module `unreal`
     - Supported channel, user, and prefix modes are negotiated on connect, but hotloading modules that change these is not supported. After changing module configuration, it is recommended to SQUIT PyLink to force a protocol renegotiation.
-* [UnrealIRCd](https://www.unrealircd.org/) 4.x - module `unreal`
-    - Linking to UnrealIRCd 3.2 servers is only supported when using an UnrealIRCd 4.x server as a hub, with topology such as  `pylink<->unreal4<->unreal3.2`. We nevertheless encourage you to upgrade so all your IRCds are running the same version.
 
 ### Extended support
 
 Support for these IRCds exist, but are not tested as frequently and thoroughly. Bugs should be filed if there are any issues, though they may not always be fixed in a timely fashion.
 
-* [Elemental-IRCd](https://github.com/Elemental-IRCd/elemental-ircd) (6.6.x / git master) - module `ts6`
+* [charybdis](https://github.com/charybdis-ircd/charybdis) (3.5+) - module `ts6`
-* [InspIRCd](http://www.inspircd.org/) 2.2 (git master) - module `inspircd`
+    - For KLINE support to work, a `shared{}` block should be added for PyLink on all servers.
-* [IRCd-Hybrid](http://www.ircd-hybrid.org/) (8.2.x / svn trunk) - module `hybrid`
+* [ChatIRCd](http://www.chatlounge.net/software) (1.2.x / git master) - module `ts6`
-    - Note: for host changing support and optimal functionality, a `service{}` block / U-line should be added for PyLink on every IRCd across your network.
+    - For KLINE support to work, a `shared{}` block should be added for PyLink on all servers.
-* [juno-ircd](https://github.com/cooper/yiria) (11.x / janet) - module `ts6` (see [configuration example](https://github.com/cooper/juno/blob/master/doc/ts6.md#pylink))
+* [juno-ircd](https://github.com/cooper/juno) (13.x / ava) - module `ts6` (see [configuration example](https://github.com/cooper/juno/blob/master/doc/ts6.md#pylink))
-* [Nefarious IRCu](https://github.com/evilnet/nefarious2) (2.0.0+) - module `nefarious`
+* [ngIRCd](https://ngircd.barton.de/) (24+) - module `ngircd`
-    - Note: Both account cloaks (user and oper) and hashed IP cloaks are optionally supported (HOST_HIDING_STYLE settings 0 to 3). Make sure you configure PyLink to match your IRCd settings.
+    - For GLINEs to propagate, the `AllowRemoteOper` option must be enabled in ngIRCd.
-    - For optimal functionality (mode overrides in relay, etc.), consider adding `UWorld{}` blocks / U-lines for every server that PyLink spawns.
+    - `+` (modeless) channels are not supported, and should be disabled for PyLink to function correctly.
+    - For use with Relay, the `CloakHostModeX` setting will work fine but `CloakHost` and `CloakUserToNick` are *not* supported.
 
-Other TS6 and P10 variations may work, but are not officially supported.
+### Legacy extended support
+
+Support for these IRCds was added at some point but is no longer actively maintained, either due to inactive upstream development or a perceived lack of interest. We recommend migrating to an IRCd in the above two sections.
+
+* [beware-ircd](http://ircd.bircd.org/) (1.6.3) - module `p10`
+    - Because bircd disallows BURST after ENDBURST for regular servers, U-lines are required for all PyLink servers. Fortunately, wildcards are supported in U-lines, so you can add something along the lines of `U:<your pylink server>:` and `U:*.relay:` (adjust accordingly for your relay server suffix).
+    - Use `ircd: snircd` as the target IRCd.
+    - Halfops, `sethost` (`+h`), and account-based cloaking (`VHostStyle=1`) are supported. Crypted IPs and static hosts (`VHostStyle` 2 and 3) are NOT.
+* [Elemental-IRCd](https://github.com/Elemental-IRCd/elemental-ircd) (6.6.x / git master) - module `ts6`
+    - For KLINE support to work, a `shared{}` block should be added for PyLink on all servers.
+* [IRCd-Hybrid](http://www.ircd-hybrid.org/) (8.2.x / svn trunk) - module `hybrid`
+    - For host changing support and optimal functionality, a `service{}` block / U-line should be added for PyLink on every IRCd across your network.
+    - For KLINE support to work, a `shared{}` block should also be added for PyLink on all servers.
+* [ircd-ratbox](http://www.ratbox.org/) (3.x) - module `ts6`
+    - Host changing is not supported.
+    - On ircd-ratbox, all known IPs of users will be shown in `/whois`, even if the client is e.g. a cloaked relay client. If you're paranoid about this, turn off Relay IP forwarding on the ratbox network(s).
+    - For KLINE support to work, a `shared{}` block should be added for PyLink on all servers.
+* [IRCu](http://coder-com.undernet.org/) (u2.10.12.16+) - module `p10`
+    - Host changing (changehost, relay) is not supported.
+* [snircd](https://development.quakenet.org/) (1.3.x+) - module `p10`
+    - Outbound host changing (i.e. for the `changehost` plugin) is not supported.
+
+### Clientbot
+
+PyLink supports connecting to IRCds as a relay bot and forwarding users back as virtual clients, similar to Janus' Clientbot. This can be useful if the IRCd a network used isn't supported, or if you want to relay certain channels without fully linking with a network.
+
+For Relay to work properly with Clientbot, be sure to load the `relay_clientbot` plugin in conjunction with `relay`.
+
+Note: **Clientbot links can only be used as a leaf for Relay links - they CANNOT be used to host channels!** This means that Relay does not support having all your networks be Clientbot - in those cases you are better off using a classic relay bot, like [RelayNext for Limnoria](https://github.com/jlu5/SupyPlugins/tree/master/RelayNext).

VERSION

@@ -1 +1 @@
-0.10-alpha1
+3.1.0

conf.py

@@ -8,13 +8,22 @@ It provides simple checks for validating and loading YAML-format configurations
 try:
     import yaml
 except ImportError:
-    raise ImportError("Please install PyYAML and try again.")
+    raise ImportError("PyLink requires PyYAML to function; please install it and try again.")
 
+import logging
+import os.path
 import sys
 from collections import defaultdict
 
 from . import world
 
+__all__ = ['ConfigurationError', 'conf', 'confname', 'validate', 'load_conf',
+           'get_database_name']
+
+
+class ConfigurationError(RuntimeError):
+    """Error when config conditions aren't met."""
+
 conf = {'bot':
                 {
                     'nick': 'PyLink',
@@ -24,7 +33,7 @@ conf = {'bot':
                 },
         'logging':
                 {
-                    'stdout': 'INFO'
+                    'console': 'INFO'
                 },
         'servers':
                 # Wildcard defaultdict! This means that
@@ -41,35 +50,104 @@ conf = {'bot':
                                      'sidrange': '0##'
                                     })
         }
+conf['pylink'] = conf['bot']
 confname = 'unconfigured'
 
-def validateConf(conf):
+def validate(condition, errmsg):
+    """Raises ConfigurationError with errmsg unless the given condition is met."""
+    if not condition:
+        raise ConfigurationError(errmsg)
+
+def _log(level, text, *args, logger=None, **kwargs):
+    if logger:
+        logger.log(level, text, *args, **kwargs)
+    else:
+        world._log_queue.append((level, text))
+
+def _validate_conf(conf, logger=None):
     """Validates a parsed configuration dict."""
-    assert type(conf) == dict, "Invalid configuration given: should be type dict, not %s." % type(conf).__name__
+    validate(isinstance(conf, dict),
+            "Invalid configuration given: should be type dict, not %s."
+            % type(conf).__name__)
 
-    for section in ('bot', 'servers', 'login', 'logging'):
+    if 'pylink' in conf and 'bot' in conf:
-        assert conf.get(section), "Missing %r section in config." % section
+        _log(logging.WARNING, "Since PyLink 1.2, the 'pylink:' and 'bot:' configuration sections have been condensed "
+             "into one. You should merge any options under these sections into one 'pylink:' block.", logger=logger)
 
-    assert type(conf['login'].get('password')) == type(conf['login'].get('user')) == str and \
+        new_block = conf['bot'].copy()
-        conf['login']['password'] != "changeme", "You have not set the login details correctly!"
+        new_block.update(conf['pylink'])
+        conf['bot'] = conf['pylink'] = new_block
+    elif 'pylink' in conf:
+        conf['bot'] = conf['pylink']
+    elif 'bot' in conf:
+        conf['pylink'] = conf['bot']
+        # TODO: add a migration warning in the next release.
+
+    for section in ('pylink', 'servers', 'login', 'logging'):
+        validate(conf.get(section), "Missing %r section in config." % section)
+
+    # Make sure at least one form of authentication is valid.
+    # Also we'll warn them that login:user/login:password is deprecated
+    if conf['login'].get('password') or conf['login'].get('user'):
+        _log(logging.WARNING, "The 'login:user' and 'login:password' options are deprecated since PyLink 1.1. "
+             "Please switch to the new 'login:accounts' format as outlined in the example config.", logger=logger)
+
+    old_login_valid = isinstance(conf['login'].get('password'), str) and isinstance(conf['login'].get('user'), str)
+    newlogins = conf['login'].get('accounts', {})
+
+    validate(old_login_valid or newlogins, "No accounts were set, aborting!")
+    for account, block in newlogins.items():
+        validate(isinstance(account, str), "Bad username format %s" % account)
+        validate(isinstance(block.get('password'), str), "Bad password %s for account %s" % (block.get('password'), account))
+
+    validate(conf['login'].get('password') != "changeme", "You have not set the login details correctly!")
+
+    if newlogins and not old_login_valid:
+        validate(conf.get('permissions'), "New-style accounts enabled but no permissions block was found. You will not be able to administrate your PyLink instance!")
+
+    if conf['logging'].get('stdout'):
+         _log(logging.WARNING, 'The log:stdout option is deprecated since PyLink 1.2 in favour of '
+                               '(a more correctly named) log:console. Please update your '
+                               'configuration accordingly!', logger=logger)
 
     return conf
 
-def loadConf(filename, errors_fatal=True):
+def load_conf(filename, errors_fatal=True, logger=None):
     """Loads a PyLink configuration file from the filename given."""
     global confname, conf, fname
+    # Note: store globally the last loaded conf filename, for REHASH in coremods/control.
     fname = filename
-    confname = filename.split('.', 1)[0]
+    # For the internal config name, strip off any .yml extensions and absolute paths
+    confname = os.path.splitext(os.path.basename(filename))[0]
     try:
         with open(filename, 'r') as f:
-            conf = yaml.load(f)
+            conf = yaml.safe_load(f)
-            conf = validateConf(conf)
+            conf = _validate_conf(conf, logger=logger)
     except Exception as e:
-        print('ERROR: Failed to load config from %r: %s: %s' % (filename, type(e).__name__, e), file=sys.stderr)
+        e = 'Failed to load config from %r: %s: %s' % (filename, type(e).__name__, e)
-        print('       Users upgrading from users < 0.9-alpha1 should note that the default configuration has been renamed to *pylink.yml*, not *config.yml*', file=sys.stderr)
+
+        if logger:  # Prefer using the Python logger when available
+            logger.exception(e)
+        else:  # Otherwise, fall back to a print() call.
+            print('ERROR: %s' % e, file=sys.stderr)
 
         if errors_fatal:
-            sys.exit(4)
+            sys.exit(1)
+
         raise
     else:
         return conf
+
+def get_database_name(dbname):
+    """
+    Returns a database filename with the given base DB name appropriate for the
+    current PyLink instance.
+
+    This returns '<dbname>.db' if the running config name is PyLink's default
+    (pylink.yml), and '<dbname>-<config name>.db' for anything else. For example,
+    if this is called from an instance running as 'pylink testing.yml', it
+    would return '<dbname>-testing.db'."""
+    if confname != 'pylink':
+        dbname += '-%s' % confname
+    dbname += '.db'
+    return dbname

coremods/__init__.py

@@ -1,2 +1,3 @@
-# Service support has to be imported first, so that utils.add_cmd works
+# Note: Service support has to be imported first, so that utils.add_cmd() works for corecommands,
-from . import service_support, control, handlers, corecommands, exttargets
+# etc.
+from . import service_support, permissions, control, handlers, corecommands, exttargets

coremods/control.py

@@ -1,11 +1,18 @@
 """
 control.py - Implements SHUTDOWN and REHASH functionality.
 """
-import signal
+import atexit
 import os
+import signal
+import threading
+
+from pylinkirc import conf, utils, world  # Do not import classes, it'll import loop
+from pylinkirc.log import _get_console_log_level, _make_file_logger, _stop_file_loggers, log
+
+from . import login
+
+__all__ = ['remove_network', 'shutdown', 'rehash']
 
-from pylinkirc import world, utils, conf, classes
-from pylinkirc.log import log, makeFileLogger, stopFileLoggers
 
 def remove_network(ircobj):
     """Removes a network object from the pool."""
@@ -14,78 +21,142 @@ def remove_network(ircobj):
     ircobj.disconnect()
     del world.networkobjects[ircobj.name]
 
-def _shutdown(irc=None):
+def _print_remaining_threads():
-    """Shuts down the Pylink daemon."""
+    log.debug('shutdown(): Remaining threads: %s', ['%s/%s' % (t.name, t.ident) for t in threading.enumerate()])
+
+def _remove_pid():
+    pidfile = "%s.pid" % conf.confname
+    if world._should_remove_pid:
+        # Remove our pid file.
+        log.info("Removing PID file %r.", pidfile)
+        try:
+            os.remove(pidfile)
+        except OSError:
+            log.exception("Failed to remove PID file %r, ignoring..." % pidfile)
+    else:
+        log.debug('Not removing PID file %s as world._should_remove_pid is False.' % pidfile)
+
+def _kill_plugins(irc=None):
+    if not world.plugins:
+        # No plugins were loaded or we were in a pre-initialized state, ignore.
+        return
+
+    log.info("Shutting down plugins.")
     for name, plugin in world.plugins.items():
         # Before closing connections, tell all plugins to shutdown cleanly first.
         if hasattr(plugin, 'die'):
             log.debug('coremods.control: Running die() on plugin %s due to shutdown.', name)
             try:
-                plugin.die(irc)
+                plugin.die(irc=irc)
             except:  # But don't allow it to crash the server.
                 log.exception('coremods.control: Error occurred in die() of plugin %s, skipping...', name)
 
+# We use atexit to register certain functions so that when PyLink cleans up after itself if it
+# shuts down because all networks have been disconnected.
+atexit.register(_remove_pid)
+atexit.register(_kill_plugins)
+
+def shutdown(irc=None):
+    """Shuts down the Pylink daemon."""
+    if world.shutting_down.is_set():  # We froze on shutdown last time, so immediately abort.
+        _print_remaining_threads()
+        raise KeyboardInterrupt("Forcing shutdown.")
+
+    world.shutting_down.set()
+
+    # HACK: run the _kill_plugins trigger with the current IRC object. XXX: We should really consider removing this
+    # argument, since no plugins actually use it to do anything.
+    atexit.unregister(_kill_plugins)
+    _kill_plugins(irc=irc)
+
     # Remove our main PyLink bot as well.
-    utils.unregisterService('pylink')
+    utils.unregister_service('pylink')
 
     for ircobj in world.networkobjects.copy().values():
         # Disconnect all our networks.
-        remove_network(ircobj)
+        try:
+            remove_network(ircobj)
+        except NotImplementedError:
+            continue
 
-def sigterm_handler(signo, stack_frame):
+    log.info("Waiting for remaining threads to stop; this may take a few seconds. If PyLink freezes "
+             "at this stage, press Ctrl-C to force a shutdown.")
+    _print_remaining_threads()
+
+    # Done.
+
+def _sigterm_handler(signo, stack_frame):
     """Handles SIGTERM and SIGINT gracefully by shutting down the PyLink daemon."""
     log.info("Shutting down on signal %s." % signo)
-    _shutdown()
+    shutdown()
 
-signal.signal(signal.SIGTERM, sigterm_handler)
+signal.signal(signal.SIGTERM, _sigterm_handler)
-signal.signal(signal.SIGINT, sigterm_handler)
+signal.signal(signal.SIGINT, _sigterm_handler)
 
-def _rehash():
+def rehash():
     """Rehashes the PyLink daemon."""
+    log.info('Reloading PyLink configuration...')
     old_conf = conf.conf.copy()
     fname = conf.fname
-    new_conf = conf.loadConf(fname, errors_fatal=False)
+    new_conf = conf.load_conf(fname, errors_fatal=False, logger=log)
-    new_conf = conf.validateConf(new_conf)
     conf.conf = new_conf
 
     # Reset any file logger options.
-    stopFileLoggers()
+    _stop_file_loggers()
     files = new_conf['logging'].get('files')
     if files:
         for filename, config in files.items():
-            makeFileLogger(filename, config.get('loglevel'))
+            _make_file_logger(filename, config.get('loglevel'))
+
+    log.debug('rehash: updating console log level')
+    world.console_handler.setLevel(_get_console_log_level())
+    login._make_cryptcontext()  # refresh password hashing settings
 
     for network, ircobj in world.networkobjects.copy().items():
         # Server was removed from the config file, disconnect them.
-        log.debug('rehash: checking if %r is in new conf still.', network)
+        log.debug('rehash: checking if %r is still in new conf.', network)
+        if ircobj.has_cap('virtual-server') or hasattr(ircobj, 'virtual_parent'):
+            log.debug('rehash: not removing network %r since it is a virtual server.', network)
+            continue
+
         if network not in new_conf['servers']:
             log.debug('rehash: removing connection to %r (removed from config).', network)
             remove_network(ircobj)
         else:
-            ircobj.conf = new_conf
+            # XXX: we should really just add abstraction to Irc to update config settings...
             ircobj.serverdata = new_conf['servers'][network]
-            ircobj.botdata = new_conf['bot']
+
+            ircobj.autoconnect_active_multiplier = 1
 
             # Clear the IRC object's channel loggers and replace them with
-            # new ones by re-running logSetup().
+            # new ones by re-running log_setup().
             while ircobj.loghandlers:
                 log.removeHandler(ircobj.loghandlers.pop())
 
-            ircobj.logSetup()
+            ircobj.log_setup()
 
-            # TODO: update file loggers here too.
+    utils._reset_module_dirs()
 
     for network, sdata in new_conf['servers'].items():
         # Connect any new networks or disconnected networks if they aren't already.
-        if (network not in world.networkobjects) or (not world.networkobjects[network].connection_thread.is_alive()):
+        if network not in world.networkobjects:
-            proto = utils.getProtocolModule(sdata['protocol'])
+            try:
-            world.networkobjects[network] = classes.Irc(network, proto, new_conf)
+                proto = utils._get_protocol_module(sdata['protocol'])
+
+                # API note: 2.0.x style of starting network connections
+                world.networkobjects[network] = newirc = proto.Class(network)
+                newirc.connect()
+            except:
+                log.exception('Failed to initialize network %r, skipping it...', network)
+
+    log.info('Finished reloading PyLink configuration.')
 
 if os.name == 'posix':
-    # Only register SIGHUP on *nix.
+    # Only register SIGHUP/SIGUSR1 on *nix.
-    def sighup_handler(_signo, _stack_frame):
+    def _sighup_handler(signo, _stack_frame):
-        """Handles SIGHUP by rehashing the PyLink daemon."""
+        """Handles SIGHUP/SIGUSR1 by rehashing the PyLink daemon."""
-        log.info("SIGHUP received, reloading config.")
+        log.info("Signal %s received, reloading config." % signo)
-        _rehash()
+        rehash()
 
-    signal.signal(signal.SIGHUP, sighup_handler)
+    signal.signal(signal.SIGHUP, _sighup_handler)
+    signal.signal(signal.SIGUSR1, _sighup_handler)

coremods/corecommands.py

@@ -2,66 +2,37 @@
 corecommands.py - Implements core PyLink commands.
 """
 
-# Get the package name that plugins are stored under.
-plugin_root = __name__.split('.')[0] + '.plugins.'
-
 import gc
 import sys
-import importlib
 
-from . import control
+from pylinkirc import utils, world
-from pylinkirc import utils, world, conf
 from pylinkirc.log import log
 
+from . import control, permissions
+
+__all__ = []
+
 # Essential, core commands go here so that the "commands" plugin with less-important,
 # but still generic functions can be reloaded.
 
-@utils.add_cmd
-def identify(irc, source, args):
-    """<username> <password>
-
-    Logs in to PyLink using the configured administrator account."""
-    if utils.isChannel(irc.called_in):
-        irc.reply('Error: This command must be sent in private. '
-                '(Would you really type a password inside a channel?)')
-        return
-    try:
-        username, password = args[0], args[1]
-    except IndexError:
-        irc.reply('Error: Not enough arguments.')
-        return
-    # Usernames are case-insensitive, passwords are NOT.
-    if username.lower() == conf.conf['login']['user'].lower() and password == conf.conf['login']['password']:
-        realuser = conf.conf['login']['user']
-        irc.users[source].account = realuser
-        irc.reply('Successfully logged in as %s.' % realuser)
-        log.info("(%s) Successful login to %r by %s",
-                 irc.name, username, irc.getHostmask(source))
-    else:
-        irc.reply('Error: Incorrect credentials.')
-        u = irc.users[source]
-        log.warning("(%s) Failed login to %r from %s",
-                    irc.name, username, irc.getHostmask(source))
-
 @utils.add_cmd
 def shutdown(irc, source, args):
     """takes no arguments.
 
     Exits PyLink by disconnecting all networks."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['core.shutdown'])
-    u = irc.users[source]
+    log.info('(%s) SHUTDOWN requested by %s, exiting...', irc.name, irc.get_hostmask(source))
-
+    control.shutdown(irc=irc)
-    log.info('(%s) SHUTDOWN requested by "%s!%s@%s", exiting...', irc.name, u.nick,
-             u.ident, u.host)
-
-    control._shutdown(irc)
 
 @utils.add_cmd
 def load(irc, source, args):
     """<plugin name>.
 
     Loads a plugin from the plugin folder."""
-    irc.checkAuthenticated(source, allowOper=False)
+    # Note: reload capability is acceptable here, because all it actually does is call
+    # load after unload.
+    permissions.check_permissions(irc, source, ['core.load', 'core.reload'])
+
     try:
         name = args[0]
     except IndexError:
@@ -70,9 +41,9 @@ def load(irc, source, args):
     if name in world.plugins:
         irc.reply("Error: %r is already loaded." % name)
         return
-    log.info('(%s) Loading plugin %r for %s', irc.name, name, irc.getHostmask(source))
+    log.info('(%s) Loading plugin %r for %s', irc.name, name, irc.get_hostmask(source))
     try:
-        world.plugins[name] = pl = utils.loadPlugin(name)
+        world.plugins[name] = pl = utils._load_plugin(name)
     except ImportError as e:
         if str(e) == ('No module named %r' % name):
             log.exception('Failed to load plugin %r: The plugin could not be found.', name)
@@ -82,7 +53,7 @@ def load(irc, source, args):
     else:
         if hasattr(pl, 'main'):
             log.debug('Calling main() function of plugin %r', pl)
-            pl.main(irc)
+            pl.main(irc=irc)
     irc.reply("Loaded plugin %r." % name)
 
 @utils.add_cmd
@@ -90,7 +61,8 @@ def unload(irc, source, args):
     """<plugin name>.
 
     Unloads a currently loaded plugin."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['core.unload', 'core.reload'])
+
     try:
         name = args[0]
     except IndexError:
@@ -99,10 +71,10 @@ def unload(irc, source, args):
 
     # Since we're using absolute imports in 0.9.x+, the module name differs from the actual plugin
     # name.
-    modulename = plugin_root + name
+    modulename = utils.PLUGIN_PREFIX + name
 
     if name in world.plugins:
-        log.info('(%s) Unloading plugin %r for %s', irc.name, name, irc.getHostmask(source))
+        log.info('(%s) Unloading plugin %r for %s', irc.name, name, irc.get_hostmask(source))
         pl = world.plugins[name]
         log.debug('sys.getrefcount of plugin %s is %s', pl, sys.getrefcount(pl))
 
@@ -122,18 +94,20 @@ def unload(irc, source, args):
                         del world.services['pylink'].commands[cmdname]
 
         # Remove any command hooks set by the plugin.
-        for hookname, hookfuncs in world.hooks.copy().items():
+        for hookname, hookpairs in world.hooks.copy().items():
-            for hookfunc in hookfuncs:
+            for hookpair in hookpairs:
+                hookfunc = hookpair[1]
                 if hookfunc.__module__ == modulename:
-                    world.hooks[hookname].remove(hookfunc)
+                    log.debug('Trying to remove hook func %s (%s) from plugin %s', hookfunc, hookname, modulename)
+                    world.hooks[hookname].remove(hookpair)
                     # If the hookfuncs list is empty, remove it.
-                    if not hookfuncs:
+                    if not hookpairs:
                         del world.hooks[hookname]
 
         # Call the die() function in the plugin, if present.
         if hasattr(pl, 'die'):
             try:
-                pl.die(irc)
+                pl.die(irc=irc)
             except:  # But don't allow it to crash the server.
                 log.exception('(%s) Error occurred in die() of plugin %s, skipping...', irc.name, pl)
 
@@ -163,6 +137,8 @@ def reload(irc, source, args):
     except IndexError:
         irc.reply("Error: Not enough arguments. Needs 1: plugin name.")
         return
+
+    # Note: these functions do permission checks, so there are none needed here.
     if unload(irc, source, args):
         load(irc, source, args)
 
@@ -171,13 +147,21 @@ def rehash(irc, source, args):
     """takes no arguments.
 
     Reloads the configuration file for PyLink, (dis)connecting added/removed networks.
-    Plugins must be manually reloaded."""
+
-    irc.checkAuthenticated(source, allowOper=False)
+    Note: plugins must be manually reloaded."""
+    permissions.check_permissions(irc, source, ['core.rehash'])
     try:
-        control._rehash()
+        control.rehash()
     except Exception as e:  # Something went wrong, abort.
-        log.exception("Error REHASHing config: ")
         irc.reply("Error loading configuration file: %s: %s" % (type(e).__name__, e))
         return
     else:
         irc.reply("Done.")
+
+@utils.add_cmd
+def clearqueue(irc, source, args):
+    """takes no arguments.
+
+    Clears the outgoing text queue for the current connection."""
+    permissions.check_permissions(irc, source, ['core.clearqueue'])
+    irc._queue.queue.clear()

coremods/exttargets.py

@@ -5,6 +5,9 @@ exttargets.py - Implements extended targets like $account:xyz, $oper, etc.
 from pylinkirc import world
 from pylinkirc.log import log
 
+__all__ = []
+
+
 def bind(func):
     """
     Binds an exttarget with the given name.
@@ -21,7 +24,7 @@ def account(irc, host, uid):
 
     $account -> Returns True (a match) if the target is registered.
     $account:accountname -> Returns True if the target's account name matches the one given, and the
-    target is connected to the local network..
+    target is connected to the local network.
     $account:accountname:netname -> Returns True if both the target's account name and origin
     network name match the ones given.
     $account:*:netname -> Matches all logged in users on the given network.
@@ -41,10 +44,10 @@ def account(irc, host, uid):
                           homenet, realuid)
             return False
 
-    slogin = irc.toLower(userobj.services_account)
+    slogin = irc.to_lower(str(userobj.services_account))
 
     # Split the given exttarget host into parts, so we know how many to look for.
-    groups = list(map(irc.toLower, host.split(':')))
+    groups = host.split(':')
     log.debug('(%s) exttargets.account: groups to match: %s', irc.name, groups)
 
     if len(groups) == 1:
@@ -52,12 +55,12 @@ def account(irc, host, uid):
         return bool(slogin)
     elif len(groups) == 2:
         # Second scenario. Return True if the user's account matches the one given.
-        return slogin == groups[1] and homenet == irc.name
+        return slogin == irc.to_lower(groups[1]) and homenet == irc.name
     else:
         # Third or fourth scenario. If there are more than 3 groups, the rest are ignored.
         # In other words: Return True if the user is logged in, the query matches either '*' or the
         # user's login, and the user is connected on the network requested.
-        return slogin and (groups[1] in ('*', slogin)) and (homenet == groups[2])
+        return slogin and (irc.to_lower(groups[1]) in ('*', slogin)) and (homenet == groups[2])
 
 @bind
 def ircop(irc, host, uid):
@@ -74,10 +77,10 @@ def ircop(irc, host, uid):
 
     if len(groups) == 1:
         # 1st scenario.
-        return irc.isOper(uid, allowAuthed=False)
+        return irc.is_oper(uid)
     else:
-        # 2nd scenario. Use matchHost (ircmatch) to match the opertype glob to the opertype.
+        # 2nd scenario. Match the opertype glob to the opertype.
-        return irc.matchHost(groups[1], irc.users[uid].opertype)
+        return irc.match_text(groups[1], irc.users[uid].opertype)
 
 @bind
 def server(irc, host, uid):
@@ -93,10 +96,10 @@ def server(irc, host, uid):
     log.debug('(%s) exttargets.server: groups to match: %s', irc.name, groups)
 
     if len(groups) >= 2:
-        sid = irc.getServer(uid)
+        sid = irc.get_server(uid)
         query = groups[1]
         # Return True if the SID matches the query or the server's name glob matches it.
-        return sid == query or irc.matchHost(query, irc.getFriendlyName(sid))
+        return sid == query or irc.match_text(query, irc.get_friendly_name(sid))
     # $server alone is invalid. Don't match anything.
     return False
 
@@ -118,12 +121,16 @@ def channel(irc, host, uid):
     except IndexError:  # No channel given, abort.
         return False
 
+    if channel not in irc.channels:
+        # Channel doesn't even exist...
+        return False
+
     if len(groups) == 2:
         # Just #channel was given as query
         return uid in irc.channels[channel].users
     elif len(groups) >= 3:
         # For things like #channel:op, check if the query is in the user's prefix modes.
-        return (uid in irc.channels[channel].users) and (groups[2].lower() in irc.channels[channel].getPrefixModes(uid))
+        return (uid in irc.channels[channel].users) and (groups[2].lower() in irc.channels[channel].get_prefix_modes(uid))
 
 @bind
 def pylinkacc(irc, host, uid):
@@ -134,8 +141,8 @@ def pylinkacc(irc, host, uid):
     $pylinkacc -> Returns True if the target is logged in to PyLink.
     $pylinkacc:accountname -> Returns True if the target's PyLink login matches the one given.
     """
-    login = irc.toLower(irc.users[uid].account)
+    login = irc.to_lower(irc.users[uid].account)
-    groups = list(map(irc.toLower, host.split(':')))
+    groups = list(map(irc.to_lower, host.split(':')))
     log.debug('(%s) exttargets.pylinkacc: groups to match: %s', irc.name, groups)
 
     if len(groups) == 1:
@@ -144,3 +151,79 @@ def pylinkacc(irc, host, uid):
     elif len(groups) == 2:
         # Second scenario. Return True if the user's login matches the one given.
         return login == groups[1]
+
+@bind
+def network(irc, host, uid):
+    """
+    $network exttarget handler. This exttarget takes one argument: a network name, and returns
+    a match for all users on that network.
+
+    Note: network names are case sensitive.
+    """
+    try:
+        targetnet = host.split(':')[1]
+    except IndexError:  # No network arg given, bail.
+        return False
+
+    userobj = irc.users[uid]
+    if hasattr(userobj, 'remote'):
+        # User is a PyLink Relay client; set the correct network name.
+        homenet = userobj.remote[0]
+    else:
+        homenet = irc.name
+
+    return homenet == targetnet
+
+# Note: "and" can't be a function name so we use this.
+def exttarget_and(irc, host, uid):
+    """
+    $and exttarget handler. This exttarget takes a series of exttargets (or hostmasks) joined with
+    a "+", and returns True if all sub exttargets match.
+
+    Examples:
+    $and:($ircop:*admin*+$network:ovd) -> Matches all opers on the network ovd.
+    $and:($account+$pylinkirc) -> Matches all users logged in to both services and PyLink.
+    $and:(*!*@localhost+$ircop) -> Matches all opers with the host `localhost`.
+    $and:(*!*@*.mibbit.com+!$ircop+!$account) -> Matches all mibbit users that aren't opered or logged in to services.
+    """
+    targets = host.split(':', 1)[-1]
+    # For readability, this requires that the exttarget list be wrapped in brackets.
+    if not (targets.startswith('(') and targets.endswith(')')):
+        return False
+
+    targets = targets[1:-1]
+    targets = list(filter(None, targets.split('+')))
+    log.debug('exttargets_and: using raw subtargets list %r (original query=%r)', targets, host)
+    # Wrap every subtarget into irc.match_host and return True if all subtargets return True.
+    return all(map(lambda sub_exttarget: irc.match_host(sub_exttarget, uid), targets))
+world.exttarget_handlers['and'] = exttarget_and
+
+@bind
+def realname(irc, host, uid):
+    """
+    $realname exttarget handler. This takes one argument: a glob, which is compared case-insensitively to the user's real name.
+
+    Examples:
+    $realname:*James* -> matches anyone with "James" in their real name.
+    """
+    groups = host.split(':')
+    if len(groups) >= 2:
+        return irc.match_text(groups[1], irc.users[uid].realname)
+
+@bind
+def service(irc, host, uid):
+    """
+    $service exttarget handler. This takes one optional argument: a glob, which is compared case-insensitively to the target user's service name (if present).
+
+    Examples:
+    $service -> Matches any PyLink service bot.
+    $service:automode -> Matches the Automode service bot.
+    """
+    if not irc.users[uid].service:
+        return False
+
+    groups = host.split(':')
+
+    if len(groups) >= 2:
+        return irc.match_text(groups[1], irc.users[uid].service)
+    return True  # It *is* a service bot because of the check at the top.

coremods/handlers.py

@@ -3,27 +3,30 @@ handlers.py - Implements miscellaneous IRC command handlers (WHOIS, services log
 """
 import time
 
-from pylinkirc import utils, conf
+from pylinkirc import conf, utils
 from pylinkirc.log import log
 
+__all__ = []
+
+
 def handle_whois(irc, source, command, args):
     """Handle WHOIS queries."""
     target = args['target']
     user = irc.users.get(target)
 
-    f = lambda num, source, text: irc.proto.numeric(irc.sid, num, source, text)
+    f = lambda num, source, text: irc.numeric(irc.sid, num, source, text)
 
     # Get the server that the target is on.
-    server = irc.getServer(target)
+    server = irc.get_server(target)
 
     if user is None:  # User doesn't exist
-        # <- :42X 401 7PYAAAAAB GL- :No such nick/channel
+        # <- :42X 401 7PYAAAAAB jlu5- :No such nick/channel
         nick = target
         f(401, source, "%s :No such nick/channel" % nick)
     else:
         nick = user.nick
-        sourceisOper = ('o', None) in irc.users[source].modes
+        source_is_oper = ('o', None) in irc.users[source].modes
-        sourceisBot = (irc.umodes.get('bot'), None) in irc.users[source].modes
+        source_is_bot = (irc.umodes.get('bot'), None) in irc.users[source].modes
 
         # Get the full network name.
         netname = irc.serverdata.get('netname', irc.name)
@@ -35,7 +38,7 @@ def handle_whois(irc, source, command, args):
         # 319: RPL_WHOISCHANNELS; Show public channels of the target, respecting
         # hidechans umodes for non-oper callers.
         isHideChans = (irc.umodes.get('hidechans'), None) in user.modes
-        if (not isHideChans) or (isHideChans and sourceisOper):
+        if (not isHideChans) or (isHideChans and source_is_oper):
             public_chans = []
             for chan in user.channels:
                 c = irc.channels[chan]
@@ -44,13 +47,13 @@ def handle_whois(irc, source, command, args):
 
                 if ((irc.cmodes.get('secret'), None) in c.modes or \
                     (irc.cmodes.get('private'), None) in c.modes) \
-                    and not (sourceisOper or source in c.users):
+                    and not (source_is_oper or source in c.users):
                         continue
 
                 # Show the highest prefix mode like a regular IRCd does, if there are any.
-                prefixes = c.getPrefixModes(target)
+                prefixes = c.get_prefix_modes(target)
                 if prefixes:
-                    highest = prefixes[-1]
+                    highest = prefixes[0]
 
                     # Fetch the prefix mode letter from the named mode.
                     modechar = irc.cmodes[highest]
@@ -74,22 +77,27 @@ def handle_whois(irc, source, command, args):
             # 2) +H is set, but the caller is oper
             # 3) +H is set, but whois_use_hideoper is disabled in config
             isHideOper = (irc.umodes.get('hideoper'), None) in user.modes
-            if (not isHideOper) or (isHideOper and sourceisOper) or \
+            if (not isHideOper) or (isHideOper and source_is_oper) or \
-                    (isHideOper and not irc.botdata.get('whois_use_hideoper', True)):
+                    (isHideOper and not conf.conf['pylink'].get('whois_use_hideoper', True)):
+                opertype = user.opertype
+
                 # Let's be gramatically correct. (If the opertype starts with a vowel,
                 # write "an Operator" instead of "a Operator")
-                n = 'n' if user.opertype[0].lower() in 'aeiou' else ''
+                n = 'n' if opertype[0].lower() in 'aeiou' else ''
 
-                # I want to normalize the syntax: PERSON is an OPERTYPE on NETWORKNAME.
+                # Remove the "(on $network)" bit in relay oper types if the target network is the
-                # This is the only syntax InspIRCd supports, but for others it doesn't
+                # same - this prevents duplicate text such as "jlu5/ovd is a Network Administrator
-                # really matter since we're handling the WHOIS requests by ourselves.
+                # (on OVERdrive-IRC) on OVERdrive-IRC" from showing.
-                f(313, source, "%s :is a%s %s on %s" % (nick, n, user.opertype, netname))
+                # XXX: does this post-processing really belong here?
+                opertype = opertype.replace(' (on %s)' % irc.get_full_network_name(), '')
+
+                f(313, source, "%s :is a%s %s" % (nick, n, opertype))
 
         # 379: RPL_WHOISMODES, used by UnrealIRCd and InspIRCd to show user modes.
         # Only show this to opers!
-        if sourceisOper:
+        if source_is_oper:
             f(378, source, "%s :is connecting from %s@%s %s" % (nick, user.ident, user.realhost, user.ip))
-            f(379, source, '%s :is using modes %s' % (nick, irc.joinModes(user.modes, sort=True)))
+            f(379, source, '%s :is using modes %s' % (nick, irc.join_modes(user.modes, sort=True)))
 
         # 301: used to show away information if present
         away_text = user.away
@@ -101,10 +109,14 @@ def handle_whois(irc, source, command, args):
             # Show botmode info in WHOIS.
             f(335, source, "%s :is a bot" % nick)
 
+        # :charybdis.midnight.vpn 317 jlu5 jlu5 1946 1499867833 :seconds idle, signon time
+        if irc.get_service_bot(target) and conf.conf['pylink'].get('whois_show_startup_time', True):
+            f(317, source, "%s 0 %s :seconds idle (placeholder), signon time" % (nick, irc.start_ts))
+
         # Call custom WHOIS handlers via the PYLINK_CUSTOM_WHOIS hook, unless the
         # caller is marked a bot and the whois_show_extensions_to_bots option is False
-        if (sourceisBot and conf.conf['bot'].get('whois_show_extensions_to_bots')) or (not sourceisBot):
+        if (source_is_bot and conf.conf['pylink'].get('whois_show_extensions_to_bots')) or (not source_is_bot):
-            irc.callHooks([source, 'PYLINK_CUSTOM_WHOIS', {'target': target, 'server': server}])
+            irc.call_hooks([source, 'PYLINK_CUSTOM_WHOIS', {'target': target, 'server': server}])
         else:
             log.debug('(%s) coremods.handlers.handle_whois: skipping custom whois handlers because '
                       'caller %s is marked as a bot', irc.name, source)
@@ -119,15 +131,15 @@ def handle_mode(irc, source, command, args):
     modes = args['modes']
     # If the sender is not a PyLink client, and the target IS a protected
     # client, revert any forced deoper attempts.
-    if irc.isInternalClient(target) and not irc.isInternalClient(source):
+    if irc.is_internal_client(target) and not irc.is_internal_client(source):
-        if ('-o', None) in modes and (target == irc.pseudoclient.uid or not irc.isManipulatableClient(target)):
+        if ('-o', None) in modes and (target == irc.pseudoclient.uid or not irc.is_manipulatable_client(target)):
-            irc.proto.mode(irc.sid, target, {('+o', None)})
+            irc.mode(irc.sid, target, {('+o', None)})
 utils.add_hook(handle_mode, 'MODE')
 
 def handle_operup(irc, source, command, args):
     """Logs successful oper-ups on networks."""
     otype = args.get('text', 'IRC Operator')
-    log.debug("(%s) Successful oper-up (opertype %r) from %s", irc.name, otype, irc.getHostmask(source))
+    log.debug("(%s) Successful oper-up (opertype %r) from %s", irc.name, otype, irc.get_hostmask(source))
     irc.users[source].opertype = otype
 
 utils.add_hook(handle_operup, 'CLIENT_OPERED')
@@ -146,11 +158,58 @@ def handle_version(irc, source, command, args):
     """Handles requests for the PyLink server version."""
     # 351 syntax is usually "<server version>. <server hostname> :<anything else you want to add>
     fullversion = irc.version()
-    irc.proto.numeric(irc.sid, 351, source, fullversion)
+    irc.numeric(irc.sid, 351, source, fullversion)
 utils.add_hook(handle_version, 'VERSION')
 
 def handle_time(irc, source, command, args):
     """Handles requests for the PyLink server time."""
     timestring = time.ctime()
-    irc.proto.numeric(irc.sid, 391, source, '%s :%s' % (irc.hostname(), timestring))
+    irc.numeric(irc.sid, 391, source, '%s :%s' % (irc.hostname(), timestring))
 utils.add_hook(handle_time, 'TIME')
+
+def _state_cleanup_core(irc, source, channel):
+    """
+    Handles PART and KICK on clientbot-like networks (where only the users and channels we see are available)
+    by deleting channels when we leave and users when they leave all shared channels.
+    """
+    if irc.has_cap('visible-state-only'):
+        # Delete channels that we were removed from.
+        if irc.pseudoclient and source == irc.pseudoclient.uid:
+            log.debug('(%s) state_cleanup: removing channel %s since we have left', irc.name, channel)
+            del irc._channels[channel]
+
+        # Delete external users no longer sharing a channel with us.
+        if (not irc.users[source].channels) and (not irc.is_internal_client(source)):
+            log.debug('(%s) state_cleanup: removing external user %s/%s who no longer shares a channel with us',
+                      irc.name, source, irc.users[source].nick)
+            irc._remove_client(source)
+
+    # Clear empty non-permanent channels.
+    if channel in irc.channels and not (irc._channels[channel].users or ((irc.cmodes.get('permanent'), None) \
+            in irc._channels[channel].modes)):
+        log.debug('(%s) state_cleanup: removing empty channel %s', irc.name, channel)
+        del irc._channels[channel]
+
+def _state_cleanup_part(irc, source, command, args):
+    for channel in args['channels']:
+        _state_cleanup_core(irc, source, channel)
+utils.add_hook(_state_cleanup_part, 'PART', priority=-100)
+
+def _state_cleanup_kick(irc, source, command, args):
+    _state_cleanup_core(irc, args['target'], args['channel'])
+utils.add_hook(_state_cleanup_kick, 'KICK', priority=-100)
+
+def _state_cleanup_mode(irc, source, command, args):
+    """
+    Cleans up and removes empty channels when -P (permanent mode) is removed from them.
+    """
+    target = args['target']
+    if target in irc.channels and 'permanent' in irc.cmodes:
+        c = irc.channels[target]
+        mode = '-%s' % irc.cmodes['permanent']
+
+        if (not c.users) and (mode, None) in args['modes']:
+            log.debug('(%s) _state_cleanup_mode: deleting empty channel %s as %s was set', irc.name, target, mode)
+            del irc._channels[target]
+            return False  # Block further hooks from running
+utils.add_hook(_state_cleanup_mode, 'MODE', priority=10000)

coremods/login.py

@@ -0,0 +1,139 @@
+"""
+login.py - Implement core login abstraction.
+"""
+
+from pylinkirc import conf, utils
+from pylinkirc.log import log
+
+__all__ = ['pwd_context', 'check_login', 'verify_hash']
+
+
+# PyLink's global password context
+pwd_context = None
+
+_DEFAULT_CRYPTCONTEXT_SETTINGS = {
+    'schemes': ["pbkdf2_sha256", "sha512_crypt"]
+}
+def _make_cryptcontext():
+    try:
+        from passlib.context import CryptContext
+    except ImportError:
+        log.warning("Hashed passwords are disabled because passlib is not installed. Please install "
+                    "it (pip3 install passlib) and rehash for this feature to work.")
+        return
+
+    context_settings = conf.conf.get('login', {}).get('cryptcontext_settings') or _DEFAULT_CRYPTCONTEXT_SETTINGS
+    global pwd_context
+    if pwd_context is None:
+        log.debug("Initialized new CryptContext with settings: %s", context_settings)
+        pwd_context = CryptContext(**context_settings)
+    else:
+        log.debug("Updated CryptContext with settings: %s", context_settings)
+        pwd_context.update(**context_settings)
+
+_make_cryptcontext()  # This runs at startup and in rehash (control.py)
+
+def _get_account(accountname):
+    """
+    Returns the login data block for the given account name (case-insensitive), or False if none
+    exists.
+    """
+    accounts = {k.lower(): v for k, v in
+                conf.conf['login'].get('accounts', {}).items()}
+
+    try:
+        return accounts[accountname.lower()]
+    except KeyError:
+        return False
+
+def check_login(user, password):
+    """Checks whether the given user and password is a valid combination."""
+    account = _get_account(user)
+
+    if account:
+        passhash = account.get('password')
+        if not passhash:
+            # No password given, return. XXX: we should allow plugins to override
+            # this in the future.
+            return False
+
+        # Hashing in account passwords is optional.
+        if account.get('encrypted', False):
+            return verify_hash(password, passhash)
+        else:
+            return password == passhash
+
+    return False
+
+def verify_hash(password, passhash):
+    """Checks whether the password given matches the hash."""
+    if password:
+        if not pwd_context:
+            raise utils.NotAuthorizedError("Cannot log in to an account with a hashed password "
+                                           "because passlib is not installed.")
+
+        return pwd_context.verify(password, passhash)
+    return False  # No password given!
+
+def _irc_try_login(irc, source, username, skip_checks=False):
+    """Internal function to process logins via IRC."""
+    if irc.is_internal_client(source):
+        irc.error("Cannot use 'identify' via a command proxy.")
+        return
+
+    if not skip_checks:
+        logindata = _get_account(username)
+
+        network_filter = logindata.get('networks')
+        require_oper = logindata.get('require_oper', False)
+        hosts_filter = logindata.get('hosts', [])
+
+        if network_filter and irc.name not in network_filter:
+            log.warning("(%s) Failed login to %r from %s (wrong network: networks filter says %r but we got %r)",
+                        irc.name, username, irc.get_hostmask(source), ', '.join(network_filter), irc.name)
+            raise utils.NotAuthorizedError("Account is not authorized to login on this network.")
+
+        elif require_oper and not irc.is_oper(source):
+            log.warning("(%s) Failed login to %r from %s (needs oper)", irc.name, username, irc.get_hostmask(source))
+            raise utils.NotAuthorizedError("You must be opered.")
+
+        elif hosts_filter and not any(irc.match_host(host, source) for host in hosts_filter):
+            log.warning("(%s) Failed login to %r from %s (hostname mismatch)", irc.name, username, irc.get_hostmask(source))
+            raise utils.NotAuthorizedError("Hostname mismatch.")
+
+    irc.users[source].account = username
+    irc.reply('Successfully logged in as %s.' % username)
+    log.info("(%s) Successful login to %r by %s",
+             irc.name, username, irc.get_hostmask(source))
+    return True
+
+def identify(irc, source, args):
+    """<username> <password>
+
+    Logs in to PyLink using the configured administrator account."""
+    if irc.is_channel(irc.called_in):
+        irc.reply('Error: This command must be sent in private. '
+                  '(Would you really type a password inside a channel?)')
+        return
+    try:
+        username, password = args[0], args[1]
+    except IndexError:
+        irc.reply('Error: Not enough arguments.')
+        return
+
+    # Process new-style accounts.
+    if check_login(username, password):
+        _irc_try_login(irc, source, username)
+        return
+
+    # Process legacy logins (login:user).
+    if username.lower() == conf.conf['login'].get('user', '').lower() and password == conf.conf['login'].get('password'):
+        realuser = conf.conf['login']['user']
+        _irc_try_login(irc, source, realuser, skip_checks=True)
+        return
+
+    # Username not found or password incorrect.
+    log.warning("(%s) Failed login to %r from %s", irc.name, username, irc.get_hostmask(source))
+    raise utils.NotAuthorizedError('Bad username or password.')
+
+utils.add_cmd(identify, aliases=('login', 'id'))

coremods/permissions.py

@@ -0,0 +1,67 @@
+"""
+permissions.py - Permissions Abstraction for PyLink IRC Services.
+"""
+
+from collections import defaultdict
+
+from pylinkirc import conf, utils
+from pylinkirc.log import log
+
+__all__ = ['default_permissions', 'add_default_permissions',
+           'remove_default_permissions', 'check_permissions']
+
+# Global variables: these store mappings of hostmasks/exttargets to lists of permissions each target has.
+default_permissions = defaultdict(set)
+
+
+def add_default_permissions(perms):
+    """Adds default permissions to the index."""
+    global default_permissions
+    for target, permlist in perms.items():
+        default_permissions[target] |= set(permlist)
+addDefaultPermissions = add_default_permissions
+
+def remove_default_permissions(perms):
+    """Remove default permissions from the index."""
+    global default_permissions
+    for target, permlist in perms.items():
+        default_permissions[target] -= set(permlist)
+removeDefaultPermissions = remove_default_permissions
+
+def check_permissions(irc, uid, perms, also_show=[]):
+    """
+    Checks permissions of the caller. If the caller has any of the permissions listed in perms,
+    this function returns True. Otherwise, NotAuthorizedError is raised.
+    """
+    # For old (< 1.1 login blocks):
+    # If the user is logged in, they automatically have all permissions.
+    olduser = conf.conf['login'].get('user')
+    if olduser and irc.match_host('$pylinkacc:%s' % olduser, uid):
+        log.debug('permissions: overriding permissions check for old-style admin user %s',
+                  irc.get_hostmask(uid))
+        return True
+
+    permissions = defaultdict(set)
+    # Enumerate the configured permissions list.
+    for k, v in (conf.conf.get('permissions') or {}).items():
+        permissions[k] |= set(v)
+
+    # Merge in default permissions if enabled.
+    if conf.conf.get('permissions_merge_defaults', True):
+        for k, v in default_permissions.items():
+            permissions[k] |= v
+
+    for host, permlist in permissions.items():
+        log.debug('permissions: permlist for %s: %s', host, permlist)
+        if irc.match_host(host, uid):
+            # Now, iterate over all the perms we are looking for.
+            for perm in permlist:
+                # Use irc.match_host to expand globs in an IRC-case insensitive and wildcard
+                # friendly way. e.g. 'xyz.*.#Channel\' will match 'xyz.manage.#channel|' on IRCds
+                # using the RFC1459 casemapping.
+                log.debug('permissions: checking if %s glob matches anything in %s', perm, permlist)
+                if any(irc.match_host(perm, p) for p in perms):
+                    return True
+    raise utils.NotAuthorizedError("You are missing one of the following permissions: %s" %
+                                   (', '.join(perms+also_show)))
+checkPermissions = check_permissions

coremods/service_support.py

@@ -2,9 +2,12 @@
 service_support.py - Implements handlers for the PyLink ServiceBot API.
 """
 
-from pylinkirc import utils, world, conf
+from pylinkirc import conf, utils, world
 from pylinkirc.log import log
 
+__all__ = []
+
+
 def spawn_service(irc, source, command, args):
     """Handles new service bot introductions."""
 
@@ -14,55 +17,67 @@ def spawn_service(irc, source, command, args):
     # Service name
     name = args['name']
 
+    if name != 'pylink' and not irc.has_cap('can-spawn-clients'):
+        log.debug("(%s) Not spawning service %s because the server doesn't support spawning clients",
+                  irc.name, name)
+        return
+
     # Get the ServiceBot object.
     sbot = world.services[name]
 
-    # Look up the nick or ident in the following order:
+    old_userobj = irc.users.get(sbot.uids.get(irc.name))
-    # 1) Network specific nick/ident settings for this service (servers::irc.name::servicename_nick)
+    if old_userobj and old_userobj.service:
-    # 2) Global settings for this service (servicename::nick)
+        # A client already exists, so don't respawn it.
-    # 3) The preferred nick/ident combination defined by the plugin (sbot.nick / sbot.ident)
+        log.debug('(%s) spawn_service: Not respawning service %r as service client %r already exists.', irc.name, name,
-    # 4) The literal service name.
+                  irc.pseudoclient.nick)
-    # settings, and then falling back to the literal service name.
+        return
-    nick = irc.serverdata.get("%s_nick" % name) or conf.conf.get(name, {}).get('nick') or sbot.nick or name
-    ident = irc.serverdata.get("%s_ident" % name) or conf.conf.get(name, {}).get('ident') or sbot.ident or name
 
-    # TODO: make this configurable?
+    if name == 'pylink' and irc.pseudoclient:
-    host = irc.hostname()
+        # irc.pseudoclient already exists, reuse values from it but
+        # spawn a new client. This is used for protocols like Clientbot,
+        # so that they can override the main service nick, among other things.
+        log.debug('(%s) spawn_service: Using existing nick %r for service %r', irc.name, irc.pseudoclient.nick, name)
+        userobj = irc.pseudoclient
+        userobj.opertype = "PyLink Service"
+        userobj.manipulatable = sbot.manipulatable
+    else:
+        # No client exists, spawn a new one
+        nick = sbot.get_nick(irc)
+        ident = sbot.get_ident(irc)
+        host = sbot.get_host(irc)
+        realname = sbot.get_realname(irc)
 
-    # Spawning service clients with these umodes where supported. servprotect usage is a
+        # Spawning service clients with these umodes where supported. servprotect usage is a
-    # configuration option.
+        # configuration option.
-    preferred_modes = ['oper', 'hideoper', 'hidechans', 'invisible', 'bot']
+        preferred_modes = ['oper', 'hideoper', 'hidechans', 'invisible', 'bot']
-    modes = []
+        modes = []
 
-    if conf.conf['bot'].get('protect_services'):
+        if conf.conf['pylink'].get('protect_services'):
-        preferred_modes.append('servprotect')
+            preferred_modes.append('servprotect')
 
-    for mode in preferred_modes:
+        for mode in preferred_modes:
-        mode = irc.umodes.get(mode)
+            mode = irc.umodes.get(mode)
-        if mode:
+            if mode:
-            modes.append((mode, None))
+                modes.append((mode, None))
 
-    # Track the service's UIDs on each network.
+        # Track the service's UIDs on each network.
-    userobj = irc.proto.spawnClient(nick, ident, host, modes=modes, opertype="PyLink Service",
+        log.debug('(%s) spawn_service: Spawning new client %s for service %s', irc.name, nick, name)
-                                    manipulatable=sbot.manipulatable)
+        userobj = irc.spawn_client(nick, ident, host, modes=modes, opertype="PyLink Service",
+                                   realname=realname, manipulatable=sbot.manipulatable)
+
+    # Store the service name in the User object for easier access.
+    userobj.service = name
 
     sbot.uids[irc.name] = u = userobj.uid
 
     # Special case: if this is the main PyLink client being spawned,
     # assign this as irc.pseudoclient.
-    if name == 'pylink':
+    if name == 'pylink' and not irc.pseudoclient:
+        log.debug('(%s) spawn_service: irc.pseudoclient set to UID %s', irc.name, u)
         irc.pseudoclient = userobj
 
-    # TODO: channels should be tracked in a central database, not hardcoded
+    # Enumerate & join network defined channels.
-    # in conf.
+    sbot.join(irc, sbot.get_persistent_channels(irc))
-    channels = set(irc.serverdata.get('channels', [])) | sbot.extra_channels.get(irc.name, set())
-
-    for chan in channels:
-        if utils.isChannel(chan):
-            irc.proto.join(u, chan)
-            irc.callHooks([irc.sid, 'PYLINK_SERVICE_JOIN', {'channel': chan, 'users': [u]}])
-        else:
-            log.warning('(%s) Ignoring invalid autojoin channel %r.', irc.name, chan)
 
 utils.add_hook(spawn_service, 'PYLINK_NEW_SERVICE')
 
@@ -86,24 +101,76 @@ def handle_endburst(irc, source, command, args):
         for name, sbot in world.services.items():
             spawn_service(irc, source, command, {'name': name})
 
-utils.add_hook(handle_endburst, 'ENDBURST')
+utils.add_hook(handle_endburst, 'ENDBURST', priority=500)
 
 def handle_kill(irc, source, command, args):
     """Handle KILLs to PyLink service bots, respawning them as needed."""
     target = args['target']
-    sbot = irc.isServiceBot(target)
+    if irc.pseudoclient and target == irc.pseudoclient.uid:
-    if sbot:
+        irc.pseudoclient = None
-        spawn_service(irc, source, command, {'name': sbot.name})
+    userdata = args.get('userdata')
-        return
+    sbot = irc.get_service_bot(target)
+    servicename = None
+
+    if userdata and hasattr(userdata, 'service'):  # Look for the target's service name attribute
+        servicename = userdata.service
+    elif sbot:  # Or their service bot instance
+        servicename = sbot.name
+    if servicename:
+        log.info('(%s) Received kill to service %r (nick: %r) from %s (reason: %r).', irc.name, servicename,
+                 userdata.nick if userdata else irc.users[target].nick, irc.get_hostmask(source), args.get('text'))
+        spawn_service(irc, source, command, {'name': servicename})
+
 utils.add_hook(handle_kill, 'KILL')
 
+def handle_join(irc, source, command, args):
+    """Monitors channel joins for dynamic service bot joining."""
+    if irc.has_cap('visible-state-only'):
+        # No-op on bot-only servers.
+        return
+
+    channel = args['channel']
+    users = irc.channels[channel].users
+    for servicename, sbot in world.services.items():
+        if channel in sbot.get_persistent_channels(irc) and \
+                sbot.uids.get(irc.name) not in users:
+            log.debug('(%s) Dynamically joining service %r to channel %r.', irc.name, servicename, channel)
+            sbot.join(irc, channel)
+utils.add_hook(handle_join, 'JOIN')
+utils.add_hook(handle_join, 'PYLINK_SERVICE_JOIN')
+
+def _services_dynamic_part(irc, channel):
+    """Dynamically removes service bots from empty channels."""
+    if irc.has_cap('visible-state-only'):
+        # No-op on bot-only servers.
+        return
+    if irc.serverdata.get('join_empty_channels', conf.conf['pylink'].get('join_empty_channels', False)):
+        return
+
+    # If all remaining users in the channel are service bots, make them all part.
+    if all(irc.get_service_bot(u) for u in irc.channels[channel].users):
+        for u in irc.channels[channel].users.copy():
+            sbot = irc.get_service_bot(u)
+            if sbot:
+                log.debug('(%s) Dynamically parting service %r from channel %r.', irc.name, sbot.name, channel)
+                irc.part(u, channel)
+        return True
+
+def handle_part(irc, source, command, args):
+    """Monitors channel joins for dynamic service bot joining."""
+    for channel in args['channels']:
+        _services_dynamic_part(irc, channel)
+utils.add_hook(handle_part, 'PART')
+
 def handle_kick(irc, source, command, args):
     """Handle KICKs to the PyLink service bots, rejoining channels as needed."""
-    kicked = args['target']
     channel = args['channel']
-    if irc.isServiceBot(kicked):
+    # Skip autorejoin routines if the channel is now empty.
-        irc.proto.join(kicked, channel)
+    if not _services_dynamic_part(irc, channel):
-        irc.callHooks([irc.sid, 'PYLINK_SERVICE_JOIN', {'channel': channel, 'users': [kicked]}])
+        kicked = args['target']
+        sbot = irc.get_service_bot(kicked)
+        if sbot and channel in sbot.get_persistent_channels(irc):
+            sbot.join(irc, channel)
 utils.add_hook(handle_kick, 'KICK')
 
 def handle_commands(irc, source, command, args):
@@ -111,18 +178,14 @@ def handle_commands(irc, source, command, args):
     target = args['target']
     text = args['text']
 
-    sbot = irc.isServiceBot(target)
+    sbot = irc.get_service_bot(target)
     if sbot:
         sbot.call_cmd(irc, source, text)
 
 utils.add_hook(handle_commands, 'PRIVMSG')
 
 # Register the main PyLink service. All command definitions MUST go after this!
-mynick = conf.conf['bot'].get("nick", "PyLink")
+# TODO: be more specific in description, and possibly allow plugins to modify this to mention
-myident = conf.conf['bot'].get("ident", "pylink")
-
-# TODO: be more specific, and possibly allow plugins to modify this to mention
 # their features?
-mydesc = "\x02%s\x02 provides extended network services for IRC." % mynick
+mydesc = "\x02PyLink\x02 provides extended network services for IRC."
-
+utils.register_service('pylink', default_nick="PyLink", desc=mydesc, manipulatable=True)
-utils.registerService('pylink', nick=mynick, ident=myident, desc=mydesc, manipulatable=True)

docs/README.md

@@ -5,8 +5,19 @@ This folder contains general documentation for PyLink IRC services.
 ## Contents
 
 - [PyLink FAQ (Frequently Asked Questions)](faq.md)
-- [PyLink Relay Tutorial & Oper Guide](pylink-opers.md)
+- [PyLink Relay Quick Start Guide](relay-quickstart.md)
-- [Automode & Exttargets Guide](automode.md)
+
+----
+
+- [Automode Tutorial](automode.md)
+- [Advanced Relay Configuration](advanced-relay-config.md)
+- [Advanced Services Configuration](advanced-services-config.md)
+- [Extended Targets (Exttargets) Guide](exttargets.md)
+- [PyLink Permissions Reference](permissions-reference.md)
+
+----
+
+- [PyLink named modes tables](modelists/)
 - [Developer documentation](technical/)
 
-There is also a (WIP) Doxygen-powered API reference at https://pylink.github.io/
+There is also a Doxygen-powered API reference at https://pylink.github.io/

docs/advanced-relay-config.md

@@ -0,0 +1,93 @@
+# Advanced Configuration for PyLink Relay
+
+PyLink Relay provides a few configuration options not documented in the example configuration, either because they have limited use or are too complicated to be described briefly.
+
+**This guide assumes that you are relatively familiar with the way YAML syntax works (lists, named arrays/dicts, etc.).** In this document, configuration options will be referred to in the format `a::b::c`, which represents the "`c`" option inside a "`b`" config block, all within an "`a`" config block.
+
+In actual YAML, that translates to this:
+
+```yaml
+a:
+    b:
+        c: "some value"
+```
+
+### Custom Clientbot Styles
+
+Custom Clientbot styles can be applied for any of Clientbot's supported events, by defining keys in the format `relay::clientbot_styles::<event name>`. As of PyLink 2.1, you can also override options per network by defining them in the form `servers::<network name>::relay_clientbot_styles::<event names>`
+
+See below for a list of supported events and their default values (as of PyLink 2.1).
+
+A common use case for this feature is to turn off or adjust colors/formatting; this is explicitly documented [below](#disabling-colorscontrol-codes).
+
+These options take template strings as documented here: https://docs.python.org/3/library/string.html#template-strings. Supported substitution values differ by event, but usually include the [hook values for each](technical/hooks-reference.md#irc-command-hooks), *plus* the following:
+
+- For all events:
+    - `$netname`: origin network name
+    - `$sender`: nick of sender
+    - `$sender_identhost`: ident@host string of the sender
+    - `$colored_sender`: color hashed version of `$sender`
+    - `$colored_netname`: color hashed version of `$netname`
+- For KICK, and other events that have a `$target` field corresponding to a user:
+    - `$target_nick`: the nick of the target (as opposed to `$target`, which is an user ID)
+- For events that have a `$channel` field attached (e.g. JOIN, PART):
+    - `$local_channel`: the *local* channel name (i.e. the channel on the clientbot network)
+    - `$channel`: the real channel name on the sender's network
+    - `$mode_prefix`: the highest prefix mode of the sender, if they are a user. This is normally either empty or one of (common prefix modes) `~&!@%+`.
+- For SJOIN, SQUIT:
+    - `$nicks`: a comma-joined list of nicks that were bursted
+    - `$colored_nicks`: a comma-joined list of each bursted nick, color hashed
+
+To disable relaying for any specific event, set the template string to an empty string (`''`).
+
+#### List of supported events
+
+|Event name|Default value|
+| :---: | :--- |
+MESSAGE  | \x02[$netname]\x02 <$colored\_sender> $text
+KICK     | \x02[$netname]\x02 - $colored_sender$sender\_identhost has kicked $target_nick from $channel ($text)
+PART     | \x02[$netname]\x02 - $colored_sender$sender\_identhost has left $channel ($text)
+JOIN     | \x02[$netname]\x02 - $colored_sender$sender\_identhost has joined $channel
+NICK     | \x02[$netname]\x02 - $colored_sender$sender\_identhost is now known as $newnick
+QUIT     | \x02[$netname]\x02 - $colored_sender$sender\_identhost has quit ($text)
+ACTION   | \x02[$netname]\x02 * $colored\_sender $text
+NOTICE   | \x02[$netname]\x02 - Notice from $colored\_sender: $text
+SQUIT    | \x02[$netname]\x02 - Netsplit lost users: $colored\_nicks
+SJOIN    | \x02[$netname]\x02 - Netjoin gained users: $colored\_nicks
+MODE     | \x02[$netname]\x02 - $colored_sender$sender_identhost sets mode $modes on $channel
+PM       | PM from $sender on $netname: $text
+PNOTICE  | <$sender> $text
+
+- Note: the `PM` and `PNOTICE` events represent private messages and private notices respectively, when they're relayed to users behind a Clientbot link.
+- Note 2: as of 1.1.x, all public channel events are sent to channels as PRIVMSG, while `PM` and `PNOTICE` are relayed privately as NOTICE.
+
+#### Disabling Colors/Control Codes
+
+If you don't want the messages PyLink sends for clientbot messages to be emboldened or colored,
+remove all escape sequences (e.g. `\x02`) from the format template and replace the colored variants
+of applicable substitutions with their non-colored versions.
+
+This is a example clientbot_styles config block, which you can copy *into* your `relay` configuration block.
+(*Do not* make multiple `relay` config blocks, or duplicate any config blocks with the same name!)
+
+```yaml
+    clientbot_styles:
+        ACTION: "[$netname] * $sender $text"
+        JOIN: "[$netname] - $sender$sender_identhost has joined $channel"
+        KICK: "[$netname] - $sender$sender_identhost has kicked $target_nick from $channel ($text)"
+        MESSAGE: "[$netname] <$sender> $text"
+        MODE: "[$netname] - $sender$sender_identhost sets mode $modes on $channel"
+        NICK: "[$netname] - $sender$sender_identhost is now known as $newnick"
+        NOTICE: "[$netname] - Notice from $sender: $text"
+        PART: "[$netname] - $sender$sender_identhost has left $channel ($text)"
+        PM: "PM from $sender on $netname: $text"
+        PNOTICE: "<$sender> $text"
+        QUIT: "[$netname] - $sender$sender_identhost has quit ($text)"
+        SJOIN: "[$netname] - Netjoin gained users: $nicks"
+        SQUIT: "[$netname] - Netsplit lost users: $nicks"
+```
+
+### Misc. options
+- `relay::clientbot_startup_delay`: Defines the amount of seconds Clientbot should wait after startup, before relaying any non-PRIVMSG events. This is used to prevent excess floods when the bot connects. Defaults to 5 seconds.
+- `servers::NETNAME::relay_force_slashes`: This network specific option forces Relay to use `/` in nickname separators. You should only use this option on TS6 or P10 variants that are less strict with nickname validation, as **it will cause protocol violations** on most IRCds. UnrealIRCd and InspIRCd users do not need to set this either, as `/` in nicks is automatically enabled.
+- `servers::NETNAME::relay_endburst_delay`: InspIRCd networks only: sets the endburst delay for relay subservers. If relay server bursts are causing +j (join flood) protection to trigger, raising this value can work around the issue.

docs/advanced-services-config.md

@@ -0,0 +1,61 @@
+# Advanced Services Configuration
+
+There are some service configuration options that you may want to be aware of.
+
+#### Nick / Ident
+
+You can override the `nick` or `ident` of a service bot using a directive liek this:
+
+```yaml
+servers:
+    somenet:
+        # ...
+        SERVICE_nick: OTHERNICK
+        SERVICE_ident: OTHERIDENT
+```
+
+You can also set an arbitrary nick/ident using a per-**service** directive.
+
+```yaml
+SERVICE:
+    nick: OTHERNICK
+    ident: OTHERIDENT
+```
+
+#### joinmodes
+
+By default, service bots join channels without giving themselves any modes. You can configure what modes a service bot joins channels with using this directive:
+
+```yaml
+SERVICE:
+    joinmodes: 'o'
+```
+
+This would request the mode 'o' (op on most IRCds) when joining the channel.
+
+Technically any mode can be put here, but if an IRCd in question doesn't support
+the mode then it will be ignored.
+
+You can also use combinations of modes, such as 'ao' (usually admin/protect + op)
+
+```yaml
+SERVICE:
+    joinmodes: 'ao'
+```
+
+Combinations should work provided an IRCd in question supports it.
+
+#### Fantasy prefix
+
+You can also set the service bot's fantasy prefix; of course this is only
+applicable if the `fantasy` plugin is loaded.
+
+The setting allows for one or more characters to be set as the prefix.
+
+```yaml
+SERVICE:
+    prefix: './'
+```
+
+The above is perfectly valid, as is any other string.
+

docs/automode.md

@@ -1,44 +1,37 @@
 # Automode Tutorial
 
-The Automode plugin was introduced in PyLink 0.9 as a simple way of managing channel access control lists. That said, it is not designed to entirely replace traditional IRC services such as ChanServ.
+The Automode plugin was introduced in PyLink 0.9 as a simple mechanism to manage channel access. That said, it is not designed to entirely replace traditional IRC services such as ChanServ.
 
 ## Starting steps
 
-Upon loading the `automode` plugin, you should see a ModeBot service client connect, using the name you defined. This client provides the commands used to manage access.
+Upon loading the `automode` plugin, you should see an Automode service bot connect, using the name that you defined (this guide uses the default, `Automode`). This service provides the commands used to manage access.
 
 For a list of commands:
-- `/msg ModeBot help`
+- `/msg Automode help`
 
-Adding access lists to a channel:
+Adding access to a channel:
-- `/msg ModeBot setacc #channel [MASK] [MODE LIST]`
+- `/msg Automode setacc #channel [MASK] [MODE LIST]`
-- The mask can be a simple `nick!user@host` hostmask or any of the extended targets (exttargets) mentioned below. MODE LIST is a string of any prefix modes that you want to set (no `+` before needed), such as `qo`, `h`, or `ov`.
+- The mask can be a simple `nick!user@host` hostmask or any of the extended targets (exttargets) mentioned below. MODE LIST is a string of any prefix modes that you want to set (no leading `+` needed): e.g. `qo`, `h`, or `ov`.
 
 Removing access from a channel:
-- `/msg ModeBot delacc #channel [MASK]`
+- `/msg Automode delacc #channel [MASK]`
 
 Listing access entries on a channel:
-- `/msg ModeBot listacc #channel`
+- `/msg Automode listacc #channel`
 
 Applying all access entries on a channel (sync):
-- `/msg ModeBot syncacc #channel`
+- `/msg Automode syncacc #channel`
 
 Clearing all access entries on a channel:
-- `/msg ModeBot clearacc #channel`
+- `/msg Automode clearacc #channel`
 
 ## Supported masks and extended targets
+Automode supports any hostmask or PyLink extended target; see the [Exttargets Guide](exttargets.md) for more details.
 
-Extended targets or exttargets *replace* regular hostmasks with conditional matching based on the given situation. The following exttargets are supported:
+## Permissions
 
-- `$account` -> Returns True (a match) if the target is registered.
+See the [Permissions Reference](permissions-reference.md#automode) for a list of permissions defined by Automode.
-- `$account:accountname` -> Returns True if the target's account name matches the one given, and the target is connected to the local network. Account names are case insensitive.
+
-- `$account:accountname:netname` -> Returns True if both the target's account name and origin network name match the ones given. Account names are case insensitive, but network names ARE case sensitive.
+## Caveats
-- `$account:*:netname` -> Matches all logged in users on the given network. Globs are not supported here; only a literal `*`.
+
-- `$ircop` -> Returns True (a match) if the target is opered.
+- Service bot joining and Relay don't always behave consistently: see https://github.com/jlu5/PyLink/issues/265
-- `$ircop:*admin*` -> Returns True if the target's is opered and their oper type matches the glob given (case insensitive).
-- `$server:server.name` -> Returns True (a match) if the target is connected on the given server. Server names are matched case insensitively.
-- `$server:*server.glob*` -> Returns True (a match) if the target is connected on a server matching the glob.
-- `$server:1XY` -> Returns True if the target's is connected on the server with the given SID. Note: SIDs ARE case sensitive.
-- `$channel:#channel` -> Returns True if the target is in the given channel (case insensitive).
-- `$channel:#channel:op` -> Returns True if the target is in the given channel, and is opped. Any supported prefix mode (owner, admin, op, halfop, voice) can be used for the last part, but only one at a time.
-- `$pylinkacc` -> Returns True if the target is logged in to PyLink.
-- `$pylinkacc:accountname` -> Returns True if the target's PyLink login matches the one given (case insensitive).

docs/exttargets.md

@@ -0,0 +1,72 @@
+# Exttargets Guide
+
+**Extended targets** or **exttargets** extend regular hostmask matching by checking users against specific conditions. PyLink exttargets are supported by most plugins in the place of `nick!user@host` masks (provided they use `IRCNetwork.match_host()` as their backend).
+
+Exttargets were introduced in PyLink 0.9 alongside [Automode](automode.md), with the goal of making user/ACL matching more versatile. As of PyLink 2.0, the following exttargets are supported:
+
+### The "$account" target (PyLink 0.9+)
+Used to match users by their services account.
+
+- `$account` -> Returns True (a match) if the target is registered.
+- `$account:accountname` -> Returns True if the target's account name matches the one given, and the target is connected to the local network. Account names are case insensitive.
+- `$account:accountname:netname` -> Returns True if both the target's account name and origin network name match the ones given. Account names are case insensitive, but network names ARE case sensitive.
+- `$account:*:netname` -> Matches all logged in users on the given network. Globs are not supported here; only a literal `*`.
+
+### The "$channel" target (PyLink 0.9+)
+Used to match users in certain channels. Channel names are matched case insensitively.
+
+- `$channel:#channel` -> Returns True if the target is in the given channel.
+- `$channel:#channel:PREFIXMODE` -> Returns True if the target is in the given channel, and is opped. Any supported prefix mode (owner, admin, op, halfop, voice) can be used for the last part, but only one at a time.
+
+### The "$ircop" target (PyLink 0.9+)
+Used to match users by IRCop status.
+
+- `$ircop` -> Returns True (a match) if the target is opered.
+- `$ircop:*admin*` -> Returns True if the target's is opered and their oper type matches the glob given (case insensitive).
+
+### Target inversion (PyLink 1.2+)
+In PyLink 1.2 and above, all targets and hostmasks can be inverted by placing a `!` before the target:
+
+- `!$account` -> Matches all users not registered with services.
+- `!$ircop` -> Matches all non-opers.
+- `!*!*@localhost` -> Matches all users not connecting from localhost.
+- `!*!*@*:*` -> Matches all non-IPv6 users.
+
+For users on PyLink version **before 1.2**, target inversion is *only* supported with exttargets (i.e. `!$account` will work, but not `!*!*@localhost`.
+
+### The "$network" target (PyLink 1.2+)
+Used to match users on specific networks.
+
+- `$network:netname` -> Returns True if the target user originates from the given network (this supports and looks up the home network of Relay users).
+
+### The "$and" target (PyLink 1.2+)
+The `$and` target is slightly more complex, and involves chaining together multiple exttargets or hosts with a `+` between each. Note that parentheses are required around the list of targets to match.
+
+- `$and:($ircop:*admin*+$network:ovd)` -> Matches all opers on the network ovd.
+- `$and:($account+$pylinkirc)` -> Matches all users logged in to both services and PyLink.
+- `$and:(*!*@localhost+$ircop)` -> Matches all opers with the host `localhost`.
+- `$and:(*!*@*.mibbit.com+!$ircop+!$account)` -> Matches all (non-CGIIRC) Mibbit users that aren't opered or logged in to services.
+
+### The "$server" target (PyLink 0.9+)
+Used to match users on specific IRC servers.
+
+- `$server:server.name` -> Returns True (a match) if the target is connected on the given server. Server names are matched case insensitively.
+- `$server:*server.glob*` -> Returns True (a match) if the target is connected on a server matching the glob.
+- `$server:1XY` -> Returns True if the target's is connected on the server with the given SID. Note: SIDs ARE case sensitive.
+
+### The "$pylinkacc" target (PyLink 0.9+)
+Used to match users logged in to *PyLink* (i.e. via the `identify` command). **As of PyLink 2.0, The "$pylinkirc:" prefix is implied if you specify a PyLink account name without it.**
+
+- `$pylinkacc` -> Returns True if the target is logged in to PyLink.
+- `$pylinkacc:accountname` -> Returns True if the target's PyLink login matches the one given (case insensitive).
+
+### The "$realname" target (PyLink 2.0+)
+Used to match users with certain realnames.
+
+- `$realname:*James*`: matches anyone with "James" in their real name (case insensitive).
+
+### The "$service" target (PyLink 2.0+)
+Used to match service bots. This exttarget takes one optional argument: a glob, which is compared case-insensitively to the target user's service name if present.
+
+- `$service`: matches any PyLink service bot.
+- `$service:automode`: matches the Automode service bot.

docs/faq.md

@@ -1,37 +1,144 @@
 # PyLink FAQ
 
+## Startup errors
+
 ### I get errors like "ImportError: No module named 'yaml'" when I start PyLink
 
-You are missing dependencies - re-read https://github.com/GLolol/PyLink#dependencies
+You are missing dependencies - re-read https://github.com/jlu5/PyLink/blob/master/README.md#installation
 
 ### I get errors like "yaml.scanner.ScannerError: while scanning for the next token, found character '\t' that cannot start any token"
 
-You must use SPACES and not tabs in your configuration! (`\t` is the escaped code for a tab, which is disallowed by YAML)
+You must use **spaces** and not tabs to indent your configuration file! (`\t` is the escaped code for a tab, which is not allowed in YAML)
+
+### I get errors like "ParserError: while parsing a block mapping ... expected <block end>, but found '<block sequence start>'
+This likely indicates an indentation issue. When you create a list in YAML (PyLink's config format), all entries must be indented consistently. For example, this is **bad**:
+
+```yaml
+# This will cause an error!
+someblock:
+    - abcd
+    - def
+  - ghi
+```
+
+This is good:
+
+```yaml
+someblock:
+    - abcd
+    - def
+    - ghi
+```
+
+### I keep getting YAML / syntax errors trying to set up my instance!
+
+Take a few minutes to familiarize yourself with YAML, the markup language we use for the config file.
+[CraftIRC](https://github.com/Animosity/CraftIRC/wiki/Complete-idiot%27s-introduction-to-yaml), [Ansible](https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html), and [Wikipedia](https://en.wikipedia.org/wiki/YAML) all provide excellent guides (with examples) on its basic structure.
+
+A common misconception is that the YAML format is something specific to Python or PyLink, but this is not the case! YAML is a programming language-independent standard which *happens* to use indents for structures like Python does, but [parsers for it exist just about everywhere](http://yaml.org/).
+
+The reason I (James) chose it for this project is essentially a restatement of its common benefits:
+
+- It's compact and human readable (compared to raw JSON or XML)
+- It's powerful, supporting everything from nested config blocks to multi-line strings
+- It fits well in the Python landscape, compared to a flat .ini or C-style config format
+- It's language independent, which means it's not a giant pain if we decide to rewrite the project in some other language one day... 🙃
+
+## Linking / Connection issues
+
+### PyLink won't connect to my network!
+
+As a general guide, you should check the following before asking for help:
+
+- Is the target network's IRCd showing failed connection attempts?
+    - If not:
+        1) Is PyLink connecting to the right port (i.e. one the IRCd is listening on?)
+        2) Is the target network's IRCd actually binding to the port you're trying to use? If there is a port conflict with another program, the IRCd may fail to bind to specific ports but *still start* on others which are free.
+        3) Is the target port firewalled on the target machine?
+        4) Is there a working connection between the source and target servers? Use ping to test this, as routing issues between providers can cause servers to become unreachable.
+            - If your servers purposely block ping, you're going to have to figure this one out yourself... 😬
+
+    - If so:
+        1) Check for recvpass/sendpass/server hostname/IP mismatches - usually the IRCd will tell you if you're running into one of these, provided you have the right server notices enabled (consult your IRCd documentation for how to enable these).
+        2) Make sure you're not connecting with SSL on a non-SSL port, or vice versa.
+
+If these steps haven't helped you so far, maybe you've found a bug...?
+
+### My networks keep disconnecting with SSL errors!
+
+See https://github.com/jlu5/PyLink/issues/463 - the problem appears to be caused somewhere in Python's SSL stack and/or OpenSSL, and not directly by our code.
+
+Unfortunately, the only workarounds so far are to either disable SSL/TLS, or wrap a plain IRC connection in an external service (stunnel, OpenVPN, etc.)
 
 ### I turned autoconnect for PyLink on, and now I'm getting errors!
 
-PyLink does not support inbound connections - much like regular services such as Atheme or Anope, it only connects outwards *to* IRCds. (If you don't understand what this means, it means you should turn autoconnect OFF for PyLink)
+PyLink does not support inbound connections - much like regular services such as Atheme or Anope, it only connects outwards *to* IRCds. (If you don't understand what this means, it means you should turn autoconnect **off** for PyLink)
 
-### Does PyLink support Clientbot relay like Janus?
+## Relay issues
-
-Yes. However, Clientbot support is in alpha stages as of PyLink 0.10 and is far from complete: [Clientbot TODO](https://github.com/GLolol/PyLink/issues?q=is%3Aissue+is%3Aopen+label%3Aprotocols%2Fclientbot).
-
-### Clientbot doesn't relay both ways!
-
-Load the `relay_clientbot` plugin. https://github.com/GLolol/PyLink/blob/e1fab8c/example-conf.yml#L303-L306
 
 ### Does everyone need to install PyLink Relay for it to work?
 
-**No!** Only the PyLink administrator needs to host a PyLink instance, as each can connect to multiple networks. Everyone else only needs to add a link block on their IRCd.
+**No!** Only the PyLink administrator needs to host a PyLink instance with the `relay` plugin loaded, as each instance can connect to multiple networks. Everyone else only needs to add a link block on their IRCd.
 
-InterJanus-style links between PyLink daemons are not supported yet; see https://github.com/GLolol/PyLink/issues/99 for any progress regarding that.
+InterJanus-style links between PyLink daemons are not supported yet; see https://github.com/jlu5/PyLink/issues/99 for any progress regarding that.
 
-### What are PyLink Relay's benefits over Janus?
+### What are PyLink's advantages over Janus?
 
-In no particular order:
+PyLink provides, in no particular order:
 - More complete support for modern IRCds (UnrealIRCd 4.x, InspIRCd 2.0, charybdis 4, Nefarious IRCu, etc.).
-- PyLink is built upon a flexible, maintainable codebase.
+- A flexible, maintainable codebase extensible beyond Relay.
-- Cross platform (*nix, Windows, and probably others too).
+- Proper protocol negotiation leading to fewer SQUIT/DoS possibilities:
-- Proper protocol negotiation leading to fewer DoS possibilities:
     - Better support for channel modes such as +fjMOR, etc.
-    - Configurable nick length limits for relayed users.
+    - Proper support for nick length limits with relayed users.
+
+### My IRCd SQUITs the Relay server with errors like "Bad nickname introduced"!
+
+First, check whether the SQUIT message includes the nick that triggered the netsplit. If this nick includes any characters not allowed in regular IRC, such as the slash ("/"), or is otherwise an invalid nick (e.g. beginning with a hyphen or number), this likely indicates a bug in PyLink Relay. These problems should be reported on the issue tracker.
+
+However, if the nick mentioned is legal on IRC, this issue is likely caused by a max nick length misconfiguration: i.e. the Relay server is introducing nicks too long for the target network. This can be fixed by setting the `maxnicklen` option in the affected network's PyLink `server:` block to the same value as that network's `005` `NICKLEN` (that is, the `NICKLEN=<num>` value in `/raw version`).
+
+### Clientbot doesn't relay both ways!
+
+Load the `relay_clientbot` plugin. https://github.com/jlu5/PyLink/blob/1.3-beta1/example-conf.yml#L465-L468
+
+### How do I turn off colors in Clientbot?
+See https://github.com/jlu5/PyLink/blob/master/docs/advanced-relay-config.md#custom-clientbot-styles, especially the section "Disabling Colors/Control Codes".
+
+### Relay is occasionally dropping users from channels!
+
+This usually indicates a serious bug in either Relay or PyLink's protocol modules, and should be reported as an issue. When asking for help, please state which IRCds your PyLink instance is linking to: specifically, which IRCd the missing users are *from* and which IRCd the users are missing *on*. Also, be prepared to send debug logs as you reproduce the issue!
+- Another tip in debugging this is to run `showchan` on the affected channels. If PyLink shows users in `showchan` that aren't in the actual user list, this is most likely a protocol module issue. If `showchan`'s output is correct, it is instead probably a Relay issue where users aren't spawning correctly.
+
+### Does Relay support mode +R, +M, etc.? How does Relay handle modes supported on one IRCd but not on another?
+Essentially, PyLink maps IRCd modes together by name, so modes that use different characters on different IRCds can be recognized as the same "mode". Tables of supported channel modes, user modes, and extbans (in 2.0+) can be found at https://github.com/jlu5/PyLink/tree/devel/docs/modelists. Note that third party/contrib modules implementing modes are generally *not* tested / supported.
+
+Relay in particular uses whitelists to determine which modes are safe to relay: for 2.0.0, this is https://github.com/jlu5/PyLink/blob/71a24b8/plugins/relay.py#L903-L969. **Most *channel* modes recognized by PyLink are whitelisted and usable with Relay**, with the following exceptions:
+
+- "registered" channel / user modes (InspIRCd, UnrealIRCd **+r**) - this is to prevent conflicts with local networks's services.
+- "permanent" channel modes (commonly **+P**) - it's not necessary for remote networks' channels to also be permanent locally.
+- Flood protection modes are only relayed between networks running the same IRCd (UnrealIRCd <-> UnrealIRCd or InspIRCd <-> InspIRCd).
+- Modes and extbans that specify a forwarding channel - mangling channel names between networks is far too complicated and desync prone.
+- InspIRCd's m_ojoin **+Y** and m_operprefix **+y** are ignored by Relay.
+- auditorium (InspIRCd **+u**), delayjoin (UnrealIRCd, P10, InspIRCd **+D**), and any other modes affecting join visibilites are not supported.
+
+Support for user modes is not as complete:
+- Filter type modes such as callerid (**+g**), regonly (**+R**), noctcp (UnrealIRCd **+T**) are *not yet* supported by Relay.
+- Service protection modes (UnrealIRCD **+S**, InspIRCd **+k**, etc.) are not forwarded by Relay to prevent abuse.
+
+### How does Relay handle kills?
+See https://github.com/jlu5/PyLink/blob/devel/docs/relay-quickstart.md#kill-handling
+
+### How does Relay handle KLINE/GLINE/ZLINE?
+
+It doesn't. https://github.com/jlu5/PyLink/issues/521#issuecomment-352316396 explains my reasons for skipping over this:
+
+* The weakest link, whether this be a malicious/compromised/mistaken oper or a misconfigured services instance, can easily wreak havoc by banning something they shouldn't.
+* KLINE relaying goes against the concept of partial network links and creates serious animosity when opers disagree on policy. If KLINEs are shared, opers are essentially shared as well, and this is not the goal of Relay.
+
+## Services issues
+
+### Service bots aren't spawning on my network, even though PyLink connects
+
+This indicates either a bug in PyLink's protocol module or (less commonly) a bug in your IRCd. Hint: ENDBURST is likely not being sent or received properly, which causes service bot spawning to never trigger.
+
+Make sure you're using an [officially supported IRCd](https://github.com/jlu5/PyLink#supported-ircds) before requesting help, as custom IRCd code can potentially trigger S2S bugs and is not something we can support.

docs/modelists/README.md

@@ -0,0 +1,5 @@
+This folder contains tables of named modes defined by PyLink modules. The following are HTML versions of the raw .csv data:
+
+- [Supported named channel modes](https://raw.githack.com/jlu5/PyLink/devel/docs/modelists/channel-modes.html)
+- [Supported named user modes](https://raw.githack.com/jlu5/PyLink/devel/docs/modelists/user-modes.html)
+- [Supported extbans](https://raw.githack.com/jlu5/PyLink/devel/docs/modelists/extbans.html)

docs/modelists/channel-modes.csv

@@ -0,0 +1,71 @@
+Channel Mode / IRCd,rfc1459,hybrid,inspircd/insp20,inspircd/insp3,ngircd,p10/ircu,p10/nefarious,p10/snircd,ts6/charybdis,ts6/chatircd,ts6/elemental,ts6/ratbox,unreal
+admin,,,"a (m_customprefix, m_chanprotect)",a (customprefix),a,,,,,a (when enabled),a (when enabled),,a
+adminonly,,,,,,,a,,A (ext/chm_adminonly),A (ext/chm_adminonly),A (ext/chm_adminonly.so),,
+allowinvite,,,A (m_allowinvite),A (allowinvite),,,,,g,g,g,,
+auditorium,,,u (m_auditorium),u (auditorium),,,,,,,,,
+autoop,,,w (m_autoop),w (autoop),,,,,,,,,
+ban,b,b,b,b,b,b,b,b,b,b,b,b,b
+banexception,,e,e (m_banexception),e (banexception),e,,e,,e,e,e,e,e
+blockcaps,,,B (m_blockcaps),"B (anticaps, blockcaps)",,,,,,,G (ext/chm_nocaps.so),,
+blockcolor,,c,c (m_blockcolor),c (blockcolor),,c,c,c,,,,,c (chanmodes/nocolor)
+blockhighlight,,,V (contrib/m_blockhighlight),V (contrib/blockhighlight),,,,,,,,,
+censor,,,G (m_censor),G (censor),,,,,,,,,G (chanmodes/censor)
+delayjoin,,,D (m_delayjoin),D (delayjoin),,D,D,D,,,,,D (chanmodes/delayjoin)
+delaymsg,,,d (m_delaymsg),d (delaymsg),,,,,,,,,
+exemptchanops,,,X (m_exemptchanops),X (exemptchanops),,,,,,,,,
+filter,,,g (m_filter),g (filter),,,,,,,,,(via extban ~T:block:)
+flood,,,f (m_messageflood),f (messageflood),,,,,,,,,
+flood_unreal,,,,,,,,,,,,,f (chanmodes/floodprot)
+freetarget,,,,,,,,,F,F,F,,
+had_delayjoin,,,,,,d,d,d,,,,,
+halfop,,h,"h (m_customprefix, m_halfop)",h (customprefix),h,,,,,h (when enabled),h (when enabled),,h
+hiddenbans,,,,,,,,,,,u,,
+hidequits,,,,,,,Q,u,,,,,
+history,,,H (m_chanhistory),H (chanhistory),,,,,,,,,
+invex,,I,I (m_inviteexception),I (inviteexception),I,,,,I,I,I,I,I
+inviteonly,i,i,i,i,i,i,i,i,i,i,i,i,i
+issecure,,,,,,,,,,,,,Z (chanmodes/issecure)
+joinflood,,,j (m_joinflood),j (joinflood),,,,,j,j,j,,
+key,k,k,k,k,k,k,k,k,k,k,k,k,k
+kicknorejoin,,,,,,,,,,,J,,
+kicknorejoin_insp,,,J (m_kicknorejoin),J (kicknorejoin),,,,,,,,,
+largebanlist,,,,,,,,,L,L,L,,
+limit,l,l,l,l,l,l,l,l,l,l,l,l,l
+moderated,m,m,m,m,m,m,m,m,m,m,m,m,m
+netadminonly,,,,,,,,,,N (ext/chm_netadminonly),,,
+nickflood,,,F (m_nickflood),F (nickflood),,,,,,,,,
+noamsg,,,,,,,T,T,,,,,
+noctcp,,C,C (m_noctcp),C (noctcp),,C,C,C,C,C,C,,C (chanmodes/noctcp)
+noextmsg,n,n,n,n,n,n,n,n,n,n,n,n,n
+noforwards,,,,,,,,,Q,Q,Q,,
+noinvite,,,,,V,,,,,,,,V (chanmodes/noinvite)
+nokick,,,Q (m_nokicks),Q (nokicks),Q,,,,,,E,,Q (chanmodes/nokick)
+noknock,,p*,K (m_knock),K (knock),,,,,p*,p*,p*,p*,K (chanmodes/noknock)
+nonick,,,N (m_nonicks),N (nonicks),N,,,,,,d,,N (chanmodes/nonickchange)
+nonotice,,,T (m_nonotice),T (nonotice),,,N,N,T (ext/chm_nonotice),T (ext/chm_nonotice),T,,T (chanmodes/nonotice)
+official-join,,,Y (m_ojoin),Y (ojoin),,,,,,,,,
+op,o,o,o,o,o,o,o,o,o,o,o,o,o
+operonly,,O,O (m_operchans),O (operchans),O,,O,,O (ext/chm_operonly),O (ext/chm_operonly),O (ext/chm_operonly.so),,O (chanmodes/operonly)
+oplevel_apass,,,,,,A,A,A,,,,,
+oplevel_upass,,,,,,U,U,U,,,,,
+opmoderated,,,U (contrib/m_opmoderated),,,,,,z,z,z,,
+owner,,,"q (m_customprefix, m_chanprotect)",q (customprefix),q,,,,,y (when enabled),y (when enabled),,q
+paranoia,,p*,,,,,,,,,,,
+permanent,,,P (m_permchannels),P (permchannels),P,,z,,P,P,P,,P (chanmodes/permanent)
+private,p,p*,p,p,p,p,p,p,p*,p*,p*,p*,p
+quiet,,,(via extban m:),(via extban m:),,,(via extban ~q:),,q,q,q,,(via extban ~q:)
+redirect,,,L (m_redirect),L (redirect),,,L,,f,f,f,,L (chanmodes/link)
+registered,,r,r (m_services_account),r (services_account),r,R,R,R,,,,,r
+regmoderated,,M,M (m_services_account),M (services_account),M,,M,M,,,,,M (chanmodes/regonlyspeak)
+regonly,,R,R (m_services_account),R (services_account),R,r,r,r,r,r,r,r,R (chanmodes/regonly)
+repeat,,,,,,,,,,,K (ext/chm_norepeat.c),,
+repeat_insp,,,,E (repeat),,,,,,,,,
+secret,s,s,s,s,s,s,s,s,s,s,s,s,s
+sslonly,,S,z (m_sslmodes),z (sslmodes),z,,,,S (ext/chm_sslonly),S (ext/chm_sslonly),S (ext/chm_sslonly.c),S,z (chanmodes/secureonly)
+stripcolor,,,S (m_stripcolor),S (stripcolor),,,S,,c,c,c,,S (chanmodes/stripcolor)
+topiclock,t,t,t,t,t,t,t,t,t,t,t,t,t
+voice,v,v,v,v,v,v,v,v,v,v,v,v,v
+,,,,,,,,,,,,,
+----,,,,,,,,,,,,,
+<b>Note</b>: Channel modes for InspIRCd and UnrealIRCd are automatically negotiated on connect; this may not be a complete list.,,,,,,,,,,,,,
+"* Mode +p corresponds to both “noknock” and “private” on TS6 IRCds, as well as “paranoia” on hybrid.",,,,,,,,,,,,,

docs/modelists/channel-modes.html

@@ -0,0 +1,281 @@
+
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="UTF-8">
+<meta name=viewport content="width=device-width, initial-scale=1">
+
+<head>
+<title>Supported Channel Modes for PyLink</title>
+<style>
+
+html {
+    background-color: white;
+}
+
+.note {
+    color: #555555;
+}
+
+/* (╮°-°)╮┳━┳ */
+table, th, td {
+    border: 1px solid black;
+}
+
+td, th {
+    text-align: center;
+    padding: 3px;
+}
+
+td:first-child, th[scope="row"] {
+    text-align: left;
+}
+
+/* Table cells */
+.tablecell-yes {
+    background-color: #A7F2A5
+}
+
+.tablecell-no {
+    background-color: #F08496
+}
+
+.tablecell-na {
+    background-color: #F0F0F0
+}
+
+.tablecell-planned, .tablecell-yes2 {
+    background-color: #B1FCDE
+}
+
+.tablecell-partial {
+    background-color: #EDE8A4
+}
+
+.tablecell-special {
+    background-color: #DCB1FC
+}
+
+</style>
+
+</head>
+
+<body>
+<table><tr>
+<th scope="col">Channel Mode / IRCd</th>
+<th scope="col">rfc1459</th>
+<th scope="col">hybrid</th>
+<th scope="col">inspircd/insp20</th>
+<th scope="col">inspircd/insp3</th>
+<th scope="col">ngircd</th>
+<th scope="col">p10/ircu</th>
+<th scope="col">p10/nefarious</th>
+<th scope="col">p10/snircd</th>
+<th scope="col">ts6/charybdis</th>
+<th scope="col">ts6/chatircd</th>
+<th scope="col">ts6/elemental</th>
+<th scope="col">ts6/ratbox</th>
+<th scope="col">unreal</th>
+</tr>
+<tr>
+<th scope="row">admin</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+a<br><span class="note">(m_customprefix, m_chanprotect)</span></td><td class="tablecell-yes2">+a<br><span class="note">(customprefix)</span></td><td class="tablecell-yes">+a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+a<br><span class="note">(when enabled)</span></td><td class="tablecell-yes2">+a<br><span class="note">(when enabled)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+a</td></tr>
+<tr>
+<th scope="row">adminonly</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+A<br><span class="note">(ext/chm_adminonly)</span></td><td class="tablecell-yes2">+A<br><span class="note">(ext/chm_adminonly)</span></td><td class="tablecell-yes2">+A<br><span class="note">(ext/chm_adminonly.so)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">allowinvite</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+A<br><span class="note">(m_allowinvite)</span></td><td class="tablecell-yes2">+A<br><span class="note">(allowinvite)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+g</td><td class="tablecell-yes">+g</td><td class="tablecell-yes">+g</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">auditorium</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+u<br><span class="note">(m_auditorium)</span></td><td class="tablecell-yes2">+u<br><span class="note">(auditorium)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">autoop</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+w<br><span class="note">(m_autoop)</span></td><td class="tablecell-yes2">+w<br><span class="note">(autoop)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban</th>
+<td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td><td class="tablecell-yes">+b</td></tr>
+<tr>
+<th scope="row">banexception</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+e</td><td class="tablecell-yes2">+e<br><span class="note">(m_banexception)</span></td><td class="tablecell-yes2">+e<br><span class="note">(banexception)</span></td><td class="tablecell-yes">+e</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+e</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+e</td><td class="tablecell-yes">+e</td><td class="tablecell-yes">+e</td><td class="tablecell-yes">+e</td><td class="tablecell-yes">+e</td></tr>
+<tr>
+<th scope="row">blockcaps</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+B<br><span class="note">(m_blockcaps)</span></td><td class="tablecell-yes2">+B<br><span class="note">(anticaps, blockcaps)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+G<br><span class="note">(ext/chm_nocaps.so)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">blockcolor</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+c</td><td class="tablecell-yes2">+c<br><span class="note">(m_blockcolor)</span></td><td class="tablecell-yes2">+c<br><span class="note">(blockcolor)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+c</td><td class="tablecell-yes">+c</td><td class="tablecell-yes">+c</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+c<br><span class="note">(chanmodes/nocolor)</span></td></tr>
+<tr>
+<th scope="row">blockhighlight</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+V<br><span class="note">(contrib/m_blockhighlight)</span></td><td class="tablecell-yes2">+V<br><span class="note">(contrib/blockhighlight)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">censor</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+G<br><span class="note">(m_censor)</span></td><td class="tablecell-yes2">+G<br><span class="note">(censor)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+G<br><span class="note">(chanmodes/censor)</span></td></tr>
+<tr>
+<th scope="row">delayjoin</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+D<br><span class="note">(m_delayjoin)</span></td><td class="tablecell-yes2">+D<br><span class="note">(delayjoin)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+D</td><td class="tablecell-yes">+D</td><td class="tablecell-yes">+D</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+D<br><span class="note">(chanmodes/delayjoin)</span></td></tr>
+<tr>
+<th scope="row">delaymsg</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+d<br><span class="note">(m_delaymsg)</span></td><td class="tablecell-yes2">+d<br><span class="note">(delaymsg)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">exemptchanops</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+X<br><span class="note">(m_exemptchanops)</span></td><td class="tablecell-yes2">+X<br><span class="note">(exemptchanops)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">filter</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+g<br><span class="note">(m_filter)</span></td><td class="tablecell-yes2">+g<br><span class="note">(filter)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-partial">(via extban ~T:block:)</td></tr>
+<tr>
+<th scope="row">flood</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+f<br><span class="note">(m_messageflood)</span></td><td class="tablecell-yes2">+f<br><span class="note">(messageflood)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">flood_unreal</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+f<br><span class="note">(chanmodes/floodprot)</span></td></tr>
+<tr>
+<th scope="row">freetarget</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+F</td><td class="tablecell-yes">+F</td><td class="tablecell-yes">+F</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">had_delayjoin</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+d</td><td class="tablecell-yes">+d</td><td class="tablecell-yes">+d</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">halfop</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+h</td><td class="tablecell-yes2">+h<br><span class="note">(m_customprefix, m_halfop)</span></td><td class="tablecell-yes2">+h<br><span class="note">(customprefix)</span></td><td class="tablecell-yes">+h</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+h<br><span class="note">(when enabled)</span></td><td class="tablecell-yes2">+h<br><span class="note">(when enabled)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+h</td></tr>
+<tr>
+<th scope="row">hiddenbans</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+u</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">hidequits</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+Q</td><td class="tablecell-yes">+u</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">history</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+H<br><span class="note">(m_chanhistory)</span></td><td class="tablecell-yes2">+H<br><span class="note">(chanhistory)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">invex</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+I</td><td class="tablecell-yes2">+I<br><span class="note">(m_inviteexception)</span></td><td class="tablecell-yes2">+I<br><span class="note">(inviteexception)</span></td><td class="tablecell-yes">+I</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+I</td><td class="tablecell-yes">+I</td><td class="tablecell-yes">+I</td><td class="tablecell-yes">+I</td><td class="tablecell-yes">+I</td></tr>
+<tr>
+<th scope="row">inviteonly</th>
+<td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td></tr>
+<tr>
+<th scope="row">issecure</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+Z<br><span class="note">(chanmodes/issecure)</span></td></tr>
+<tr>
+<th scope="row">joinflood</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+j<br><span class="note">(m_joinflood)</span></td><td class="tablecell-yes2">+j<br><span class="note">(joinflood)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+j</td><td class="tablecell-yes">+j</td><td class="tablecell-yes">+j</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">key</th>
+<td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td></tr>
+<tr>
+<th scope="row">kicknorejoin</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+J</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">kicknorejoin_insp</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+J<br><span class="note">(m_kicknorejoin)</span></td><td class="tablecell-yes2">+J<br><span class="note">(kicknorejoin)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">largebanlist</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+L</td><td class="tablecell-yes">+L</td><td class="tablecell-yes">+L</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">limit</th>
+<td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td></tr>
+<tr>
+<th scope="row">moderated</th>
+<td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td><td class="tablecell-yes">+m</td></tr>
+<tr>
+<th scope="row">netadminonly</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+N<br><span class="note">(ext/chm_netadminonly)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">nickflood</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+F<br><span class="note">(m_nickflood)</span></td><td class="tablecell-yes2">+F<br><span class="note">(nickflood)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">noamsg</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+T</td><td class="tablecell-yes">+T</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">noctcp</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+C</td><td class="tablecell-yes2">+C<br><span class="note">(m_noctcp)</span></td><td class="tablecell-yes2">+C<br><span class="note">(noctcp)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+C</td><td class="tablecell-yes">+C</td><td class="tablecell-yes">+C</td><td class="tablecell-yes">+C</td><td class="tablecell-yes">+C</td><td class="tablecell-yes">+C</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+C<br><span class="note">(chanmodes/noctcp)</span></td></tr>
+<tr>
+<th scope="row">noextmsg</th>
+<td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td></tr>
+<tr>
+<th scope="row">noforwards</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+Q</td><td class="tablecell-yes">+Q</td><td class="tablecell-yes">+Q</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">noinvite</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+V</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+V<br><span class="note">(chanmodes/noinvite)</span></td></tr>
+<tr>
+<th scope="row">nokick</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+Q<br><span class="note">(m_nokicks)</span></td><td class="tablecell-yes2">+Q<br><span class="note">(nokicks)</span></td><td class="tablecell-yes">+Q</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+E</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+Q<br><span class="note">(chanmodes/nokick)</span></td></tr>
+<tr>
+<th scope="row">noknock</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-special">+p*</td><td class="tablecell-yes2">+K<br><span class="note">(m_knock)</span></td><td class="tablecell-yes2">+K<br><span class="note">(knock)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-special">+p*</td><td class="tablecell-special">+p*</td><td class="tablecell-special">+p*</td><td class="tablecell-special">+p*</td><td class="tablecell-yes2">+K<br><span class="note">(chanmodes/noknock)</span></td></tr>
+<tr>
+<th scope="row">nonick</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+N<br><span class="note">(m_nonicks)</span></td><td class="tablecell-yes2">+N<br><span class="note">(nonicks)</span></td><td class="tablecell-yes">+N</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+d</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+N<br><span class="note">(chanmodes/nonickchange)</span></td></tr>
+<tr>
+<th scope="row">nonotice</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+T<br><span class="note">(m_nonotice)</span></td><td class="tablecell-yes2">+T<br><span class="note">(nonotice)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+N</td><td class="tablecell-yes">+N</td><td class="tablecell-yes2">+T<br><span class="note">(ext/chm_nonotice)</span></td><td class="tablecell-yes2">+T<br><span class="note">(ext/chm_nonotice)</span></td><td class="tablecell-yes">+T</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+T<br><span class="note">(chanmodes/nonotice)</span></td></tr>
+<tr>
+<th scope="row">official-join</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+Y<br><span class="note">(m_ojoin)</span></td><td class="tablecell-yes2">+Y<br><span class="note">(ojoin)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">op</th>
+<td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td></tr>
+<tr>
+<th scope="row">operonly</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+O</td><td class="tablecell-yes2">+O<br><span class="note">(m_operchans)</span></td><td class="tablecell-yes2">+O<br><span class="note">(operchans)</span></td><td class="tablecell-yes">+O</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+O</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+O<br><span class="note">(ext/chm_operonly)</span></td><td class="tablecell-yes2">+O<br><span class="note">(ext/chm_operonly)</span></td><td class="tablecell-yes2">+O<br><span class="note">(ext/chm_operonly.so)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+O<br><span class="note">(chanmodes/operonly)</span></td></tr>
+<tr>
+<th scope="row">oplevel_apass</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+A</td><td class="tablecell-yes">+A</td><td class="tablecell-yes">+A</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">oplevel_upass</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+U</td><td class="tablecell-yes">+U</td><td class="tablecell-yes">+U</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">opmoderated</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+U<br><span class="note">(contrib/m_opmoderated)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+z</td><td class="tablecell-yes">+z</td><td class="tablecell-yes">+z</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">owner</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+q<br><span class="note">(m_customprefix, m_chanprotect)</span></td><td class="tablecell-yes2">+q<br><span class="note">(customprefix)</span></td><td class="tablecell-yes">+q</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+y<br><span class="note">(when enabled)</span></td><td class="tablecell-yes2">+y<br><span class="note">(when enabled)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+q</td></tr>
+<tr>
+<th scope="row">paranoia</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-special">+p*</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">permanent</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+P<br><span class="note">(m_permchannels)</span></td><td class="tablecell-yes2">+P<br><span class="note">(permchannels)</span></td><td class="tablecell-yes">+P</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+z</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+P</td><td class="tablecell-yes">+P</td><td class="tablecell-yes">+P</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+P<br><span class="note">(chanmodes/permanent)</span></td></tr>
+<tr>
+<th scope="row">private</th>
+<td class="tablecell-yes">+p</td><td class="tablecell-special">+p*</td><td class="tablecell-yes">+p</td><td class="tablecell-yes">+p</td><td class="tablecell-yes">+p</td><td class="tablecell-yes">+p</td><td class="tablecell-yes">+p</td><td class="tablecell-yes">+p</td><td class="tablecell-special">+p*</td><td class="tablecell-special">+p*</td><td class="tablecell-special">+p*</td><td class="tablecell-special">+p*</td><td class="tablecell-yes">+p</td></tr>
+<tr>
+<th scope="row">quiet</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-partial">(via extban m:)</td><td class="tablecell-partial">(via extban m:)</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-partial">(via extban ~q:)</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+q</td><td class="tablecell-yes">+q</td><td class="tablecell-yes">+q</td><td class="tablecell-na note">n/a</td><td class="tablecell-partial">(via extban ~q:)</td></tr>
+<tr>
+<th scope="row">redirect</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+L<br><span class="note">(m_redirect)</span></td><td class="tablecell-yes2">+L<br><span class="note">(redirect)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+L</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+f</td><td class="tablecell-yes">+f</td><td class="tablecell-yes">+f</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+L<br><span class="note">(chanmodes/link)</span></td></tr>
+<tr>
+<th scope="row">registered</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+r</td><td class="tablecell-yes2">+r<br><span class="note">(m_services_account)</span></td><td class="tablecell-yes2">+r<br><span class="note">(services_account)</span></td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+R</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+r</td></tr>
+<tr>
+<th scope="row">regmoderated</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+M</td><td class="tablecell-yes2">+M<br><span class="note">(m_services_account)</span></td><td class="tablecell-yes2">+M<br><span class="note">(services_account)</span></td><td class="tablecell-yes">+M</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+M</td><td class="tablecell-yes">+M</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+M<br><span class="note">(chanmodes/regonlyspeak)</span></td></tr>
+<tr>
+<th scope="row">regonly</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+R</td><td class="tablecell-yes2">+R<br><span class="note">(m_services_account)</span></td><td class="tablecell-yes2">+R<br><span class="note">(services_account)</span></td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-yes2">+R<br><span class="note">(chanmodes/regonly)</span></td></tr>
+<tr>
+<th scope="row">repeat</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+K<br><span class="note">(ext/chm_norepeat.c)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">repeat_insp</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+E<br><span class="note">(repeat)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">secret</th>
+<td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td></tr>
+<tr>
+<th scope="row">sslonly</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+S</td><td class="tablecell-yes2">+z<br><span class="note">(m_sslmodes)</span></td><td class="tablecell-yes2">+z<br><span class="note">(sslmodes)</span></td><td class="tablecell-yes">+z</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+S<br><span class="note">(ext/chm_sslonly)</span></td><td class="tablecell-yes2">+S<br><span class="note">(ext/chm_sslonly)</span></td><td class="tablecell-yes2">+S<br><span class="note">(ext/chm_sslonly.c)</span></td><td class="tablecell-yes">+S</td><td class="tablecell-yes2">+z<br><span class="note">(chanmodes/secureonly)</span></td></tr>
+<tr>
+<th scope="row">stripcolor</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+S<br><span class="note">(m_stripcolor)</span></td><td class="tablecell-yes2">+S<br><span class="note">(stripcolor)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+S</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+c</td><td class="tablecell-yes">+c</td><td class="tablecell-yes">+c</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+S<br><span class="note">(chanmodes/stripcolor)</span></td></tr>
+<tr>
+<th scope="row">topiclock</th>
+<td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td><td class="tablecell-yes">+t</td></tr>
+<tr>
+<th scope="row">voice</th>
+<td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td><td class="tablecell-yes">+v</td></tr>
+<p><b>Note</b>: Channel modes for InspIRCd and UnrealIRCd are automatically negotiated on connect; this may not be a complete list.</p><p>* Mode +p corresponds to both “noknock” and “private” on TS6 IRCds, as well as “paranoia” on hybrid.</p>
+
+</table>
+</body>
+</html>

docs/modelists/extbans.csv

@@ -0,0 +1,40 @@
+Extban / IRCd,inspircd,p10/nefarious,ts6/charybdis,unreal
+ban_account,R:,~a:,$a:,~a:
+ban_account_legacy,,,,~R:
+ban_all_opers,,,$o,
+ban_all_registered,,,$a,
+ban_all_ssl,,,$z,
+ban_banshare,,~j:,$j:,
+ban_blockcaps,B:,,,
+ban_blockcolor,c:,,,
+ban_certfp,z:,,,~S:
+ban_extgecos,,,$x:,
+ban_inchannel,j:,~c:,$c:,~c:
+ban_invites,A:,,,
+ban_mark,,~m:,,
+ban_noctcp,C:,,,
+ban_nojoins,,,,~j:
+ban_nokicks,Q:,,,
+ban_nonick,N:,~n:,,~n:
+ban_nonotice,T:,,,~m:notice: (+e only)
+ban_not_account,,,$~a:,
+ban_not_banshare,,,$~j:,
+ban_not_extgecos,,,$~x:,
+ban_not_inchannel,,,$~c:,
+ban_not_opers,,,$~o,
+ban_not_realname,,,$~r:,
+ban_not_server,,,$~s:,
+ban_not_ssl,,,$~z,
+ban_opertype,O:,,,~O:
+ban_partmsgs,p:,,,
+ban_realname,r:,~r:,$r:,~r:
+ban_server,s:,,$s:,
+ban_stripcolor,S:,,,~m:color: (+e only)
+ban_unregistered,,,$~a,
+ban_unregistered_mark,,~M:,,
+ban_unregistered_matching,U:,,,
+msgbypass_external,,,,~m:external:
+msgbypass_censor,,,,~m:censor:
+msgbypass_moderated,,,,~m:moderated:
+quiet,m:,~q:,(via cmode +q),~q:
+timedban_unreal,,,,~t:

docs/modelists/extbans.html

@@ -0,0 +1,191 @@
+
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="UTF-8">
+<meta name=viewport content="width=device-width, initial-scale=1">
+
+<head>
+<title>Supported Extbans for PyLink</title>
+<style>
+
+html {
+    background-color: white;
+}
+
+.note {
+    color: #555555;
+}
+
+/* (╮°-°)╮┳━┳ */
+table, th, td {
+    border: 1px solid black;
+}
+
+td, th {
+    text-align: center;
+    padding: 3px;
+}
+
+td:first-child, th[scope="row"] {
+    text-align: left;
+}
+
+/* Table cells */
+.tablecell-yes {
+    background-color: #A7F2A5
+}
+
+.tablecell-no {
+    background-color: #F08496
+}
+
+.tablecell-na {
+    background-color: #F0F0F0
+}
+
+.tablecell-planned, .tablecell-yes2 {
+    background-color: #B1FCDE
+}
+
+.tablecell-partial {
+    background-color: #EDE8A4
+}
+
+.tablecell-special {
+    background-color: #DCB1FC
+}
+
+</style>
+
+</head>
+
+<body>
+<table><tr>
+<th scope="col">Extban / IRCd</th>
+<th scope="col">inspircd</th>
+<th scope="col">p10/nefarious</th>
+<th scope="col">ts6/charybdis</th>
+<th scope="col">unreal</th>
+</tr>
+<tr>
+<th scope="row">ban_account</th>
+<td class="tablecell-yes">R:</td><td class="tablecell-yes">~a:</td><td class="tablecell-yes">$a:</td><td class="tablecell-yes">~a:</td></tr>
+<tr>
+<th scope="row">ban_account_legacy</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~R:</td></tr>
+<tr>
+<th scope="row">ban_all_opers</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$o</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_all_registered</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_all_ssl</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$z</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_banshare</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">~j:</td><td class="tablecell-yes">$j:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_blockcaps</th>
+<td class="tablecell-yes">B:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_blockcolor</th>
+<td class="tablecell-yes">c:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_certfp</th>
+<td class="tablecell-yes">z:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~S:</td></tr>
+<tr>
+<th scope="row">ban_extgecos</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$x:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_inchannel</th>
+<td class="tablecell-yes">j:</td><td class="tablecell-yes">~c:</td><td class="tablecell-yes">$c:</td><td class="tablecell-yes">~c:</td></tr>
+<tr>
+<th scope="row">ban_invites</th>
+<td class="tablecell-yes">A:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_mark</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">~m:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_noctcp</th>
+<td class="tablecell-yes">C:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_nojoins</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~j:</td></tr>
+<tr>
+<th scope="row">ban_nokicks</th>
+<td class="tablecell-yes">Q:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_nonick</th>
+<td class="tablecell-yes">N:</td><td class="tablecell-yes">~n:</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~n:</td></tr>
+<tr>
+<th scope="row">ban_nonotice</th>
+<td class="tablecell-yes">T:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">~m:notice:<br><span class="note">(+e only)</span></td></tr>
+<tr>
+<th scope="row">ban_not_account</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~a:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_not_banshare</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~j:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_not_extgecos</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~x:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_not_inchannel</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~c:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_not_opers</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~o</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_not_realname</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~r:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_not_server</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~s:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_not_ssl</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~z</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_opertype</th>
+<td class="tablecell-yes">O:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~O:</td></tr>
+<tr>
+<th scope="row">ban_partmsgs</th>
+<td class="tablecell-yes">p:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_realname</th>
+<td class="tablecell-yes">r:</td><td class="tablecell-yes">~r:</td><td class="tablecell-yes">$r:</td><td class="tablecell-yes">~r:</td></tr>
+<tr>
+<th scope="row">ban_server</th>
+<td class="tablecell-yes">s:</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$s:</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_stripcolor</th>
+<td class="tablecell-yes">S:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">~m:color:<br><span class="note">(+e only)</span></td></tr>
+<tr>
+<th scope="row">ban_unregistered</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">$~a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_unregistered_mark</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">~M:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">ban_unregistered_matching</th>
+<td class="tablecell-yes">U:</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">msgbypass_external</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~m:external:</td></tr>
+<tr>
+<th scope="row">msgbypass_censor</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~m:censor:</td></tr>
+<tr>
+<th scope="row">msgbypass_moderated</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~m:moderated:</td></tr>
+<tr>
+<th scope="row">quiet</th>
+<td class="tablecell-yes">m:</td><td class="tablecell-yes">~q:</td><td class="tablecell-partial">(via cmode +q)</td><td class="tablecell-yes">~q:</td></tr>
+<tr>
+<th scope="row">timedban_unreal</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">~t:</td></tr>
+
+
+</table>
+</body>
+</html>

docs/modelists/modelists-genhtml.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Generates HTML versions of the mode list .csv definitions.
+"""
+
+import csv
+import os
+import os.path
+
+os.chdir(os.path.dirname(__file__))
+
+FILES = {
+    'user-modes.csv': 'Supported User Modes for PyLink',
+    'channel-modes.csv': 'Supported Channel Modes for PyLink',
+    'extbans.csv': 'Supported Extbans for PyLink',
+}
+
+def _write(outf, text):
+    print(text, end='')
+    outf.write(text)
+
+def _format(articlename, text):
+    # More formatting
+    if text:
+        if text.startswith('('):
+            text = '<td class="tablecell-partial">%s</td>' % text
+        else:
+            if 'modes' in articlename:
+                text = '+' + text
+            try:
+                text, note = text.split(' ', 1)
+            except ValueError:
+                if text.endswith('*'):
+                    text = '<td class="tablecell-special">%s</td>' % text
+                else:
+                    text = '<td class="tablecell-yes">%s</td>' % text
+            else:
+                text = '%s<br><span class="note">%s</span>' % (text, note)
+                text = '<td class="tablecell-yes2">%s</td>' % text
+    else:
+        text = '<td class="tablecell-na note">n/a</td>'
+    return text
+
+for fname, title in FILES.items():
+    outfname = os.path.splitext(fname)[0] + '.html'
+    print('Generating HTML for %s to %s:' % (fname, outfname))
+    with open(fname) as csvfile:
+        csvdata = csv.reader(csvfile)
+
+        with open(outfname, 'w') as outf:
+            # CSS in HTML in Python, how lovely...
+            _write(outf, """
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="UTF-8">
+<meta name=viewport content="width=device-width, initial-scale=1">
+
+<head>
+<title>%s</title>
+<style>
+
+html {
+    background-color: white;
+}
+
+.note {
+    color: #555555;
+}
+
+/* (╮°-°)╮┳━┳ */
+table, th, td {
+    border: 1px solid black;
+}
+
+td, th {
+    text-align: center;
+    padding: 3px;
+}
+
+td:first-child, th[scope="row"] {
+    text-align: left;
+}
+
+/* Table cells */
+.tablecell-yes {
+    background-color: #A7F2A5
+}
+
+.tablecell-no {
+    background-color: #F08496
+}
+
+.tablecell-na {
+    background-color: #F0F0F0
+}
+
+.tablecell-planned, .tablecell-yes2 {
+    background-color: #B1FCDE
+}
+
+.tablecell-partial {
+    background-color: #EDE8A4
+}
+
+.tablecell-special {
+    background-color: #DCB1FC
+}
+
+</style>
+
+</head>
+
+<body>
+<table>""" % title)
+            notes = False
+            for idx, row in enumerate(csvdata):
+                if not any(row):  # Empty row
+                    continue
+                elif row[0] == '----':
+                    notes = True
+                    continue
+
+                if notes:
+                    _write(outf, "<p>%s</p>" % row[0])
+                    continue
+
+                _write(outf, "<tr>\n")
+                for colidx, coltext in enumerate(row):
+                    if idx == 0:
+                        text = '<th scope="col">%s</th>\n' % coltext
+                    elif colidx == 0:
+                        text = '<th scope="row">%s</th>\n' % coltext
+                    else:
+                        text = _format(fname, coltext)
+                    _write(outf, text)
+
+                _write(outf, "</tr>\n")
+            _write(outf, """
+
+</table>
+</body>
+</html>""")

docs/modelists/user-modes.csv

@@ -0,0 +1,55 @@
+User Mode / IRCd,RFC 1459,hybrid,inspircd,ngircd,p10/ircu,p10/nefarious,p10/snircd,ts6/charybdis,ts6/chatircd,ts6/elemental,ts6/ratbox,unreal
+admin,,a,,,,a,,a,a,a,a,
+away,,,,a,,,,,,,,
+bot,,,B (botmode),B,,B,,,B,B,,B (usermodes/bot)
+callerid,,g,g (callerid),,,,,g,g,g,g,
+censor,,,,,,,,,,,,G (usermodes/censor)
+cloak,,x,x (cloaking),x,x,x,x,x,x,x,,x
+cloak_fakehost,,,,,,f,,,,,,
+cloak_hashedhost,,,,,,C,,,,,,
+cloak_hashedip,,,,,,c,,,,,,
+cloak_sethost,,,,,,h,h,,,,,
+deaf,,D,d (deaf),,d,d,d,D,D,D,D,d
+deaf_commonchan,,G,c (commonchans),C,,q,,,,,,
+debug,,d,,,,,,,,,,
+filter,,,,,,,,,,,,G
+floodexempt,,,,F,,,,,,,,
+helpop,,,h (helpop),,,,,,,,,
+hidechans,,p,I (hidechans),I,,n,n,,,I,,p (usermodes/privacy)
+hideidle,,q,,,,I,I,,,,,I
+hideoper,,H,H (hideoper),,,H,,,,,,H
+invisible,i,i,i,i,i,i,i,i,i,i,i,i
+locops,,l,,,O,O,O,l,l,l,l,
+netadmin,,,,,,,,,N,,,
+noctcp,,,,,,,,,,C,,T (usermodes/noctcp)
+noforward,,,L (redirect),,,L,,Q,Q,Q,,
+noinvite,,,,,,,,,,V,,
+oper,o,o,o,o,o,o,o,o,o,o,o,o
+operwall,,,,,,,,z,z,z,z,
+override,,,,,,X,X,p,p,p,,
+privdeaf,,,,b,,D,,,,,,D (usermodes/privdeaf)
+protected,,,,,,,,,,,,q (usermodes/nokick)
+regdeaf,,R,R (services_account),,,R,R,R,R,R,,R (usermodes/regonlymsg)
+registered,,r,r (services_account),R,r,r,r,,,,,r
+restricted,,,,r,,,,,,,,
+servprotect,,,k (servprotect),q,k,k,k,S,S,S,S,S (usermodes/servicebot)
+showwhois,,,W (showwhois),,,W,,,,,,W (usermodes/showwhois)
+sno_badclientconnections,,u,,,,,,,,,u,
+sno_botfloods,,b,,,,,,,,,b,
+sno_clientconnections,,c,,c,,,,,,,c,
+sno_debug,,,,,g,g,g,,,,d,
+sno_extclientconnections,,,,,,,,,,,C,
+sno_fullauthblock,,f,,,,,,,,,f,
+sno_nickchange,,n,,,,,,,,,,
+sno_rejectedclients,,j,,,,,,,,,r,
+sno_remoteclientconnections,,F,,,,,,,,,,
+sno_serverconnects,,e,,,,,,,,,x,
+sno_skill,,k,,,,,,,,,k,
+sno_stats,,y,,,,,,,,,y,
+snomask,s,s,s,s,s,s,s,s,s,s,s,s
+ssl,,S,,,,z,,,,,,z
+sslonlymsg,,,,,,,,,t,,,Z (usermodes/secureonlymsg)
+stripcolor,,,S (stripcolor),,,,,,,,,
+vhost,,,,,,,,,,,,t
+wallops,w,w,w,w,w,w,w,w,w,w,w,w
+webirc,,W,,,,,,,,,,

docs/modelists/user-modes.html

@@ -0,0 +1,244 @@
+
+<!DOCTYPE html>
+<html lang="en">
+<meta charset="UTF-8">
+<meta name=viewport content="width=device-width, initial-scale=1">
+
+<head>
+<title>Supported User Modes for PyLink</title>
+<style>
+
+html {
+    background-color: white;
+}
+
+.note {
+    color: #555555;
+}
+
+/* (╮°-°)╮┳━┳ */
+table, th, td {
+    border: 1px solid black;
+}
+
+td, th {
+    text-align: center;
+    padding: 3px;
+}
+
+td:first-child, th[scope="row"] {
+    text-align: left;
+}
+
+/* Table cells */
+.tablecell-yes {
+    background-color: #A7F2A5
+}
+
+.tablecell-no {
+    background-color: #F08496
+}
+
+.tablecell-na {
+    background-color: #F0F0F0
+}
+
+.tablecell-planned, .tablecell-yes2 {
+    background-color: #B1FCDE
+}
+
+.tablecell-partial {
+    background-color: #EDE8A4
+}
+
+.tablecell-special {
+    background-color: #DCB1FC
+}
+
+</style>
+
+</head>
+
+<body>
+<table><tr>
+<th scope="col">User Mode / IRCd</th>
+<th scope="col">RFC 1459</th>
+<th scope="col">hybrid</th>
+<th scope="col">inspircd</th>
+<th scope="col">ngircd</th>
+<th scope="col">p10/ircu</th>
+<th scope="col">p10/nefarious</th>
+<th scope="col">p10/snircd</th>
+<th scope="col">ts6/charybdis</th>
+<th scope="col">ts6/chatircd</th>
+<th scope="col">ts6/elemental</th>
+<th scope="col">ts6/ratbox</th>
+<th scope="col">unreal</th>
+</tr>
+<tr>
+<th scope="row">admin</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+a</td><td class="tablecell-yes">+a</td><td class="tablecell-yes">+a</td><td class="tablecell-yes">+a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">away</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">bot</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+B<br><span class="note">(botmode)</span></td><td class="tablecell-yes">+B</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+B</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+B</td><td class="tablecell-yes">+B</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+B<br><span class="note">(usermodes/bot)</span></td></tr>
+<tr>
+<th scope="row">callerid</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+g</td><td class="tablecell-yes2">+g<br><span class="note">(callerid)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+g</td><td class="tablecell-yes">+g</td><td class="tablecell-yes">+g</td><td class="tablecell-yes">+g</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">censor</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+G<br><span class="note">(usermodes/censor)</span></td></tr>
+<tr>
+<th scope="row">cloak</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+x</td><td class="tablecell-yes2">+x<br><span class="note">(cloaking)</span></td><td class="tablecell-yes">+x</td><td class="tablecell-yes">+x</td><td class="tablecell-yes">+x</td><td class="tablecell-yes">+x</td><td class="tablecell-yes">+x</td><td class="tablecell-yes">+x</td><td class="tablecell-yes">+x</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+x</td></tr>
+<tr>
+<th scope="row">cloak_fakehost</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+f</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">cloak_hashedhost</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+C</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">cloak_hashedip</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+c</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">cloak_sethost</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+h</td><td class="tablecell-yes">+h</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">deaf</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+D</td><td class="tablecell-yes2">+d<br><span class="note">(deaf)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+d</td><td class="tablecell-yes">+d</td><td class="tablecell-yes">+d</td><td class="tablecell-yes">+D</td><td class="tablecell-yes">+D</td><td class="tablecell-yes">+D</td><td class="tablecell-yes">+D</td><td class="tablecell-yes">+d</td></tr>
+<tr>
+<th scope="row">deaf_commonchan</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+G</td><td class="tablecell-yes2">+c<br><span class="note">(commonchans)</span></td><td class="tablecell-yes">+C</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+q</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">debug</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+d</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">filter</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+G</td></tr>
+<tr>
+<th scope="row">floodexempt</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+F</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">helpop</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+h<br><span class="note">(helpop)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">hidechans</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+p</td><td class="tablecell-yes2">+I<br><span class="note">(hidechans)</span></td><td class="tablecell-yes">+I</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+n</td><td class="tablecell-yes">+n</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+I</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+p<br><span class="note">(usermodes/privacy)</span></td></tr>
+<tr>
+<th scope="row">hideidle</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+q</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+I</td><td class="tablecell-yes">+I</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+I</td></tr>
+<tr>
+<th scope="row">hideoper</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+H</td><td class="tablecell-yes2">+H<br><span class="note">(hideoper)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+H</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+H</td></tr>
+<tr>
+<th scope="row">invisible</th>
+<td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td><td class="tablecell-yes">+i</td></tr>
+<tr>
+<th scope="row">locops</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+l</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+O</td><td class="tablecell-yes">+O</td><td class="tablecell-yes">+O</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-yes">+l</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">netadmin</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+N</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">noctcp</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+C</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+T<br><span class="note">(usermodes/noctcp)</span></td></tr>
+<tr>
+<th scope="row">noforward</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+L<br><span class="note">(redirect)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+L</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+Q</td><td class="tablecell-yes">+Q</td><td class="tablecell-yes">+Q</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">noinvite</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+V</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">oper</th>
+<td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td><td class="tablecell-yes">+o</td></tr>
+<tr>
+<th scope="row">operwall</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+z</td><td class="tablecell-yes">+z</td><td class="tablecell-yes">+z</td><td class="tablecell-yes">+z</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">override</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+X</td><td class="tablecell-yes">+X</td><td class="tablecell-yes">+p</td><td class="tablecell-yes">+p</td><td class="tablecell-yes">+p</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">privdeaf</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+b</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+D</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+D<br><span class="note">(usermodes/privdeaf)</span></td></tr>
+<tr>
+<th scope="row">protected</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+q<br><span class="note">(usermodes/nokick)</span></td></tr>
+<tr>
+<th scope="row">regdeaf</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+R</td><td class="tablecell-yes2">+R<br><span class="note">(services_account)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+R</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+R<br><span class="note">(usermodes/regonlymsg)</span></td></tr>
+<tr>
+<th scope="row">registered</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+r</td><td class="tablecell-yes2">+r<br><span class="note">(services_account)</span></td><td class="tablecell-yes">+R</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-yes">+r</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+r</td></tr>
+<tr>
+<th scope="row">restricted</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+r</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">servprotect</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+k<br><span class="note">(servprotect)</span></td><td class="tablecell-yes">+q</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+k</td><td class="tablecell-yes">+S</td><td class="tablecell-yes">+S</td><td class="tablecell-yes">+S</td><td class="tablecell-yes">+S</td><td class="tablecell-yes2">+S<br><span class="note">(usermodes/servicebot)</span></td></tr>
+<tr>
+<th scope="row">showwhois</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+W<br><span class="note">(showwhois)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+W</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+W<br><span class="note">(usermodes/showwhois)</span></td></tr>
+<tr>
+<th scope="row">sno_badclientconnections</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+u</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+u</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_botfloods</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+b</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+b</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_clientconnections</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+c</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+c</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+c</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_debug</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+g</td><td class="tablecell-yes">+g</td><td class="tablecell-yes">+g</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+d</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_extclientconnections</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+C</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_fullauthblock</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+f</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+f</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_nickchange</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+n</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_rejectedclients</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+j</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+r</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_remoteclientconnections</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+F</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_serverconnects</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+e</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+x</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_skill</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+k</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+k</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">sno_stats</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+y</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+y</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">snomask</th>
+<td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td><td class="tablecell-yes">+s</td></tr>
+<tr>
+<th scope="row">ssl</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+S</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+z</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+z</td></tr>
+<tr>
+<th scope="row">sslonlymsg</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+t</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+Z<br><span class="note">(usermodes/secureonlymsg)</span></td></tr>
+<tr>
+<th scope="row">stripcolor</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes2">+S<br><span class="note">(stripcolor)</span></td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+<tr>
+<th scope="row">vhost</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-yes">+t</td></tr>
+<tr>
+<th scope="row">wallops</th>
+<td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td><td class="tablecell-yes">+w</td></tr>
+<tr>
+<th scope="row">webirc</th>
+<td class="tablecell-na note">n/a</td><td class="tablecell-yes">+W</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td><td class="tablecell-na note">n/a</td></tr>
+
+
+</table>
+</body>
+</html>

docs/permissions-reference.md

@@ -0,0 +1,121 @@
+# PyLink Permissions Reference
+
+Below is a list of all the permissions defined by PyLink and its official plugins.
+
+## PyLink Core
+- `core.clearqueue` - Grants access to the `clearqueue` command.
+- `core.load` - Grants access to the `load` command.
+- `core.rehash` - Grants access to the `rehash` command.
+- `core.reload` - Grants access to the `reload`, `load`, and `unload` commands. (This implies access to `load` and `unload` because `reload` is really just those two commands combined.)
+- `core.shutdown` - Grants access to the `shutdown` command.
+- `core.unload` - Grants access to the `unload` command.
+
+## Automode
+
+By default, Automode integrates with Relay by only allowing access lists to be created / manipulated on channels that are owned by a network via Relay.
+
+- `automode.manage` OR `automode.manage.*`: ability to manage Automode (use `setacc` and `delacc`) on all channels on the network where the user is connected.
+- `automode.manage.relay_owned`: ability to manage Automode on channels owned by the current network in Relay. If Relay isn't loaded or the channel in question isn't shared via Relay, this permission check FAILS. **With the default permissions set, this is granted to all opers.**
+- `automode.manage.#channel`: ability to manage Automode on the specific given channel.
+
+- `automode.list` OR `automode.list.*`: ability to list Automode on all channels. **With the default permissions set, this is granted to all opers.**
+- `automode.list.relay_owned`: ability to list automode on channels owned via Relay. If Relay isn't loaded or the channel in question isn't shared via Relay, this permission check FAILS.
+- `automode.list.#channel`: ability to list Automode access entries on the specific given channel.
+
+- `automode.sync` OR `automode.sync.*`: ability to sync automode on all channels.
+- `automode.sync.relay_owned`: ability to sync automode on channels owned via Relay. If Relay isn't loaded or the channel in question isn't shared via Relay, this permission check FAILS. **With the default permissions set, this is granted to all opers.**
+- `automode.sync.#channel`: ability to sync automode on the specific given channel.
+
+- `automode.clear` OR `automode.clear.*`: ability to clear automode on all channels.
+- `automode.clear.relay_owned`: ability to clear automode on channels owned via Relay. If Relay isn't loaded or the channel in question isn't shared via Relay, this permission check FAILS.
+- `automode.clear.#channel`: ability to clear automode on the specific given channel.
+
+- `automode.savedb`: ability to save the automode DB.
+
+Remote versions of the `manage`, `list`, `sync`, and `clear` commands also exist for cross-network manipulation (e.g. `automode.remotemanage.*`)
+
+## Bots
+
+- `bots.join` - Grants access to the `join` command. `bots.joinclient` is a deprecated alias for this, retained for compatibility with PyLink < 2.0-rc1.
+- `bots.msg` - Grants access to the `msg` command.
+- `bots.nick` - Grants access to the `nick` command.
+- `bots.part` - Grants access to the `part` command.
+- `bots.quit` - Grants access to the `quit` command.
+- `bots.spawnclient` - Grants access to the `spawnclient` command.
+
+## Changehost
+
+- `changehost.applyhosts` - Grants access to the `applyhosts` command.
+
+## Commands
+- `commands.echo` - Grants access to the `echo` command.
+- `commands.loglevel` - Grants access to the `loglevel` command.
+- `commands.logout.force` - Allows forcing logouts on other users via the `logout` command.
+- `commands.showchan` - Grants access to the `showchan` command. **With the default permissions set, this is granted to all users.**
+- `commands.shownet` - Grants access to the `shownet` command (basic info including netname, protocol module, and encoding). **With the default permissions set, this is granted to all users.**
+- `commands.shownet.extended` - Grants access to extended info in `shownet`, including connected status, target IP:port, and configured PyLink hostname / SID.
+- `commands.showuser` - Grants access to the `showuser` command. **With the default permissions set, this is granted to all users.**
+- `commands.status` - Grants access to the `status` command. **With the default permissions set, this is granted to all users.**
+
+## Exec
+- `exec.exec` - Grants access to the `exec` and `iexec` commands.
+- `exec.eval` - Grants access to the `eval`, `ieval`, `peval`, and `pieval` commands.
+- `exec.inject` - Grants access to the `inject` command.
+- `exec.threadinfo` - Grants access to the `threadinfo` command.
+
+## Global
+- `global.global` - Grants access to the `global` command.
+
+## Networks
+- `networks.autoconnect` - Grants access to the `autoconnect` command.
+- `networks.disconnect` - Grants access to the `disconnect` command.
+- `networks.reloadproto` - Grants access to the `reloadproto` command.
+- `networks.remote` - Grants access to the `remote` command.
+
+## Opercmds
+- `opercmds.checkban` - Grants access to the `checkban` command.
+- `opercmds.checkban.re` - Grants access to the `checkbanre` command **if** the caller also has `opercmds.checkban`.
+- `opercmds.chghost` - Grants access to the `chghost` command.
+- `opercmds.chgident` - Grants access to the `chgident` command.
+- `opercmds.chgname` - Grants access to the `chgname` command.
+- `opercmds.jupe` - Grants access to the `jupe` command.
+- `opercmds.kick` - Grants access to the `kick` command.
+- `opercmds.kill` - Grants access to the `kill` command.
+- `opercmds.massban` - Grants access to the `massban` command.
+- `opercmds.massban.re` - Grants access to the `massbanre` command **if** the caller also has `opercmds.massban`.
+- `opercmds.mode` - Grants access to the `mode` command.
+- `opercmds.topic` - Grants access to the `topic` command.
+
+## Raw
+- `raw.raw` - Grants access to the `raw` command. `exec.raw` is equivalent to this and retained for compatibility with PyLink 1.x.
+- `raw.raw.unsupported_network` - Allows use of the `raw` command on servers other than Clientbot.
+
+## Relay
+These permissions are granted to all opers when the `relay::allow_free_oper_links` option is set (this is the default):
+
+- `relay.chandesc.remove` - Allows removing channel descriptions via the `chandesc` command.
+- `relay.chandesc.set` - Allows setting / updating channel descriptions via the `chandesc` command.
+- `relay.claim` - Grants access to the `claim` command.
+- `relay.create` - Grants access to the `create` command.
+- `relay.delink` - Grants access to the `delink` command.
+- `relay.destroy` - Grants access to the `destroy` command.
+- `relay.link` - Grants access to the `link` command.
+
+These permissions are always granted to all opers:
+- `relay.linkacl` - Allows managing LINKACL entries via the `linkacl` command.
+- `relay.linkacl.view` - Allows viewing LINKACL entries via the `linkacl` command.
+
+These permissions are not granted to anyone by default:
+- `relay.destroy.remote` - Allows destroying remote channels.
+- `relay.link.force_ts` - Grants access to the `link` command's `--force-ts` option (skip TS and target network is connected checks).
+- `relay.linked` - Grants access to the `link` command. **With the default permissions set, this is granted to all users.**
+- `relay.purge` - Grants access to the `purge` command.
+- `relay.savedb` - Grants access to the `savedb` command.
+
+## Servermaps
+- `servermaps.localmap` - Grants access to the `localmap` command.
+- `servermaps.map` - Grants access to the `map` command.
+
+## Stats
+- `stats.c`, `stats.o`, `stats.u` - Grants access to remote `/stats` calls with the corresponding letter.
+- `stats.uptime` - Grants access to the `stats` command.

docs/pylink-opers.md

@@ -1,82 +1 @@
-# Opering with PyLink Relay
+Moved to [relay-quickstart.md](relay-quickstart.md).
-
-*This guide was written for the OVERdrive-IRC network, but may be applicable elsewhere.*
-
-PyLink Relay behaves much like Janus, an extended service used to relay channels together. This guide goes over some of the basic oper commands in Relay, along with the best ways to handle channel emergencies.
-
-## How nick suffixing work
-
-When joining a relay channel, every user from another network will have a network tag attached to their name. The purpose of this is to prevent nick collisions from the same nick being used on multiple nets, and ensure that different networks' registered nicks remain separate.
-
-How is this relevant? Firstly, it means that you **cannot ban users from entire networks** using banmasks such as `*/net1!*@*`! The nick suffix is something PyLink adds artificially; on `net1`'s IRCd, which is checking the bans locally, the nick suffix simply doesn't exist.
-
-However, this *does* mean that you can effectively give access to remote users via services, by specifying masks such as `*/net1@someident@someperson.opers.somenet.org`. Just don't make masks too wide, or you risk getting channel takeovers.
-
-## Relay commands
-The concept of relay channels in PyLink is greatly inspired from the original Janus implementation, though with a few differences in command syntax.
-
-To create a channel:
-- `/msg PyLink create #channelname`
-
-To link to a channel already created on a different network:
-- `/msg PyLink link othernet #channelname`
-
-You can also link remote channels to take a different name on your network. (This is the third argument to the LINK command)
-- `/msg PyLink link othernet #lobby #othernet-lobby`
-
-Also, to list the available channels:
-- `/msg PyLink linked`
-
-To remove a relay channel that you've created:
-- `/msg PyLink destroy #channelname`
-
-To delink a channel linked to another network:
-- `/msg PyLink delink #channelname`
-
-### Claiming channels
-
-PyLink offers channel claims similarly to Janus, except that it is on by default when you create a channel on any network. Unless the claimed network list of a channel is EMPTY, oper override (MODE, KICK, TOPIC) will only be allowed from networks on that list.
-
-To set a claim (note: for these commands, you must be on the network which created the channel in question!):
-- `/msg PyLink claim #channel yournet,net2,net3` (the last parameter is a comma-separated list of networks, case-sensitive)
-
-To list claims on a channel:
-- `/msg PyLink claim #channel`
-
-To remove claims from a channel
-- `/msg PyLink claim #channel -`
-
-### Access control for links (LINKACL)
-LINKACL allows you to block certain networks from linking to your relay channels, based on a blacklist. By default, this blacklist is empty.
-
-To list blocked networks for a channel:
-- `/msg PyLink linkacl #channel list`
-
-To add a network to the blacklist:
-- `/msg PyLink linkacl #channel allow badnet`
-
-To remove a network from the blacklist:
-- `/msg PyLink linkacl #channel deny goodnet`
-
-Whitelists with LINKACL are not supported at this time.
-
-## Dealing with channel emergencies
-
-PyLink is not designed with the ability to forward KILLs, G:Lines, or any network bans. **The best thing to do in the case of emergencies is to delink the problem networks / channels!** Kills are actively blocked by the PyLink daemon (user is just respawned), while X:Lines are simply ignored, as there isn't any code to handle them yet.
-
-To delink another network from a channel your network owns:
-
-- `/msg PyLink delink #yourchannel badnetwork`
-
-To delink your network from a bad network's channel:
-
-- `/msg PyLink delink #badchannel`
-
-Basically, only one of the two above commands will work for one specific channel. Almost always, the network that owns a channel should be the one who has it registered via their services. You can see a list of channels by typing `/msg PyLink linked`.
-
-## When a network starts causing disconnect spam
-
-Juping an individual `net.relay` server will likely cause PyLink Relay to break or disconnect completely. When a network starts acting up and disconnecting frequently (and causing netsplit/quit floods), you should disable autoconnect for this network:
-
-- `/msg PyLink autoconnect badnetwork -1` (setting autoconnect to 0 or below will cause it to be disabled)
-

docs/relay-quickstart.md

@@ -0,0 +1,146 @@
+# PyLink Relay Quick Start
+
+## What is Relay?
+
+PyLink Relay is a plugin that provides transparent relays between channels on different networks. On participating networks, PyLink connects as a services server and mirrors messages as well as user lists from relayed channels, the latter by creating "puppet" service clients for all remote users in common channels. Relay offers an alternative to classic IRC linking, letting networks share channels on demand while retaining their services, policies, and distinct branding. By default, Relay also secures channels from remote oper overrides via a CLAIM feature, which restricts /kick, /mode, and /topic changes from un-opped users unless they are granted permissions via CLAIM.
+
+Relay shares many ideas from its predecessor Janus, but is a complete rewrite in Python. This guide goes over some of the basic commands in Relay, as well as some must-know gotchas.
+
+## Important notes (READ FIRST!)
+
+### How nick suffixing work
+
+By default, Relay will automatically tag users from other networks with a suffix such as `/net`. This prevents confusing nick collisions if the same nick is used on multiple linked networks, and ensure that nicks from remote networks are all isolated into their own namespaces.
+
+How is this relevant to an operator? It means that you **cannot ban users** using banmasks such as `*/net1!*@*`! The nick suffix is something PyLink adds artificially; on `net1`'s IRCd, which check the bans locally, the nick suffix doesn't exist and will therefore *not* match anyone.
+
+### Services compatibility
+
+While PyLink is generally able to run independently of individual networks' services, there are some gotchas. This list briefly details services features that have been known to cause problems with Relay. **Using any of these features in conjunction with Relay is *not* supported.**
+
+- Anope, Atheme: **Clones prevention should be DISABLED** (or at a minimum, set to use G/KLINE instead of KILL)
+    - Rationale: it is common for a person to want to connect to multiple networks in a Relay instance, because they are still independent entities. You can still use IRCd-side clones prevention, which sanely blocks connections instead of killing / banning everyone involved.
+- Anope: **SQLINE nicks should NOT be used**
+    - Rationale: Anope falls back to killing target clients matching a SQLINE, which will obviously cause conflicts with other services.
+- Atheme: **The ChanFix service should be disabled**
+    - Rationale: ChanFix is incompatible with Relay CLAIM because it overrides ops on relay channels whenever they appear "opless". This basic op check is unable to consider the case of remote channel services not being set to join channels, and will instead cause join/message/part spam as CLAIM reverts the ChanFix service's mode changes.
+- *Any*: **Do NOT register a relayed channel on multiple networks**
+    - Rationale: It is very easy for this to start kick or mode wars. (Bad akick mask? Secure ops enabled?)
+    - Clientbot is an exception to this, though you may want to add Clientbot networks to CLAIM so that PyLink doesn't try to reverse modes set by services on the Clientbot network.
+- *Any*: **Do NOT jupe virtual Relay servers** (e.g. `net.relay`)
+    - Rationale: This will just make PyLink split off - you should instead [delink any problem networks / channels](#dealing-with-disputes-and-emergencies).
+- Multiple PyLink Relay instances:
+    - **Do NOT connect a network twice to any PyLink instance**.
+    - **Do NOT connect a network to 2+ separate PyLink instances if there is another network already acting as a hub for them**.
+    - Not following these rules means that it's very easy for the Relay instances to go in a loop should an operator run the wrong command, which will hammer your CPU and relentlessly spam your channels.
+
+Note: P10-specific services packages have not been particularly tested - your feedback is welcome.
+
+## Relay commands
+
+The basic steps for setting up a relay is to first CREATE the channel with PyLink on the network that owns it, and run LINK from each network that wants to link to it. In most cases, you want to run CREATE on the network where the channel is registered with services.
+
+Importantly, this means that CREATE and LINK have to be run on different networks for any particular channel, and that you should only run CREATE once for each distinct channel! This setup is intended to allow individual network admins to pick and choose channels they want to participate in.
+
+First, to list all available channels:
+- `/msg PyLink linked`
+
+To create a channel on Relay:
+- `/msg PyLink create #channelname`
+- Note: **you can only create channels on full IRCd links - this will NOT work with Clientbot.**
+- A channel created on a particular network is considered to be _owned_ by that network; this affects how CLAIM works for instance (see the next section)
+
+To link to a channel already created on a different network:
+- `/msg PyLink link othernet #channelname`
+- You should replace `othernet` with the *short name* for the network that owns the channel.
+- Note: network names are case sensitive!
+
+You can also link remote channels while using a different name for it on your network. (This is the third argument to the LINK command)
+- `/msg PyLink link othernet #lobby #othernet-lobby`
+
+To completely remove a relay channel (on the network that created it):
+- `/msg PyLink destroy #channelname`
+
+To delink a channel *linked to another network*:
+- `/msg PyLink delink #localchannelname`
+
+To delink one of *your* channels from another network:
+- `/msg PyLink delink #yourchannelname <name-of-other-network>`
+
+Then, to list all available channels:
+- `/msg PyLink linked`
+
+### Claiming channels
+
+Channel claiming is a feature which prevents oper override (MODE, KICK, TOPIC, KILL, OJOIN, ...) by other networks' operators from affecting your channels. By default, CLAIM is enabled for all new channels, though this can be configured via the [`relay::enable_default_claim` option](https://github.com/jlu5/PyLink/blob/3.0.0/example-conf.yml#L828-L831). Unless the claimed network list of a channel is _empty__, oper override will only be allowed from networks on the CLAIM list (plus the network that owns the channel).
+
+Note: these commands must be run from the network which owns the channel in question!
+
+To set a claim:
+- `/msg PyLink claim #channel yournet,net2,net3` (the last parameter is a case-sensitive comma-separated list of networks)
+
+To list claim networks on a channel:
+- `/msg PyLink claim #channel`
+
+To clear the claim list for a channel:
+- `/msg PyLink claim #channel -`
+
+### Access control for links (LINKACL)
+
+LINKACL allows you to allow or deny networks from linking to your channel. New channels are created using a blacklist by default, though this can be configured via the [`relay::linkacl_use_whitelist` option](https://github.com/jlu5/PyLink/blob/3.0.0/example-conf.yml#L823-L826).
+
+To change between blacklist and whitelist mode:
+- `/msg PyLink linkacl whitelist #channel true/false`
+- Note that when you switch between LINKACL modes, the LINKACL entries from the previous mode are stored and stashed away. This means that you will get an empty LINKACL list in the new LINKACL mode if you haven't used it already, and that you can reload the previous LINKACL mode's entries by switching back to it at any point.
+
+To view the LINKACL networks for a channel:
+- `/msg PyLink linkacl #channel list`
+
+To add a network to the whitelist **OR** remove a network from the blacklist:
+- `/msg PyLink linkacl #channel allow goodnet`
+
+To remove a network from the whitelist **OR** add a network to the blacklist:
+- `/msg PyLink linkacl #channel deny badnet`
+
+### Adding channel descriptions
+
+Starting with PyLink 2.0, you can annotate your channels with a description to use in LINKED:
+
+To view the description for a channel:
+- `/msg PyLink chandesc #channel`
+
+To change the description for a channel:
+- `/msg PyLink chandesc #channel your text goes here`
+
+To remove the description for a channel:
+- `/msg PyLink chandesc #channel -`
+
+## Dealing with disputes and emergencies
+
+The best thing to do in the event of a dispute is to delink the problem networks / channels. In order for individual networks to maintain their autonomy, KILLs and network bans (K/G/ZLINE) will most often *not* behave the way you expect them to.
+
+### Kill handling
+
+Special kill handling was introduced in PyLink 2.0, while in previous versions they were always bounced:
+
+1) If the sender was a server and not a client, reject the kill. (This prevents services messups from wreaking havoc across the relay)
+2) If the target and source networks share a [kill share pool](https://github.com/jlu5/PyLink/blob/3.0.0/example-conf.yml#L782-L792), relay the kill as-is.
+3) Otherwise, check every channel that the kill target is in:
+    - If the sender is opped or has claim access in a channel, forward the KILL as a kick in that channel.
+    - Otherwise, bounce the kill silently (i.e. rejoin the user immediately).
+
+### Network bans (K/G/ZLINE)
+
+Network bans are purposely not supported; see https://github.com/jlu5/PyLink/issues/521#issuecomment-352316396.
+
+### Delinking channels
+
+To delink another network from a channel your network owns:
+
+- `/msg PyLink delink #yourchannel badnetwork`
+
+To delink your network from a bad network's channel:
+
+- `/msg PyLink delink #badchannel`
+
+Basically, only one of the two above commands will work for one specific channel. Almost always, the network that owns a channel should be the one who has it registered via their services. You can see a list of channels by typing `/msg PyLink linked`.

docs/technical/README.md

@@ -1,25 +1,28 @@
 # PyLink Developer Documentation
 
-Please note that as PyLink is still in its development phase, its APIs are subject to change.
+This documentation is provided for reference only, and may not always be up to date as APIs change.
-Any documentation here is provided for reference only.
+Patches are welcome if something looks wrong or *is* wrong. In such cases, consulting the source code is probably your best bet.
 
-The docs are also really incomplete (contributors welcome!)
+The docs are also really incomplete (contributions are appreciated!)
 
 ## Introduction
 
 PyLink is an a modular, plugin-based IRC services framework. It uses swappable protocol modules and a hooks system for calling plugins, allowing them to function regardless of the IRCd used.
 
-<img src="core-structure.png" width="50%" height="50%"> <img src="protocol-modules.png" width="50%" height="50%">
-
 ## Contents
 
 - [Writing plugins for PyLink](writing-plugins.md)
-- [PyLink protocol module specification](pmodule-spec.md)
 - [PyLink hooks reference](hooks-reference.md)
-- [Supported named channel modes](channel-modes.csv)
-- [Supported named user modes](user-modes.csv)
 - [Services bot API/Creating your own service bots](services-api.md)
+- [Permissions API Introduction](permissions-api.md)
+- [Using `utils.IRCParser()`](using-ircparser.md)
 
-### Future topics (not yet available)
+----
-- [Writing tests for PyLink modules](writing-tests.md)
 
+- [PyLink protocol module specification](pmodule-spec.md)
+
+----
+
+- [Release Process for PyLink](release-process.md)
+
+![Graph of protocol module inheritance tree](protocol-modules.svg)

docs/technical/channel-modes.csv

@@ -1,59 +0,0 @@
-Channel Mode / IRCd,InspIRCd,charybdis,Elemental-IRCd,UnrealIRCd,IRCd-Hybrid,Nefarious IRCu
-admin,a (m_customprefix/m_chanprotect),,a (when enabled),a,,
-adminonly,,A (extensions/chm_adminonly),A (extensions/chm_adminonly.c),,,a
-allowinvite,A (m_allowinvite),g,g,,,
-autoop,w (m_autoop),,,,,
-ban,b,b,b,b,b,b
-banexception,e (m_banexception),e,e,e,e,e
-blockcaps,B (m_blockcaps),,G (extensions/chm_nocaps.c),,,
-blockcolor,c (m_blockcolor),,,c,c,c
-delayjoin,,,,,,D
-exemptchanops,X (m_exemptchanops),,,,,
-filter,g (m_filter),,,,,
-flood,f (m_messageflood),,,,,
-flood_unreal,,,,f,,
-freetarget,,F,F,,,
-had_delayjoins,,,,,,d
-halfop,h (m_customprefix/m_halfop),,h (when enabled),h,h,
-hiddenbans,,,u,,,
-hidequits,,,,,,Q
-history,H (m_chanhistory),,,,,
-invex,I (m_inviteexception),I,I,I,I,
-inviteonly,i,i,i,i,i,i
-issecure,,,,Z,,
-joinflood,j (m_joinflood),j,j,,,
-key,k,k,k,k,k,k
-kicknorejoin,J (m_kicknorejoin),,J,,,
-largebanlist,,L,L,,,
-limit,l,l,l,l,l,l
-moderated,m,m,m,m,m,m
-nickflood,F (m_nickflood),,,,,
-noamsg,,,,,,T
-noctcp,C (m_noctcp),C,C,C,C,C
-noextmsg,n,n,n,n,n,n
-noforwards,,Q,Q,,,
-noinvite,,,,,,
-nokick,Q (m_nokicks),,E,Q,,
-noknock,K (m_knock),p,,K,p,
-nonick,N (m_nonicks),,d,N,,
-nonotice,T (m_nonotice),T (extensions/chm_nonotice),T,T,,N
-official-join,Y (m_ojoin),,,,,
-op,o,o,o,o,o,o
-operonly,O (m_operchans),O (extensions/chm_operonly),O (extensions/chm_operonly.c),O,O,O
-oplevel_apass,,,,,,A
-oplevel_upass,,,,,,U
-opmoderated,U (extras/m_opmoderated),z,z,,,
-owner,q (m_customprefix/m_chanprotect),,y (when enabled),q,,
-permanent,P (m_permchannels),P,P,P,,z
-private,p,,,p,,p
-quiet,,q,q,,,
-redirect,L (m_redirect),f,f,L,,L
-registered,r (m_services_account),,,r,r,R
-regmoderated,M (m_services_account),,,M,M,M
-regonly,R (m_services_account),r,r,R,R,r
-repeat,E (m_repeat),,K (extensions/chm_norepeat.c),,,
-secret,s,s,s,s,s,s
-sslonly,z (m_sslmodes),S (extensions/chm_sslonly),S (extensions/chm_sslonly.c),z,S,
-stripcolor,S (m_stripcolor),c,c,S,,S
-topiclock,t,t,t,t,t,t
-voice,v,v,v,v,v,v

docs/technical/core-structure.dot

@@ -1,27 +0,0 @@
-/* Graph for the PyLink Application Structure:
- * Update using: dot -Tpng core-structure.dot > core-structure.png
- */
-
-digraph G {
-	ratio = 0.8; /* make the graph wider than tall */
-	subgraph cluster_core {
-		label="PyLink Application Structure";
-		style="filled";
-		node [style="filled",color="white"];
-		color="lightblue";
-
-		"IRC objects" -> "Protocol modules" [label="Data relayed"]
-		"Protocol modules" -> "PyLink hooks" -> Plugins;
-		"IRC objects" -> "PyLink hooks";
-		"Main program" -> "IRC objects" [color=indigo] [label="One per network\nspawned"] [fontcolor=indigo];
-		"Main program" -> "IRC objects" [color=indigo];
-		"Main program" -> "IRC objects" [color=indigo];
-		"Protocol modules" -> "IRC objects" [label="States updated"] [color=darkgreen] [fontcolor=darkgreen];
-		"Main program" -> Plugins [label="Plugin loaders"];
-	}
-
-	"Protocol modules" -> "IRCds" -> "Protocol modules";
-	Plugins -> "Protocol modules" [label="Communication via\nIRC command\nsenders"] [color=navyblue] [fontcolor=navyblue];
-	Plugins -> "Main program" [label="Registers commands\n& hook handlers"] [color=brown] [fontcolor=brown];
-
-}

docs/technical/hooks-reference.md

@@ -1,13 +1,13 @@
 # PyLink hooks reference
 
-## Introduction
+***Last updated for 3.1-dev (2021-06-13).***
 
 In PyLink, protocol modules communicate with plugins through a system of hooks. This has the benefit of being IRCd-independent, allowing most plugins to function regardless of the IRCd being used.
 Each hook payload is formatted as a Python `list`, with three arguments: `numeric`, `command`, and `args`.
 
 1) **numeric**: The sender of the hook payload (normally a UID or SID).
 
-2) **command**: The command name (hook name) of the payload. These are *always* UPPERCASE, and those starting with "PYLINK_" indicate hooks sent out by PyLink IRC objects themselves; i.e. they don't require protocol modules to handle them.
+2) **command**: The command name (hook name) of the payload. These are *always* UPPERCASE, and those starting with "PYLINK_" indicate hooks sent out by PyLink IRC objects themselves (i.e. they don't require protocol modules to handle them).
 
 3) **args**: The hook data (args), a Python `dict`, with different data keys and values depending on the command given.
 
@@ -19,26 +19,17 @@ The command `:42XAAAAAB PRIVMSG #dev :test` would result in the following raw ho
 
 - `['42XAAAAAB', 'PRIVMSG', {'target': '#dev', 'text': 'test', 'ts': 1451174041}]`
 
-On UnrealIRCd, because SETHOST is mapped to CHGHOST, `:GL SETHOST blah` would return the raw hook data of this (with the nick converted into UID automatically by the protocol module):
+On UnrealIRCd, because SETHOST is mapped to CHGHOST, `:jlu5 SETHOST blah` would return the raw hook data of this (with the nick converted into UID automatically by the protocol module):
 
 - `['001ZJZW01', 'CHGHOST', {'ts': 1451174512, 'target': '001ZJZW01', 'newhost': 'blah'}]`
 
-Some hooks, like MODE, are more complex and can include the entire state of a channel! This will be further described later. `:GL MODE #chat +o PyLink-devel` is converted into (pretty-printed for readability):
+Some hooks, like MODE, are more complex and can include the entire state of a channel. This will be further described later. `:jlu5 MODE #chat +o PyLink-devel` is converted into (pretty-printed for readability):
 
 ```
 ['001ZJZW01',
  'MODE',
  {'modes': [('+o', '38QAAAAAA')],
-  'oldchan': IrcChannel({'modes': set(),
+  'channeldata': Channel(...),
-                        'prefixmodes': {'admin': set(),
-                                        'halfop': set(),
-                                        'op': set(),
-                                        'owner': set(),
-                                        'voice': set()},
-                        'topic': '',
-                        'topicset': False,
-                        'ts': 1451169448,
-                        'users': {'38QAAAAAA', '001ZJZW01'}}),
   'target': '#chat',
   'ts': 1451174702}]
 ```
@@ -52,8 +43,9 @@ These following hooks, sent with their correct data keys, are required for PyLin
     - This payload should be sent whenever a server finishes its burst, with the SID of the bursted server as the sender.
     - The service bot API and plugins like relay use this to make sure networks are properly connected. Should ENDBURST not be sent or emulated, they will likely fail to spawn users entirely.
 
-- **PYLINK_DISCONNECT**: `{}`
+- **PYLINK_DISCONNECT**: `{'was_successful': False}`
     - This is sent to plugins by IRC object instances whenever their network has disconnected. The sender here is always **None**.
+    - The `was_successful` key shows whether the last connection before this message was successful (i.e. whether the disconnect wasn't caused by a configuration error, etc.)
 
 ## IRC command hooks
 
@@ -67,50 +59,53 @@ The following hooks represent regular IRC commands sent between servers.
     - `modes` returns a list of parsed modes: `(mode character, mode argument)` tuples, where the mode argument is either `None` (for modes without arguments), or a string.
     - The sender of this hook payload is IRCd-dependent, and is determined by whether the command was originally a SJOIN or regular JOIN - SJOIN is only sent by servers, and JOIN is only sent by users.
     - For IRCds that support joining multiple channels in one command (`/join #channel1,#channel2`), consecutive JOIN hook payloads of this format will be sent (one per channel).
+    - For SJOIN, the `channeldata` key may also be sent, with a copy of the `classes.Channel` object *before* any mode changes from this burst command were processed.
 
 - **KICK**: `{'channel': '#channel', 'target': 'UID1', 'text': 'some reason'}`
     - `text` refers to the kick reason. The `target` and `channel` fields send the target's UID and the channel they were kicked from, and the sender of the hook payload is the kicker.
 
-- **KILL**: `{'target': killed, 'text': args[1], 'userdata': data}`
+- **KILL**: `{'target': killed, 'text': 'Killed (james (absolutely not))', 'userdata': User(...)}`
     - `text` refers to the kill reason. `target` is the target's UID.
-    - The `userdata` key may include an `IrcUser` instance, depending on the IRCd. On IRCds where QUITs are explicitly sent (InspIRCd), `userdata` will be `None`. Other IRCds do not explicitly send QUIT messages for KILLed clients, so the daemon must assume that they've quit, and deliver their last state to plugins that require this info.
+    - `userdata` includes a `classes.User` instance, containing the information of the killed user.
 
-- **MODE**: `{'target': '#channel', 'modes': [('+m', None), ('+i', None), ('+t', None), ('+l', '3'), ('-o', 'person')], 'oldchan': IrcChannel(...)}`
+- **MODE**: `{'target': '#channel', 'modes': [('+m', None), ('+i', None), ('+t', None), ('+l', '3'), ('-o', 'person')], 'channeldata': Channel(...)}`
-    - `target` is the target the mode is being set on: it may be either a channel (for channel modes) OR a UID (for user modes).
+    - `target` is the target the mode is being set on: it may be either a channel (for channel modes) *or* a UID (for user modes).
     - `modes` is a list of prefixed parsed modes: `(mode character, mode argument)` tuples, but with `+/-` prefixes to denote whether each mode is being set or unset.
-    - For channels, the `oldchan` key is also sent, with the state of the channel BEFORE this MODE hook was processed.
+    - For channels, the `channeldata` key is also sent, with a copy of the `classes.Channel` object *before* this MODE hook was processed.
-        - One such use for this is to prevent oper-override hacks: checks for whether a sender is opped have to be done before the MODE is processed; otherwise, someone can simply op themselves and circumvent this detection.
+        - One use for this is to prevent oper-override hacks: checks for whether a sender is opped have to be done before the MODE is processed; otherwise, someone can simply op themselves and circumvent this detection.
 
 - **NICK**: `{'newnick': 'Alakazam', 'oldnick': 'Abracadabra', 'ts': 1234567890}`
 
 - **NOTICE**: `{'target': 'UID3', 'text': 'hi there!'}`
-    - *Note:* `target` can not only be a channel or a UID, but also a channel with a prefix attached (e.g. `@#lounge`). These cases should not be overlooked!
+    - STATUSMSG targets (e.g. `@#lounge`) are also allowed here.
 
 - **PART**: `{'channels': ['#channel1', '#channel2'], 'text': 'some reason'}`
     - `text` can also be an empty string, as part messages are *optional* on IRC.
-    - Unlike the JOIN hook, multiple channels can be specified in a list for PART. This means that a user PARTing one channel will cause a payload to be sent with `channels` as a one-length *list* with the channel name.
+    - Unlike the JOIN hook, multiple channels can be specified in a list for PART. This means that a user parting one channel will cause a payload to be sent with `channels` as a one-length *list* with the channel name.
 
 - **PRIVMSG**: `{'target': 'UID3', 'text': 'hi there!'}`
-    - Ditto with NOTICE: `target` can be a channel or a UID, or a channel with a prefix attached (e.g. `@#lounge`).
+    - Ditto with NOTICE: STATUSMSG targets (e.g. `@#lounge`) are also allowed here.
 
-- **QUIT**: `{'text': 'Quit: Bye everyone!'}`
+- **QUIT**: `{'text': 'Quit: Bye everyone!', 'userdata': User(...)}`
     - `text` corresponds to the quit reason.
+    - `userdata` includes a `classes.User` instance, containing the information of the killed user.
 
-- **SQUIT**: `{'target': '800', 'users': ['UID1', 'UID2', 'UID6'], 'name': 'some.server', 'uplink': '24X', 'nicks': {'#channel1: ['tester1', 'tester2'], '#channel3': ['somebot']}, 'serverdata': IrcServer(...)`
+- **SQUIT**: `{'target': '800', 'users': ['UID1', 'UID2', 'UID6'], 'name': 'some.server', 'uplink': '24X', 'nicks': {'#channel1: ['tester1', 'tester2'], '#channel3': ['somebot']}, 'serverdata': Server(...), 'affected_servers': ['SID1', 'SID2', 'SID3']`
     - `target` is the SID of the server being split, while `name` is the server's name.
     - `users` is a list of all UIDs affected by the netsplit. `nicks` maps channels to lists of nicks affected.
-    - `serverdata` provides the `IrcServer` object of the server that was split.
+    - `serverdata` provides the `classes.Server` object of the server that split off.
-    - `chandata` provides the channel index of the network before the netsplit was processed, allowing plugins to track who was affected by a netsplit in a channel specific way.
+    - `channeldata` provides the channel index of the network before the netsplit was processed, allowing plugins to track who was affected by a netsplit in a channel specific way.
 
 - **TOPIC**: `{'channel': channel, 'setter': numeric, 'text': 'Welcome to #Lounge!, 'oldtopic': 'Welcome to#Lounge!'}`
     - `oldtopic` denotes the original topic, and `text` indicates the new one being set.
-    - `setter` is the raw sender field given to us by the IRCd; it may be a `nick!user@host`, a UID, a SID, a server name, or a nick. This is not processed any further.
+    - `setter` is the raw sender field given to us by the IRCd; it may be a `nick!user@host`, a UID, a SID, a server name, or a nick. This is not processed at the protocol level.
 
-- **UID**: `{'uid': 'UID1', 'ts': 1234567891, 'nick': 'supercoder', 'realhost': 'localhost', 'host': 'admin.testnet.local', 'ident': ident, 'ip': '127.0.0.1'}`
+- **UID**: `{'uid': 'UID1', 'ts': 1234567891, 'nick': 'supercoder', 'realhost': 'localhost', 'host': 'admin.testnet.local', 'ident': ident, 'ip': '127.0.0.1', 'secure': True}`
     - This command is used to introduce users; the sender of the message should be the server bursting or announcing the connection.
     - `ts` refers to the user's signon time.
+    - `secure` is a ternary value (True/False/None) that determines whether the user is connected over a secure connection (SSL/TLS). This value is only available on some IRCds: currently UnrealIRCd, P10, Charybdis TS6, and Hybrid; on other servers this will be `None`.
 
-### Extra commands (where supported by the IRCd)
+### Extra commands (where supported)
 
 - **AWAY**: `{'text': text}`
     - `text` denotes the away reason. It is an empty string (`''`) when a user is unsetting their away status.
@@ -124,13 +119,14 @@ The following hooks represent regular IRC commands sent between servers.
 - **CHGNAME**: `{'target': 'UID2', 'newgecos': "I ain't telling you!"}`
     - SETNAME and CHGNAME commands, where available, both share this hook name.
 
-- **INVITE**: `{'target': 'UID3', 'channel': '#myroom'}`
+- **INVITE**: `{'target': 'UID3', 'channel': '#hello'}`
 
-- **KNOCK**: `{'text': 'let me in please!', 'channel': '#myroom'}`
+- **KNOCK**: `{'text': 'let me in please!', 'channel': '#hello'}`
     - This is not actually implemented by any protocol module as of writing.
 
 - **SAVE**: `{'target': 'UID8', 'ts': 1234567892, 'oldnick': 'Abracadabra'}`
     - For protocols that use TS6-style nick saving. During nick collisions, instead of killing the losing client, servers that support SAVE will send such a command targeting the losing client, which forces that user's nick to their UID.
+    - As of 1.1.x, SAVE is also used internally to alert plugins of nick collisions, when protocol modules receive a user introduction for a nick that already exists.
 
 - **SVSNICK**: `{'target': 'UID1', 'newnick': 'abcd'}`
     - PyLink does not comply with SVSNICK requests, but instead forwards it to plugins that listen for it.
@@ -138,11 +134,11 @@ The following hooks represent regular IRC commands sent between servers.
 
 - **VERSION**: `{}`
     - This is used for protocols that send VERSION requests between servers when a client requests it (e.g. `/raw version pylink.local`).
-    - `coreplugin` automatically handles this by responding with a 351 numeric, with the data being the output of `utils.fullVersion(irc)`.
+    - `coremods/handlers.py` automatically handles this by responding with a 351 numeric, with the data being the output of `irc.version()`.
 
 - **WHOIS**: `{'target': 'UID1'}`
     - On protocols supporting it (everything except InspIRCd), the WHOIS command is sent between servers for remote WHOIS requests.
-    - This requires servers to respond with a complete WHOIS reply (using all the different numerics), as done in `coreplugin`.
+    - PyLink has built-in handling for this via `coremods/handlers.py` - plugins wishing to add custom WHOIS text should use the PYLINK_CUSTOM_WHOIS hook below.
 
 ## Hooks that don't map to IRC commands
 Some hooks do not map directly to IRC commands, but to events that protocol modules should handle.
@@ -159,9 +155,26 @@ Some hooks do not map directly to IRC commands, but to events that protocol modu
     - The sender here is always **None**.
 
 - **PYLINK_CUSTOM_WHOIS**: `{'target': UID1, 'server': SID1}`
-    - This hook is called by `coreplugin` during its WHOIS handling process, to allow plugins to provide custom WHOIS information. The `target` field represents the target UID, while the `server` field represents the SID that should be replying to the WHOIS request. The source of the payload is the user using `/whois`.
+    - This hook is called by `coremods/handlers.py` during its WHOIS handling process, to allow plugins to provide custom WHOIS information. The `target` field represents the target UID, while the `server` field represents the SID that should be replying to the WHOIS request. The source of the payload is the user using `/whois`.
-    - Plugins wishing to implement this should use the standard WHOIS numerics, using `irc.proto.numeric()` to reply to the source from the given server.
+    - Plugins wishing to implement this should use the standard WHOIS numerics, using `irc.numeric()` to reply to the source from the given server.
-    - This hook replaces the pre-0.8 fashion of defining custom WHOIS handlers, which was non-standard and poorly documented.
+    - This hook replaces the pre-0.8.x fashion of defining custom WHOIS handlers, which was never standardized and poorly documented.
 
 ## Commands handled WITHOUT hooks
 At this time, commands that are handled by protocol modules without returning any hook data include PING, PONG, and various commands sent during the initial server linking phase.
+
+## Changes
+
+* 2021-06-13 (3.1-dev)
+    - Added the `secure` field to `UID` hooks.
+* 2019-07-01 (2.1-alpha2)
+    - KILL and QUIT hooks now always include a non-empty `userdata` key. Now, if a QUIT message for a killed user is received before the corresponding KILL (or vice versa), only the first message received will have the corresponding hook payload broadcasted.
+* 2018-12-27 (2.1-dev)
+    - Add the `affected_servers` argument to SQUIT hooks.
+* 2018-07-11 (2.0.0)
+    - Version bump for 2.0 stable release; no meaningful content changes.
+* 2018-01-13 (2.0-alpha2)
+    - Replace `IrcChannel`, `IrcUser`, and `IrcServer` with their new class names (`classes.Channel`, `classes.User`, and `classes.Server`)
+    - Replace `irc.fullVersion()` with `irc.version()`
+    - Various minor wording tweaks.
+* 2017-02-24 (1.2-dev)
+    - The `was_successful` key was added to PYLINK_DISCONNECT.

docs/technical/permissions-api.md

@@ -0,0 +1,33 @@
+# The Permissions API
+
+Permissions were introduced in PyLink 1.0 as a way for plugins to manage command access, replacing the old `irc.checkAuthenticated()`. The permissions system in PyLink is fairly simple, globally assigning a list of permissions to each hostmask/exttarget.
+
+Permissions conventionally take the format `pluginname.commandname.optional_extra_portion(s)`, and support globs† in matching. Permission nodes are case-insensitive and casemapping aware, but are defined as being all lowercase for consistency.
+
+The permissions module is available as `pylinkirc.coremods.permissions`. Usually, plugins import it this format:
+
+```python
+from pylinkirc.coremods import permissions
+```
+
+† The globbing used by the permissions module is just generic IRC-style globbing. For example, anyone with `*`, `perm.*`, `perm.?`, `*.1`, etc. in their permissions list will be allowed to use a command checking for a permission named `perm.1`.
+
+## Checking for permissions
+
+Individual functions check for permissions using the `permissions.checkPermissions(irc, source, ['perm.1', 'perm.2'])` function, where the last argument is an OR'ed list of permissions matched against a list of permission string globs that a user may have. If the user has any of the permissions in the permission list, they will be allowed to call the command. This function returns `True` when a permission check passes, and raises `utils.NotAuthorizedError` when a check fails, automatically aborting the execution of the command function.
+
+`utils.NotAuthorizedError` can be treated like any other exception, so it's possible to wrap it around `try:` / `except:` for more complex access checking ([example in the Automode plugin](https://github.com/jlu5/PyLink/blob/1.1.1/plugins/automode.py#L64-L68)).
+
+## Assigning default permissions
+
+Plugins are also allowed to assign default permissions to their commands, though this should be used sparingly to ensure maximum configurability (explicitly removing permissions isn't supported yet). Default permissions are specified as a `dict` mapping targets to permission lists.
+
+Example of this in [Automode](https://github.com/jlu5/PyLink/blob/1.1-alpha1/plugins/automode.py#L38-L39):
+
+```python
+# The default set of Automode permissions.
+default_permissions = {"$ircop": ['automode.manage.relay_owned', 'automode.sync.relay_owned',
+                                  'automode.list']}
+```
+
+Default permissions are registered in a plugin's `main()` function via `permissions.addDefaultPermissions(default_permissions_dict)`, and should always be erased on `die()` through `permissions.removeDefaultPermissions(default_permissions_dict)`.

docs/technical/pmodule-spec.md

@@ -1,46 +1,69 @@
 # PyLink Protocol Module Specification
 
-In PyLink, each protocol module is a single file consisting of a protocol class, and a global `Class` attribute that is set equal to it (e.g. `Class = InspIRCdProtocol`). These classes should be based off of either [`classes.Protocol`](https://github.com/GLolol/PyLink/blob/e4fb64aebaf542122c70a8f3a49061386a00b0ca/classes.py#L532), a boilerplate class that only defines a few basic things, or something like  [`ts6_common.TS6BaseProtocol`](https://github.com/GLolol/PyLink/blob/0.9-alpha1/protocols/ts6_common.py). (`ts6_common.TS6BaseProtocol`  includes elements of the TS6 protocol that are shared by the InspIRCd, UnrealIRCd, and TS6 protocols.) IRC objects load protocol modules by creating an instance of its main class, and sends it commands accordingly.
+***Last updated for 3.1-dev (2021-06-15).***
 
-## Tasks
+Starting with PyLink 2.x, a *protocol module* is any module containing a class derived from `PyLinkNetworkCore` (e.g. `InspIRCdProtocol`), along with a global `Class` attribute set equal to it (e.g. `Class = InspIRCdProtocol`). These modules do everything from managing connections to providing plugins with an API to send and receive data. New protocol modules may be implemented based off any of the classes in the following inheritance tree, with each containing a different amount of abstraction.
 
-Protocol modules have some very important jobs. If any of these aren't done correctly, you will be left with a broken, desynced services server:
+![[Protocol module inheritence graph]](protocol-modules.svg)
 
-1) Handle incoming commands from the uplink IRCd.
+## Starting Steps
 
-2) Return [hook data](hooks-reference.md) for relevant commands, so that plugins can receive data from IRC.
+**Before you proceed, we highly recommend protocol module coders to get in touch with us** (e.g. via IRC at `#PyLink @ irc.overdrivenetworks.com`). Letting us know what you are working on can help coordinate coding efforts and better prepare for potential API breaks.
 
-3) Make sure channel/user states are kept correctly. Joins, quits, parts, kicks, mode changes, nick changes, etc. should all be handled accurately.
+Note: The following notes in this section assume that you are working on some IRCd's server protocol, such that PyLink can spawn subservers and its own pseudoclients. If this is not the case, *virtual* clients and servers have to be spawned instead to emulate the correct state - the `clientbot` protocol module is a functional (though not very elegant) example of this.
 
-4) Respond to both pings *and* pongs - the `irc.lastping` attribute **must** be set to the current time whenever a `PONG` is received from the uplink, so PyLink's doesn't [lag out the uplink thinking that it isn't responding to our pings](https://github.com/GLolol/PyLink/blob/e4fb64aebaf542122c70a8f3a49061386a00b0ca/classes.py#L309-L311).
+When writing new protocol modules, it is recommended to subclass from one of the following classes:
 
-5) Implement a series of outgoing command functions, used by plugins to send commands to IRC. See the `Outbound commands` section below for a list of which ones are needed.
+### `classes.IRCNetwork`
 
-6) Set the threading.Event object `irc.connected` (via `irc.connected.set()`) when the protocol negotiation with the uplink is complete. This is important for plugins like relay which must check that links are ready before spawning clients, and they will fail to work if this is not set.
+`IRCNetwork` is the base IRC class which includes the state checking utilities from `PyLinkNetworkCore`, the generic IRC utilities from `PyLinkNetworkCoreWithUtils`, along with abstraction for establishing IRC connections and pinging the uplink at a set interval.
 
-7) Check to see that RECVPASS is correct. Always.
+To use `classes.IRCNetwork`, the following functions must be defined:
 
-## Core functions
+- `handle_events(self, data)`: given a line of text containing an IRC command, parse it and return a hook payload as specified in the [PyLink hooks reference](hooks-reference.md).
+    - In all of the official PyLink modules so far, handling for specific commands is delegated into submethods via [`getattr()`](https://github.com/jlu5/PyLink/blob/3922d44173593e4bcceae1218bbc6f267caa9fc1/protocols/ircs2s_common.py#L409-L412), and unknown commands are ignored.
+- `post_connect(self)`: This method sends the server introduction commands to the uplink IRC server. This method replaces the `connect()` function defined by protocol modules prior to PyLink 2.x.
+- `_ping_uplink(self)`: Sends a ping command to the uplink. No return value is expected / used.
 
-The following functions *must* be implemented by any protocol module within its main class, since they are used by the IRC object internals.
+This class offers the most flexibility because the protocol module can choose how it wants to handle any command. However, because most IRC server protocols use the same RFC 1459-style message format, rewriting the entire event handler is often not worth doing. Instead, it may be better to use `IRCS2SProtocol`, as documented below, which includes a `handle_events` method which handles most cases (TS5/6, P10, and TS-less protocols such as ngIRCd).
 
-- **`connect`**`(self)` - Initializes a connection to a server.
+- An exception to this general statement is `clientbot`, whose event handler also checks for unknown message senders and enumerates them when such a message is received.
 
-- **`handle_events`**`(self, line)` - Handles inbound data (lines of text) from the uplink IRC server. Normally, this will pass commands to other command handlers within the protocol module, while dropping commands that are unrecognized (wildcard handling). But, it's really up to you how to structure your modules. You will want to be able to parse command arguments properly into a list: many protocols send RFC1459-style commands that can be parsed using the [`Protocol.parseArgs()`](https://github.com/GLolol/PyLink/blob/e4fb64aebaf542122c70a8f3a49061386a00b0ca/classes.py#L539) function.
+### `protocols.ircs2s_common.IRCCommonProtocol`
 
-- **`ping`**`(self, source=None, target=None)` - Sends a PING to a target server. Periodic PINGs are sent to our uplink automatically by the [`Irc()`
+`IRCCommonProtocol` (based off `IRCNetwork`) includes more IRC-specific methods such as parsers for ISUPPORT, as well as helper methods to parse arguments and recursively handle SQUIT. It also defines a default `_ping_uplink()` and incoming command handlers for commands that are the same across known protocols (AWAY, PONG, ERROR).
-internals](https://github.com/GLolol/PyLink/blob/0.4.0-dev/classes.py#L267-L272); plugins shouldn't have to use this.
 
-### Outgoing command functions
+`IRCCommonProtocol` does *not*, however, define an `handle_events` method.
 
-- **`spawnClient`**`(self, nick, ident='null', host='null', realhost=None, modes=set(), server=None, ip='0.0.0.0', realname=None, ts=None, opertype=None, manipulatable=False)` - Spawns a client on the PyLink server. No nick collision / valid nickname checks are done by protocol modules, as it is up to plugins to make sure they don't introduce anything invalid.
+### `protocols.ircs2s_common.IRCS2SProtocol`
-    - `modes` is a set of `(mode char, mode arg)` tuples in the form of [`utils.parseModes()` output](using-utils.md#parseModes).
+`IRCS2SProtocol` is the most complete base server class, including a generic `handle_events()` supporting most IRC S2S message styles (i.e. prefix-less messages, protocols with and without UIDs). It also defines some incoming and outgoing command functions that hardly vary between protocols: `invite()`, `kick()`, `message()`, `notice()`, `numeric()`, `part()`, `quit()`, `squit()`, and `topic()` as of PyLink 2.0. This list is subject to change in future releases.
-    - `ident` and `host` default to "null", while `realhost` defaults to the same things as `host` if not defined.
+
-    - `realname` defaults to the real name specified in the PyLink config, if not given.
+### `classes.PyLinkNetworkCoreWithUtils`
-    - `ts` defaults to the current time if not given.
+
-    - `opertype` (the oper type name, if applicable) defaults to the simple text of `IRC Operator`.
+`PyLinkNetworkCoreWithUtils` contains various state checking and IRC-related utility functions. Originally this abstraction was intended to support non-IRC protocols (Discord, Telegram, Slack, ...), but I (jlu5) no longer support this as a development focus. The main reason being is that in order to keep track of IRC server state correctly, PyLink makes a lot of assumptions specific to IRC (e.g. explicit join/part, mode formats, etc.). Trying to reconcile this with other platforms is a large undertaking and ideally requires a different, more generic protocol specification. (In PyLink 2.x there was a [Discord module](https://github.com/PyLink/pylink-discord) that is no longer supported - see https://jlu5.com/blog/the-trouble-with-pylink for a more in depth explanation as to why.)
+
+Subclassing one of the `PyLinkNetworkCore*` classes means that a protocol module only needs to define one method of entry: `connect()`, and must set up its own message handling stack. Protocol configuration validation checks and autoconnect must also be reimplemented. IRC-style utility functions (i.e. `PyLinkNetworkCoreWithUtils` methods) should also be reimplemented / overridden when applicable.
+
+(Unfortunately, this work is complicated, so please get in touch with us if you're stuck or want tips!)
+
+### Other
+
+For protocols that are closely related to existing ones, it may be wise to subclass off of an existing protocol class. For example, the `hybrid` and `ratbox` modules are based off of `ts6`. However, these protocol modules *do not guarantee API stability*, so we recommend letting us know of your intentions beforehand.
+
+## Outgoing command functions
+
+The methods defined below are integral to any protocol module, as they are needed by plugins to communicate with the rest of the world.
+
+Unless otherwise noted, the camel-case variants of command functions (e.g. "`spawnClient`) are supported but deprecated. Protocol modules do *not* need to implement these aliases themselves; attempts to missing camel case functions are automatically coersed into their snake case variants via the [`structures.CamelCaseToSnakeCase`](https://github.com/jlu5/PyLink/blob/3922d44173593e4bcceae1218bbc6f267caa9fc1/structures.py#L172-L197) wrapper.
+
+- **`spawn_client`**`(self, nick, ident='null', host='null', realhost=None, modes=set(), server=None, ip='0.0.0.0', realname=None, ts=None, opertype=None, manipulatable=False)` - Spawns a client on the PyLink server. No nick collision / valid nickname checks are done by protocol modules, as it is up to plugins to make sure they don't introduce anything invalid.
+    - `modes` is a list or set of `(mode char, mode arg)` tuples in the [PyLink mode format](#mode-formats).
+    - `ident` and `host` should default to "null", while `realhost` should default to the same things as `host` if not defined.
+    - `realname` should default to the real name specified in the PyLink config, if not given.
+    - `ts` should default to the current time if not given.
+    - `opertype` (the oper type name, if applicable) should default to the simple text of `IRC Operator`.
     - The `manipulatable` option toggles whether the client spawned should be considered protected. Currently, all this does is prevent commands from plugins like `bots` from modifying these clients, but future client protections (anti-kill flood, etc.) may also depend on this.
-    - The `server` option optionally takes a SID of any PyLink server, and spawns the client on the one given. It will default to the root PyLink server.
+    - The `server` option optionally takes a SID of any PyLink server, and spawns the client on the one given. It should default to the root PyLink server if not specified.
 
 - **`join`**`(self, client, channel)` - Joins the given client UID given to a channel.
 
@@ -48,11 +71,11 @@ internals](https://github.com/GLolol/PyLink/blob/0.4.0-dev/classes.py#L267-L272)
 
 - **`invite`**`(self, source, target, channel)` - Sends an INVITE from a PyLink client.
 
-- **`kick`**`(self, source, channel, target, reason=None)` - Sends a kick from a PyLink client/server.
+- **`kick`**`(self, source, channel, target, reason=None)` - Sends a kick from a PyLink client/server. This should raise `NotImplementedError` if not supported by a protocol.
 
-- **`kill`**`(self, source, target, reason)` - Sends a kill from a PyLink client/server.
+- **`kill`**`(self, source, target, reason)` - Sends a kill from a PyLink client/server. This should raise `NotImplementedError` if not supported by a protocol.
 
-- **`knock`**`(self, source, target, text)` - Sends a KNOCK from a PyLink client.
+- **`knock`**`(self, source, target, text)` - Sends a KNOCK from a PyLink client. This should raise `NotImplementedError` if not supported by a protocol.
 
 - **`message`**`(self, source, target, text)` - Sends a PRIVMSG from a PyLink client.
 
@@ -60,9 +83,11 @@ internals](https://github.com/GLolol/PyLink/blob/0.4.0-dev/classes.py#L267-L272)
 
 - **`nick`**`(self, source, newnick)` - Changes the nick of a PyLink client.
 
-- **`notice`**`(self, source, target, text)` - Sends a NOTICE from a PyLink client.
+- **`oper_notice`**`(self, source, target)` - Sends a notice to all operators on the network.
 
-- **`numeric`**`(self, source, numeric, target, text)` - Sends a raw numeric `numeric` with `text` from the `source` server to `target`.
+- **`notice`**`(self, source, target, text)` - Sends a NOTICE from a PyLink client or server.
+
+- **`numeric`**`(self, source, numeric, target, text)` - Sends a raw numeric `numeric` with `text` from the `source` server to `target`. This should raise `NotImplementedError` if not supported on a protocol.
 
 - **`part`**`(self, client, channel, reason=None)` - Sends a part from a PyLink client.
 
@@ -71,79 +96,205 @@ internals](https://github.com/GLolol/PyLink/blob/0.4.0-dev/classes.py#L267-L272)
 - **`sjoin`**`(self, server, channel, users, ts=None, modes=set())` - Sends an SJOIN for a group of users to a channel. The sender should always be a Server ID (SID). TS is
 optional, and defaults to the one we've stored in the channel state if not given. `users` is a list of `(prefix mode, UID)` pairs. Example uses:
     - `sjoin('100', '#test', [('', '100AAABBC'), ('qo', 100AAABBB'), ('h', '100AAADDD')])`
-    - `sjoin(self.irc.sid, '#test', [('o', self.irc.pseudoclient.uid)])`
+    - `sjoin(self.sid, '#test', [('o', self.pseudoclient.uid)])`
 
-- **`spawnServer`**`(self, name, sid=None, uplink=None, desc=None)` - Spawns a server off another PyLink server. `desc` (server description) defaults to the one in the config. `uplink` defaults to the main PyLink server, and `sid` (the server ID) is automatically generated if not given. Sanity checks for server name and SID validity ARE done by the protocol module here.
+- **`spawn_server`**`(self, name, sid=None, uplink=None, desc=None)` - Spawns a server off another PyLink server. `desc` (server description) defaults to the one in the config. `uplink` defaults to the main PyLink server, and `sid` (the server ID) is automatically generated if not given. Sanity checks for server name and SID validity ARE done by the protocol module here.
 
 - **`squit`**`(self, source, target, text='No reason given')` - SQUITs a PyLink server.
 
-- **`topic`**`(self, source, target, text)` - Sends a topic change from a PyLink client.
+- **`topic`**`(self, source, target, text)` - Sends a topic change from a PyLink *client.
 
-- **`topicBurst`**`(self, source, target, text)` - Sends a topic change from a PyLink server. This is usually used on burst.
+- **`topic_burst`**`(self, source, target, text)` - Sends a topic change from a PyLink server. This is usually used on burst.
 
-- **`updateClient`**`(self, source, field, text)` - Updates the ident, host, or realname of a PyLink client. `field` should be either "IDENT", "HOST", "GECOS", or
+- **`update_client`**`(self, source, field, text)` - Updates the ident, host, or realname of a PyLink client. `field` should be either "IDENT", "HOST", "GECOS", or "REALNAME". If changing the field given on the IRCd isn't supported, `NotImplementedError` should be raised.
-"REALNAME". If changing the field given on the IRCd isn't supported, `NotImplementedError` should be raised.
 
-## Things to note
+## Special variables
 
-### Special variables
+A protocol module should also set the following variables in each instance:
 
-A protocol module should also set the following variables in their protocol class:
+- `self.casemapping`: a string (`'rfc1459'` or `'ascii'`) to determine which case mapping the IRCd uses.
-
+- `self.hook_map`: this is a `dict`, which maps non-standard command names sent by the IRCd to those used by [PyLink hooks](hooks-reference.md).
-- `self.casemapping`: set this to `rfc1459` (default) or `ascii` to determine which case mapping the IRCd uses.
+    - Examples exist in the [UnrealIRCd](https://github.com/jlu5/PyLink/blob/1.0-beta1/protocols/unreal.py#L24-L27) and [InspIRCd](https://github.com/jlu5/PyLink/blob/1.0-beta1/protocols/inspircd.py#L25-L28) modules.
-- `self.hook_map`: this is a `dict`, which maps non-standard command names sent by the IRCd to those that PyLink plugins use internally.
+- `self.conf_keys`: a set of strings determining which server configuration options a protocol module needs to function; see the [Configuration key validation](#configuration-key-validation) section below.
-    - Examples exist in the [UnrealIRCd](https://github.com/GLolol/PyLink/blob/0.5-dev/protocols/unreal.py#L22) and [InspIRCd](https://github.com/GLolol/PyLink/blob/0.5-dev/protocols/inspircd.py#L24) modules.
+- `self.cmodes` / `self.umodes`: These are mappings of named IRC modes (e.g. `inviteonly` or `moderated`) to a string list of mode letters, that should be either set during link negotiation or hardcoded into the protocol module. There are also special keys: `*A`, `*B`, `*C`, and `*D`, which **must** be set properly with a list of mode characters for that type of mode.
-- `self.cmodes` / `self.umodes`: These are mappings of named IRC modes to mode letters, that should be either negotiated during link or preset in the `connect()` function of the protocol module. There are also special keys: `*A`, `*B`, `*C`, and `*D`, which should each be filled with a list of mode characters for that type of modes.
     - Types of modes are defined as follows (from http://www.irc.org/tech_docs/005.html):
         - A = Mode that adds or removes a nick or address to a list. Always has a parameter.
         - B = Mode that changes a setting and always has a parameter.
         - C = Mode that changes a setting and only has a parameter when set.
         - D = Mode that changes a setting and never has a parameter.
-    - Examples in the TS6 protocol module: https://github.com/GLolol/PyLink/blob/cb3187c/protocols/ts6.py#L259-L300
+    - If not defined, these will default to modes defined by RFC 1459: https://github.com/jlu5/PyLink/blob/1.0-beta1/classes.py#L127-L152
-    - If not defined, these will default to modes defined by RFC 1459: https://github.com/GLolol/PyLink/blob/cb3187c/classes.py#L118-L152
+    - An example of mode mapping hardcoding can be found here: https://github.com/jlu5/PyLink/blob/1.0-beta1/protocols/ts6.py#L259-L311
+    - You can find a list of supported (named) channel modes [here](channel-modes.csv), and a list of user modes [here](user-modes.csv).
 - `self.prefixmodes`: This defines a mapping of prefix modes (+o, +v, etc.) to their respective mode prefix. This will default to `{'o': '@', 'v': '+'}` (the standard op and voice) if not defined.
     - Example: `self.prefixmodes = {'o': '@', 'h': '%', 'v': '+'}`
+- `self.connected`: this is a `threading.Event` object that plugins use to determine if the network has finished bursting. Protocol modules should set this to True via `self.connected.set()` when ready.
 
-### Topics
+## PyLink Protocol capabilities
+PyLink 1.2 introduced the concept of protocol-defined capabilities, so that plugins wishing to use IRCd-specific features don't have to hard code protocol modules by name. Protocol capabilities are defined in `self.protocol_caps` (a set of strings) and may be changed freely before `self.connected` is set. Individual capabilities are then checked by plugins via `irc.has_cap(capability_name)`.
 
-When receiving or sending topics, there is a `topicset` attribute in the IRC channel (IrcChannel) object that should be set **True**. It simply denotes that a topic has been set in the channel at least once.
+As of writing, the following protocol capabilities (case-sensitive) are implemented:
 
-(Relay uses this so it doesn't overwrite topics with empty ones during burst, when a relay channel initialize before the uplink has sent the topic for it)
+### Supported protocol capabilities
+- `can-host-relay` - whether servers using this protocol can host a relay channel (for sanity reasons, this should be off for anything that's not IRC S2S)
+- `can-manage-bot-channels` - whether PyLink can manage which channels the bot itself is in. This is off for platforms such as Discord.
+- `can-spawn-clients` - determines whether any spawned clients are real or virtual (mainly for `services_support`).
+- `can-track-servers` - determines whether servers are accurately tracked (for `servermaps` and other statistics)
+- `freeform-nicks` - if set, nicknames for PyLink's virtual clients are not subject to validity and nick collision checks. This implies the `slash-in-nicks` capability.
+    - Note: PyLink already allows incoming nicks to be freeform, provided they are encoded correctly and don't cause parsing conflicts (i.e. containing reserved chars on IRC)
+- `has-irc-modes` - whether IRC style modes are supported
+- `has-statusmsg` - whether STATUSMSG messages (e.g. `@#channel`) are supported
+- `has-ts` - determines whether channel and user timestamps are tracked (and not spoofed)
+- `slash-in-hosts` - determines whether `/` is allowed in hostnames
+- `slash-in-nicks` - determines whether `/` is allowed in nicks
+- `ssl-should-verify` - determines whether TLS certificates should be checked for validity by default - this should be enabled for any protocol modules needing to verify a remote server (e.g. Clientbot or a non-IRC API endpoint), and disabled for most IRC S2S links (where self-signed certs are widespread)
+- `underscore-in-hosts` - determines whether `_` is allowed in client hostnames (yes, support for this actually varies by IRCd)
+- `virtual-server` - marks the server as virtual, i.e. controlled by protocol module under a different server. Virtual servers are ignored by `rehash` and `disconnect` in the `networks` plugin.
+    - This is used by pylink-discord as of v0.2.0.
+- `visible-state-only` - determines whether channels should be autocleared when the PyLink client leaves (for clientbot, etc.)
+    - Note: enabling this in a protocol module lets `coremods/handlers` automatically clean up old channels for you!
+
+New protocol capabilities are generally added when needed - see https://github.com/jlu5/PyLink/issues/620
+
+### Abstraction defaults
+
+For reference, the `IRCS2SProtocol` class defines the following by default:
+- `can-host-relay`
+- `can-spawn-clients`
+- `can-track-servers`
+- `has-ts`
+
+Whereas `PyLinkNetworkCore` defines no capabilities (i.e. an empty set) by default.
+
+## PyLink structures
+In this section, `self` refers to the network object/protocol module instance itself (i.e. from its own perspective).
+
+### Server, User, Channel classes
+PyLink defines classes named `Server`, `User`, and `Channel` in the `classes` module, and stores dictionaries of these in the `servers`, `users`, and `channels` attributes of a protocol object respectively.
+
+- `self.servers` is a dictionary mapping server IDs (SIDs) to `Server` objects. If a protocol module does not use SIDs, servers are stored by server name instead.
+
+- `self.users` is a dictionary mapping user IDs (UIDs) to `User` objects. If a protocol module does not use UIDs, a pseudo UID (PUID) generator such as [`classes.PUIDGenerator`](https://github.com/jlu5/PyLink/blob/3922d44173593e4bcceae1218bbc6f267caa9fc1/classes.py#L1710-L1726) *must* be used instead.
+    - The rationale behind this is because plugins tracking user lists are not designed to remove and re-add users when they change their nicks.
+    - When sending text back to the protocol module, it may be helpful to use the [`_expandPUID()`](https://github.com/jlu5/PyLink/blob/4a363aee509c5a0488a38b9e60f93ec59a274c3c/classes.py#L1213-L1231) function in `PyLinkNetworkCoreWithUtils` to expand these pseudo-UIDs back to regular nicks.
+
+- `self._channels` and `self.channels` are [IRC case-insensitive dictionaries](https://github.com/jlu5/PyLink/blob/4a363aee509c5a0488a38b9e60f93ec59a274c3c/structures.py#L114-L116) mapping channel names to Channel objects.
+    - The key difference between these two dictionaries is that `_channels` is powered by `classes.ChannelState` and creates new channels *automatically* when they are accessed by index. This makes writing protocol modules easier, as they can assume that the channels they wish to modify always exist (no chance of `KeyError`!).
+    - `self.channels`, on the other hand, does *not* implicitly create channels and is thus better suited for plugins.
+
+The `Channel`, `User`, and `Server` classes are initiated as follows:
+
+- `Channel(self, name)` - First arg is the protocol object, second is the channel name.
+- `User(self, nick, ts, uid, server, ident='null', host='null', realname='PyLink dummy client', realhost='null', ip='0.0.0.0', manipulatable=False, opertype='IRC Operator')` - These arguments are essentially the same as `spawn_client()`'s.
+- `Server(self, uplink, name, internal=False, desc="(None given)")`
+    - The `uplink` (type `str`) option sets the SID of the uplink server, or *None* for both the main PyLink server and its uplink.
+    - The `name` option sets the server name.
+    - The `internal` boolean sets whether the server is an internal PyLink server.
+    - The `desc` option sets the server description, when applicable.
+
+#### Statekeeping specifics
+- When a user is introduced, their UID must be added to both `self.users` and to the `users` set in the `Server` object hosting the user (`self.servers[SID].users`). The latter list is used internally to track SQUITs.
+- When a user joins a channel, the channel name is added to the User object's `channels` set (`self.users[UID].channels`), as well as the Channel object's user list (`self.channels[CHANNELNAME].users`)
+- When a user disconnects, the `_remove_client` helper method can be called on their UID to automatically remove them from the relevant Server object, as well as all channels they were in.
+- When a user leaves a channel, the `Channel.remove_user()` method can be used to easily remove them from the channel state, and vice versa.
 
 ### Mode formats
 
-Modes are stored a special format in PyLink, different from raw mode strings in order to make them easier to parse. Mode strings can be turned into mode *lists*, which are used to represent mode changes in hooks, and when storing modes internally.
+Modes are stored not stored as strings, but lists of mode pairs in order to ease parsing. These lists of mode pairs are used both to represent mode changes in hooks and store modes internally.
 
-`utils.parseModes(irc, target, split_modestring)` is used to convert mode strings to mode lists, where `irc` is the IRC object, `target` is the channel name or UID the mode is being set on, and `split_modestring` is the string of modes to parse, *split at each space* (meaning that it's really a list).
+`self.parse_modes(target, modestring)` is used to convert mode strings to mode lists. `target` is the channel name/UID the mode is being set on, while `modestring` takes either a string or string split by spaces (really a list).
 
-- `utils.parseModes(irc, '#chat', ['+tHIs', '*!*@is.sparta'])` would give:
+- `self.parse_modes('#chat', ['+tHIs', '*!*@is.sparta'])` would give:
     - `[('+t', None), ('+H', None), ('+I', '*!*@is.sparta'), ('+s', None)]`
 
-Also, `parseModes` will automatically convert prefix mode targets from nicks to UIDs, and drop invalid modes settings.
+`parse_modes()` will also automatically convert prefix mode targets from nicks to UIDs, and drop any duplicate (already set) or invalid (e.g. missing argument) modes.
 
-- `utils.parseModes(irc, '#chat', ['+ol', 'invalidnick'])`:
+- `self.parse_modes('#chat', ['+ol invalidnick'])`:
     - `[]`
-- `utils.parseModes(irc, '#chat', ['+o', 'GLolol'])`:
+- `self.parse_modes('#chat', ['+o jlu5'])`:
     - `[('+o', '001ZJZW01')]`
 
-Then, a parsed mode list can be applied to channel name or UID using `utils.applyModes(irc, target, parsed_modelist)`.
+Afterwords, a parsed mode list can be applied to channel name or UID using `self.apply_modes(target, parsed_modelist)`.
 
-Internally, modes are stored in channel and user objects as sets: `(userobj or chanobj).modes`:
+**Note**: for protocols that accept or reject mode changes based on TS (i.e. practically every IRCd), you will want to use [`updateTS(...)`](https://github.com/jlu5/PyLink/blob/master/classes.py#L1484-L1487) instead to only apply the modes if the source TS is lower.
+
+Internally, modes are stored in `Channel` and `User` objects as sets, **with the `+` prefixing each mode character omitted**. These sets are accessed via the `modes` attribute:
 
 ```
-<+GLolol> PyLink-devel, eval irc.users[source].modes
+<+jlu5> PyLink-devel, eval irc.users[source].modes
 <@PyLink-devel> {('i', None), ('x', None), ('w', None), ('o', None)}
-<+GLolol> PyLink-devel, eval irc.channels['#chat'].modes
+<+jlu5> PyLink-devel, eval irc.channels['#chat'].modes
 <@PyLink-devel> {('n', None), ('t', None)}
 ```
 
-*With the exception of channel prefix modes* (op, voice, etc.), which are stored as a dict of sets in `chanobj.prefixmodes`:
+**Exception**: the owner, admin, op, halfop, and voice channel prefix modes are stored separately as a dict of sets in `Channel.prefixmodes`:
 
 ```
-<@GLolol> PyLink-devel, eval irc.channels['#chat'].prefixmodes
+<@jlu5> PyLink-devel, eval irc.channels['#chat'].prefixmodes
 <+PyLink-devel> {'op': set(), 'halfop': set(), 'voice': {'38QAAAAAA'}, 'owner': set(), 'admin': set()}
 ```
 
 When a certain mode (e.g. owner) isn't supported on a network, the key still exists in `prefixmodes` but is simply unused.
 
-You can see a list of supported (named) channel modes [here](channel-modes.csv), and a list of user modes [here](user-modes.csv).
+### Topics
+
+When receiving or sending topics, there is a `topicset` attribute in the `Channel` object that should be set to **True**. This boolean denotes that a topic has been set in the channel at least once; Relay uses it to know not to overwrite topics with empty ones during startup, when topics have not been received from all networks yet.
+
+*Caveat:* Topic handlers on the current protocol modules do not follow TS rules (which vary by IRCd), and blindly accept data. See issue https://github.com/jlu5/PyLink/issues/277
+
+## Configuration key validation
+
+Starting with PyLink 1.x, protocol modules can specify which config values within a server block they need in order to work. This is done by adjusting the `self.conf_keys` attribute, usually in the protocol module's `__init__()` method. The default set, defined in [`Classes.Protocol`](https://github.com/jlu5/PyLink/blob/1.0-beta1/classes.py#L1202-L1204), includes `{'ip', 'port', 'hostname', 'sid', 'sidrange', 'protocol', 'sendpass', 'recvpass'}`. Should any of these keys be missing from a server block, PyLink will bail with a configuration error.
+
+As an example, one protocol module that tweaks this is [`Clientbot`](https://github.com/jlu5/PyLink/blob/1.0-beta1/protocols/clientbot.py#L17-L18), which removes all options except `ip`, `protocol`, and `port`.
+
+## The final checklist
+
+In short, protocol modules have some very important jobs. If any of these aren't done correctly, you will be left with a very broken, desynced services server:
+
+1) Handle incoming commands from the uplink.
+
+2) Return [hook data](hooks-reference.md) for relevant commands, so that plugins can receive data from the uplink.
+
+3) Make sure channel/user states are kept correctly. Joins, quits, parts, kicks, mode changes, nick changes, etc. should all be handled accurately where relevant.
+
+4) Implement the specified outgoing command functions, which are used by plugins to send commands to the uplink.
+
+5) Set the `threading.Event` instance `self.connected` to True (via `self.connected.set()`) when the connection with the uplink is fully established. This is important for Relay and the services API, which will refuse to initialize if the connection is not marked ready.
+
+6) Check that `recvpass` is correct when applicable, and raise `ProtocolError` with a relevant error message if not.
+
+7) Declare the correct set of protocol module capabilities to prevent confusing PyLink's plugins.
+
+## Changes to this document
+* 2021-06-15 (3.1-dev)
+   - Added `oper_notice()` function to send notices to opers (GLOBOPS / OPERWALL on most IRCds)
+   - Update notes about non-IRC protocols and PyLinkNetworkCoreWithUtils
+* 2019-11-02 (2.1-beta1)
+   - Added protocol capability: `can-manage-bot-channels`
+* 2019-10-10 (2.1-beta1)
+   - Added protocol capability: `has-irc-modes`
+* 2019-06-23 (2.1-alpha2)
+   - Added new protocol capabilities: `virtual-server` and `freeform-nicks`
+* 2018-07-11 (2.0.0)
+   - Version bump for 2.0 stable release; no meaningful content changes.
+* 2018-06-26 (2.0-beta1)
+   - Added documentation for PyLink protocol capabilities
+   - Wording tweaks, restructured headings
+   - Consistently refer to protocol module attributes as `self.<whatever>` instead of `irc.<whatever>`
+* 2018-05-09 (2.0-alpha3)
+   - `kill` and `kick` implementations should raise `NotImplementedError` if not supported (anti-desync measure).
+   - Future PyLink versions will further standardize which functions should be stubbed (no-op) when not available and which should raise an error.
+* 2017-10-05 (2.0-alpha1)
+   - Added notes on user statekeeping and the tracking/helper functions used.
+   - Mention the `post_connect()` function that must be defined by protocols inheriting from IRCNetwork.
+* 2017-08-30 (2.0-dev)
+   - Rewritten specification for the IRC-protocol class convergence in PyLink 2.0.
+   - Updated the spec for 2.0 method renames and class restructures.
+   - Added a proper "Starting Steps" section detailing which new classes inherit from and when.
+   - Explicitly document the Server, User, and Channel classes.
+* 2017-03-15 (1.2-dev)
+   - Corrected the location of `self.cmodes/umodes/prefixmodes` attributes
+   - Mention `self.conf_keys` as a special variable for completeness
+* 2017-01-29 (1.2-dev)
+   - NOTICE can now be sent from servers.
+   - This section was added.

docs/technical/protocol-modules.dot

@@ -1,28 +1,47 @@
-/* Graph showing inheritance with the current PyLink protocol protocols:
+/* Graph showing inheritance with the current PyLink protocol modules:
- * Update using: dot -Tpng protocol-modules.dot > protocol-modules.png
+ * Update using: dot -Tsvg protocol-modules.dot > protocol-modules.svg
  */
 
 digraph G {
-	ratio = 0.8; /* make the graph wider than tall */
+
+	edge [ penwidth=0.75, color="#111111CC" ];
+	subgraph cluster_core {
+		label="Core classes (pylinkirc.classes)";
+		style="filled";
+		node [style="filled",color="white"];
+		color="#90EE90";
+
+		"PyLinkNetworkCore" -> "PyLinkNetworkCoreWithUtils" -> "IRCNetwork";
+	}
+
 	subgraph cluster_helper {
-		label="Protocol module helpers";
+		label="Protocol module helpers\n(pylinkirc.protocols.ircs2s_common)";
 		style="filled";
 		node [style="filled",color="white"];
 		color="lightblue";
 
-		"ircs2s_common.py" -> "ts6_common.py";
+		"IRCNetwork" -> "IRCCommonProtocol" -> "IRCS2SProtocol" -> "TS6BaseProtocol";
+
+		subgraph cluster_helper {
+			label="pylinkirc.protocols.ts6_common";
+			style="filled";
+			color="lightcyan";
+
+			"TS6BaseProtocol";
+		}
 	}
 
 	subgraph cluster_pluggable {
-		label="Pluggable (full) protocol modules";
+		label="Complete protocol modules (pylinkirc.protocols.*)";
 		style="filled";
 		node [style="filled",color="white"];
 		color="khaki";
 
-		"ircs2s_common.py" -> "nefarious.py";
+		"IRCS2SProtocol" -> "p10";
-		"ts6_common.py" -> "ts6.py" -> "hybrid.py";
+		"IRCS2SProtocol" -> "ngircd";
-		"ts6_common.py" -> "inspircd.py";
+		"TS6BaseProtocol" -> "ts6" -> "hybrid";
-		"ts6_common.py" -> "unreal.py";
+		"TS6BaseProtocol" -> "inspircd";
-		"clientbot.py";
+		"TS6BaseProtocol" -> "unreal";
+		"IRCCommonProtocol" -> "clientbot";
 	}
 }

docs/technical/protocol-modules.svg

@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.38.0 (20140413.2041)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="530pt" height="651pt"
+ viewBox="0.00 0.00 530.00 651.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 647)">
+<title>G</title>
+<polygon fill="white" stroke="none" points="-4,4 -4,-647 526,-647 526,4 -4,4"/>
+<g id="clust1" class="cluster"><title>cluster_core</title>
+<polygon fill="#90ee90" stroke="#90ee90" points="77,-416 77,-635 333,-635 333,-416 77,-416"/>
+<text text-anchor="middle" x="205" y="-619.8" font-family="Times,serif" font-size="14.00">Core classes (pylinkirc.classes)</text>
+</g>
+<g id="clust2" class="cluster"><title>cluster_helper</title>
+<polygon fill="lightblue" stroke="lightblue" points="83,-163 83,-408 302,-408 302,-163 83,-163"/>
+<text text-anchor="middle" x="192.5" y="-392.8" font-family="Times,serif" font-size="14.00">Protocol module helpers</text>
+<text text-anchor="middle" x="192.5" y="-377.8" font-family="Times,serif" font-size="14.00">(pylinkirc.protocols.ircs2s_common)</text>
+</g>
+<g id="clust3" class="cluster"><title>cluster_helper</title>
+<polygon fill="lightcyan" stroke="lightcyan" points="91,-171 91,-246 285,-246 285,-171 91,-171"/>
+<text text-anchor="middle" x="188" y="-230.8" font-family="Times,serif" font-size="14.00">pylinkirc.protocols.ts6_common</text>
+</g>
+<g id="clust4" class="cluster"><title>cluster_pluggable</title>
+<polygon fill="khaki" stroke="khaki" points="8,-8 8,-155 514,-155 514,-8 8,-8"/>
+<text text-anchor="middle" x="261" y="-139.8" font-family="Times,serif" font-size="14.00">Complete protocol modules (pylinkirc.protocols.*)</text>
+</g>
+<!-- PyLinkNetworkCore -->
+<g id="node1" class="node"><title>PyLinkNetworkCore</title>
+<ellipse fill="white" stroke="white" cx="205" cy="-586" rx="84.485" ry="18"/>
+<text text-anchor="middle" x="205" y="-582.3" font-family="Times,serif" font-size="14.00">PyLinkNetworkCore</text>
+</g>
+<!-- PyLinkNetworkCoreWithUtils -->
+<g id="node2" class="node"><title>PyLinkNetworkCoreWithUtils</title>
+<ellipse fill="white" stroke="white" cx="205" cy="-514" rx="119.679" ry="18"/>
+<text text-anchor="middle" x="205" y="-510.3" font-family="Times,serif" font-size="14.00">PyLinkNetworkCoreWithUtils</text>
+</g>
+<!-- PyLinkNetworkCore&#45;&gt;PyLinkNetworkCoreWithUtils -->
+<g id="edge1" class="edge"><title>PyLinkNetworkCore&#45;&gt;PyLinkNetworkCoreWithUtils</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M205,-567.697C205,-559.983 205,-550.712 205,-542.112"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="208.5,-542.104 205,-532.104 201.5,-542.104 208.5,-542.104"/>
+</g>
+<!-- IRCNetwork -->
+<g id="node3" class="node"><title>IRCNetwork</title>
+<ellipse fill="white" stroke="white" cx="205" cy="-442" rx="55.7903" ry="18"/>
+<text text-anchor="middle" x="205" y="-438.3" font-family="Times,serif" font-size="14.00">IRCNetwork</text>
+</g>
+<!-- PyLinkNetworkCoreWithUtils&#45;&gt;IRCNetwork -->
+<g id="edge2" class="edge"><title>PyLinkNetworkCoreWithUtils&#45;&gt;IRCNetwork</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M205,-495.697C205,-487.983 205,-478.712 205,-470.112"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="208.5,-470.104 205,-460.104 201.5,-470.104 208.5,-470.104"/>
+</g>
+<!-- IRCCommonProtocol -->
+<g id="node4" class="node"><title>IRCCommonProtocol</title>
+<ellipse fill="white" stroke="white" cx="205" cy="-344" rx="89.0842" ry="18"/>
+<text text-anchor="middle" x="205" y="-340.3" font-family="Times,serif" font-size="14.00">IRCCommonProtocol</text>
+</g>
+<!-- IRCNetwork&#45;&gt;IRCCommonProtocol -->
+<g id="edge3" class="edge"><title>IRCNetwork&#45;&gt;IRCCommonProtocol</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M205,-423.837C205,-409.503 205,-388.807 205,-372.216"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="208.5,-372.014 205,-362.014 201.5,-372.014 208.5,-372.014"/>
+</g>
+<!-- IRCS2SProtocol -->
+<g id="node5" class="node"><title>IRCS2SProtocol</title>
+<ellipse fill="white" stroke="white" cx="214" cy="-272" rx="69.5877" ry="18"/>
+<text text-anchor="middle" x="214" y="-268.3" font-family="Times,serif" font-size="14.00">IRCS2SProtocol</text>
+</g>
+<!-- IRCCommonProtocol&#45;&gt;IRCS2SProtocol -->
+<g id="edge4" class="edge"><title>IRCCommonProtocol&#45;&gt;IRCS2SProtocol</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M207.225,-325.697C208.217,-317.983 209.408,-308.712 210.514,-300.112"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="213.997,-300.469 211.801,-290.104 207.054,-299.576 213.997,-300.469"/>
+</g>
+<!-- clientbot -->
+<g id="node13" class="node"><title>clientbot</title>
+<ellipse fill="white" stroke="white" cx="464" cy="-106" rx="41.6928" ry="18"/>
+<text text-anchor="middle" x="464" y="-102.3" font-family="Times,serif" font-size="14.00">clientbot</text>
+</g>
+<!-- IRCCommonProtocol&#45;&gt;clientbot -->
+<g id="edge12" class="edge"><title>IRCCommonProtocol&#45;&gt;clientbot</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M233.699,-326.923C251.178,-316.837 273.771,-303.27 293,-290 319.052,-272.023 326.338,-268.098 349,-246 386.799,-209.142 424.679,-160.559 446.146,-131.67"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="449.147,-133.498 452.263,-123.372 443.512,-129.345 449.147,-133.498"/>
+</g>
+<!-- TS6BaseProtocol -->
+<g id="node6" class="node"><title>TS6BaseProtocol</title>
+<ellipse fill="white" stroke="white" cx="187" cy="-197" rx="72.2875" ry="18"/>
+<text text-anchor="middle" x="187" y="-193.3" font-family="Times,serif" font-size="14.00">TS6BaseProtocol</text>
+</g>
+<!-- IRCS2SProtocol&#45;&gt;TS6BaseProtocol -->
+<g id="edge5" class="edge"><title>IRCS2SProtocol&#45;&gt;TS6BaseProtocol</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M207.738,-254.069C204.49,-245.287 200.448,-234.36 196.798,-224.49"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="200.073,-223.257 193.322,-215.092 193.508,-225.685 200.073,-223.257"/>
+</g>
+<!-- p10 -->
+<g id="node7" class="node"><title>p10</title>
+<ellipse fill="white" stroke="white" cx="293" cy="-106" rx="27" ry="18"/>
+<text text-anchor="middle" x="293" y="-102.3" font-family="Times,serif" font-size="14.00">p10</text>
+</g>
+<!-- IRCS2SProtocol&#45;&gt;p10 -->
+<g id="edge6" class="edge"><title>IRCS2SProtocol&#45;&gt;p10</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M269.936,-261.101C277.361,-257.496 284.089,-252.611 289,-246 313.082,-213.582 307.4,-164.181 300.569,-133.833"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="303.928,-132.834 298.152,-123.952 297.129,-134.497 303.928,-132.834"/>
+</g>
+<!-- ngircd -->
+<g id="node8" class="node"><title>ngircd</title>
+<ellipse fill="white" stroke="white" cx="371" cy="-106" rx="33.2948" ry="18"/>
+<text text-anchor="middle" x="371" y="-102.3" font-family="Times,serif" font-size="14.00">ngircd</text>
+</g>
+<!-- IRCS2SProtocol&#45;&gt;ngircd -->
+<g id="edge7" class="edge"><title>IRCS2SProtocol&#45;&gt;ngircd</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M276.712,-264.047C289.461,-260.278 302.009,-254.58 312,-246 345.593,-217.151 360.541,-165.861 366.824,-134.301"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="370.276,-134.882 368.637,-124.415 363.391,-133.62 370.276,-134.882"/>
+</g>
+<!-- ts6 -->
+<g id="node9" class="node"><title>ts6</title>
+<ellipse fill="white" stroke="white" cx="43" cy="-106" rx="27" ry="18"/>
+<text text-anchor="middle" x="43" y="-102.3" font-family="Times,serif" font-size="14.00">ts6</text>
+</g>
+<!-- TS6BaseProtocol&#45;&gt;ts6 -->
+<g id="edge8" class="edge"><title>TS6BaseProtocol&#45;&gt;ts6</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M137.801,-183.792C118.234,-177.355 96.3078,-168.023 79,-155 70.5984,-148.678 63.2254,-139.88 57.3761,-131.54"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="60.2708,-129.572 51.8527,-123.138 54.4215,-133.417 60.2708,-129.572"/>
+</g>
+<!-- inspircd -->
+<g id="node11" class="node"><title>inspircd</title>
+<ellipse fill="white" stroke="white" cx="209" cy="-106" rx="38.9931" ry="18"/>
+<text text-anchor="middle" x="209" y="-102.3" font-family="Times,serif" font-size="14.00">inspircd</text>
+</g>
+<!-- TS6BaseProtocol&#45;&gt;inspircd -->
+<g id="edge10" class="edge"><title>TS6BaseProtocol&#45;&gt;inspircd</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M191.242,-178.84C194.377,-166.158 198.707,-148.64 202.307,-134.077"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="205.769,-134.655 204.771,-124.107 198.974,-132.975 205.769,-134.655"/>
+</g>
+<!-- unreal -->
+<g id="node12" class="node"><title>unreal</title>
+<ellipse fill="white" stroke="white" cx="120" cy="-106" rx="32.4942" ry="18"/>
+<text text-anchor="middle" x="120" y="-102.3" font-family="Times,serif" font-size="14.00">unreal</text>
+</g>
+<!-- TS6BaseProtocol&#45;&gt;unreal -->
+<g id="edge11" class="edge"><title>TS6BaseProtocol&#45;&gt;unreal</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M174.398,-179.26C164.099,-165.579 149.413,-146.071 137.895,-130.772"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="140.682,-128.653 131.871,-122.769 135.089,-132.863 140.682,-128.653"/>
+</g>
+<!-- hybrid -->
+<g id="node10" class="node"><title>hybrid</title>
+<ellipse fill="white" stroke="white" cx="50" cy="-34" rx="33.5952" ry="18"/>
+<text text-anchor="middle" x="50" y="-30.3" font-family="Times,serif" font-size="14.00">hybrid</text>
+</g>
+<!-- ts6&#45;&gt;hybrid -->
+<g id="edge9" class="edge"><title>ts6&#45;&gt;hybrid</title>
+<path fill="none" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" d="M44.7303,-87.6966C45.5017,-79.9827 46.4288,-70.7125 47.2888,-62.1124"/>
+<polygon fill="#111111" fill-opacity="0.800000" stroke="#111111" stroke-width="0.75" stroke-opacity="0.800000" points="50.7771,-62.403 48.2896,-52.1043 43.8118,-61.7064 50.7771,-62.403"/>
+</g>
+</g>
+</svg>

docs/technical/release-process.md

@@ -0,0 +1,13 @@
+# Release Process for PyLink
+
+This document documents the steps that I (James) use to release updates to PyLink.
+
+1) Draft the next release's changelog in `RELNOTES.md`
+
+2) Bump the version in the [`VERSION`](VERSION) file.
+
+3) Commit the changes to `VERSION` and `RELNOTES.md`, and tag+sign this commit as the new release. Do not prefix version numbers with "v".
+
+4) Publish the release via the GitHub release page, using the same changelog content as `RELNOTES.md`.
+
+5) For stable releases, ~~also upload to PyPI: `python3 setup.py sdist upload`~~ PyPI uploads are handled automatically via Travis-CI.

docs/technical/services-api.md

@@ -1,10 +1,10 @@
 # PyLink Services Bot API
 
-Starting with PyLink 0.9.x, a services bot API was introduced to make writing custom services slightly easier. PyLink's Services API automatically connects service bots, and handles rejoin on kick/kill all by itself, meaning less code is needed per plugin to have functional service bots.
+The goal of PyLink's Services API is to make writing custom services easier. It is able to automatically spawn service bots on connect, handle rejoins on kill and kick, and expose a way for plugins to bind commands to various services bots. It also handles U-line servprotect modes when enabled and supported on a particular network (i.e. the `protect_services` option).
 
-## Creating new services
+## Basic service creation
 
-Services can be created (registered) using code similar to the following in a plugin:
+Services can be registered and created using code similar to the following in a plugin:
 
 ```python
 
@@ -14,40 +14,67 @@ from pylinkirc import utils, world
 desc = "Optional description of servicenick, in sentence form."
 
 # First argument is the internal service name.
-# utils.registerService() returns a utils.ServiceBot instance, which can also be found
+# utils.register_service() returns a utils.ServiceBot instance, which is also stored
-# by calling world["myservice"].
+# as world.services['myservice'].
-myservice = utils.registerService("myservice", desc=desc)
+myservice = utils.register_service('myservice', desc=desc)
 ```
 
-`utils.registerService()` passes its arguments directly to the `utils.ServiceBot` class constructor, which in turn supports the following options:
+`utils.register_service()` passes its arguments directly to the `utils.ServiceBot` class constructor, which in turn supports the following options:
 
 - **`name`** - defines the service name (mandatory)
 - `default_help` - Determines whether the default HELP command should be used for the service. Defaults to True.
 - `default_list` - Determines whether the default LIST command should be used for the service. Defaults to True.
-- `nick`, `ident` - Sets the default nick and ident for the service bot. If not given, these simply default to the service name.
+- `default_nick` - Sets the default nick this service should use if the user doesn't provide it. Defaults to the same as the service name.
 - `manipulatable` - Determines whether the bot is marked manipulatable. Only manipulatable clients can be force joined, etc. using PyLink commands. Defaults to False.
-- `extra_channels` - Defines a dict mapping network names to a set of channels that the bot should autojoin on that network.
 - `desc` - Sets the command description of the service. This is shown in the default HELP command if enabled.
 
+**NOTE**: It is convention for the service name in `utils.register_service('SERVICE')` to match your plugin name, as the services API implicitly loads [configuration options](../advanced-services-config.md) from config blocks named `SERVICE:` (which you may want to put plugin options in as well).
+
+Implementation note: if the `spawn_service` option is disabled (either globally or for your service bot), `register_service` will return the main PyLink `ServiceBot` instance (i.e. `world.services['pylink']`), which you can modify as usual. `unregister_service` calls to your service name will be silently ignored as no `ServiceBot` instance is actually registered for that name. Altogether, this allows service-spawning plugins to function normally regardless of the `spawn_service` value.
+
 ### Getting the UID of a bot
 
-Should you want to get the UID of a service bot on a specific server, use `myservice.uids.get('irc.name')`
+To obtain the UID of a service bot on a specific network, use `myservice.uids.get(irc.name)` (where `irc` is the network object).
 
-### Setting channels to join
+### Removing services on unload
 
-All services bots wil automatically join the autojoin channels configured for a specific network, if any.
+All plugins using the services API should have a `die()` function that unregisters all service bots that they've created. A simple example would be in the `games` plugin:
-
-However, plugins can modify the autojoin entries of a specific bot by adding items to the `myservice.extra_channels` channel set. After sending `irc.proto.join(...)` using the service bot's UID as a source, the bot should permanently remain on that channel throughout KILLs or disconnects.
-
-## Removing services on unload
-
-All plugins using the services API **MUST** have a `die()` function that unregisters all services that they've created. A simple example would be in the `games` plugin:
 
 ```python
-def die(irc):
+def die(irc=None):
-    utils.unregisterService('games')
+    utils.unregister_service('games')
 ```
 
+Should your service bot define any persistent channels, you will also want to clear them on unload via `myservice.clear_persistent_channels(None, 'your-namespace', ...)`
+
+## Persistent channel joining
+
+Since PyLink 2.0-alpha3, persistent channels are handled in a plugin specific manner. For any service bot on any network, a plugin can register a list of channels that the bot should join persistently (i.e. through kicks and kills). Instead of removing channels from service bots directly, plugins then "request" parts through the services API, which succeed only if no other plugins still mark the channel as persistent. This rework fixes [edge-case desyncs](https://github.com/jlu5/PyLink/issues/265) in earlier versions when multiple plugins change a service bot's channel list, and replaces the `ServiceBot.extra_channels` attribute (which is no longer supported).
+
+Note: Autojoin channels defined in a network's server block are always treated as persistent on that network.
+
+### Channel management methods
+
+Channels, persistent and otherwise are managed through the following functions implemented by `ServiceBot`. While namespaces for channel registrations can technically be any string, it is preferable to keep them close (or equal) to your plugin name.
+
+- `myservice.add_persistent_channel(irc, namespace, channel, try_join=True)`: Adds a persistent channel to the service bot on the given network and namespace.
+    - `try_join` determines whether the service bot should try to join the channel immediately; you can disable this if you prefer to manage joins by yourself.
+- `myservice.remove_persistent_channel(irc, namespace, channel, try_part=True, part_reason='')`: Removes a persistent channel from the service bot on the given network and namespace.
+    - `try_part` determines whether a part should be requested from the channel immediately. (`part_reason` is ignored if this is False)
+- `myservice.get_persistent_channels(irc, namespace=None)`: Returns a set of persistent channels for the IRC network, optionally filtering by namespace is one is given. The channels defined in the network's server block are also included because they are always treated as persistent.
+- `myservice.clear_persistent_channels(irc, namespace, try_part=True, part_reason='')`: Clears all the persistent channels defined by a namespace. `irc` can also be `None` to clear persistent channels for all networks in this namespace.
+- `myservice.join(irc, channels, ignore_empty=True)`: Joins the given service bot to the given channel(s). `channels` can be an iterable of channel names or the name of a single channel (type `str`).
+    - The `ignore_empty` option sets whether we should skip joining empty channels and join them later when we see someone else join (if it is marked persistent). This option is automatically *disabled* on networks where we cannot monitor channels we're not in (e.g. on Clientbot).
+    - Before 2.0-alpha3, this function implicitly marks channels it receives to be persistent - this is no longer the case!
+- `myservice.part(irc, channels, reason='')`: Requests a part from the given channel(s) - that is, leave only if no other plugins still register it as a persistent channel.
+    - `channels` can be an iterable of channel names or the name of a single channel (type `str`).
+
+### A note on dynamicness
+
+As of PyLink 2.0-alpha3, persistent channels are also "dynamic" in the sense that PyLink service bots will part channels marked persistent when they become empty, and rejoin when they are recreated. This feature will hopefully be more fine-tunable in future releases.
+
+Dynamic channels are disabled on networks with the [`visible-state-only` protocol capability](pmodule-spec.md#pylink-protocol-capabilities) (e.g. Clientbot), where it is impossible to monitor the state of channels the bot is not in.
+
 ## Service bots and commands
 
 Commands for service bots and commands for the main PyLink bot have two main differences.
@@ -58,4 +85,17 @@ Commands for service bots and commands for the main PyLink bot have two main dif
 
 ### Featured commands
 
-Commands for service bots can also be marked as *featured*, which shows it with its command arguments in the default `LIST` command. To mark a command as featured, use `myservice.add_cmd(cmdfunc, 'cmdname', featured=True)`.
+Commands for service bots can also be marked as *featured*, which shows it with its command arguments in the default `LIST` command. To mark a command as featured, enable the `featured` option when binding it: e.g. `myservice.add_cmd(cmdfunc, featured=True)`.
+
+### Command aliases
+
+Since PyLink 2.0-alpha1, `ServiceBot.add_cmd(...)` and `utils.add_cmd(...)` support assigning aliases to a command by defining the `aliases` argument. Command aliases do not show in `LIST`, allowing command listings to be much cleaner. Instead, they are only mentioned when `HELP` is called on an alias command name or its parent.
+
+Example:
+
+```python
+myservice.add_cmd(functwo, aliases=('abc',))
+myservice.add_cmd(somefunc, aliases=('command1', 'command2'))
+```
+
+Note: use `(variable,)` when defining one length tuples to [prevent them from being parsed as a single string](https://wiki.python.org/moin/TupleSyntax).

docs/technical/user-modes.csv

@@ -1,47 +0,0 @@
-User Mode / IRCd,InspIRCd,charybdis,Elemental-IRCd,UnrealIRCd,IRCd-Hybrid,Nefarious IRCu
-admin,,a,a,,a,a
-bot,B,,B,B,,B
-callerid,g,g,g,,g,g
-cloak,x,x,x,x,x,x
-cloak_fakehost,,,,,,f
-cloak_hashedhost,,,,,,C
-cloak_hashedip,,,,,,c
-cloak_sethost,,,,,,h
-deaf,d,D,D,d,D,D
-deaf_commonchan,c,,,,G,q
-debug,,,,,d,
-filter,,,,G,,
-helpop,h,,,,,
-hidechans,I,,I,p,p,n
-hideidle,,,,I,q,I
-hideoper,H,,,H,H,H
-invisible,i,i,i,i,i,i
-locops,,l,l,,l,O
-noctcp,,,C,T,,
-noforward,,Q,Q,,,L
-noinvite,,,V,,,
-oper,o,o,o,,o,o
-operwall,,z,z,,,
-override,,p,p,,,X
-protected,,,,q,,
-regdeaf,R,R,R,R,R,R
-registered,r,,,r,r,r
-servprotect,k,S,S,S,,k
-showwhois,W,,,W,,W
-sno_admin_requests,,,,,y,
-sno_badclientconnections,,,,,u,
-sno_botfloods,,,,,b,
-sno_clientconnections,,,,,c,
-sno_debug,,,,,,g
-sno_fullauthblock,,,,,f,
-sno_nickchange,,,,,n,
-sno_rejectedclients,,,,,j,
-sno_remoteclientconnections,,,,,F,
-sno_server_connects,,,,,e,
-sno_skill,,,,,k,
-snomask,s,s,s,s,s,s
-ssl,,,,z,S,z
-stripcolor,S,,,,,
-vhost,,,,t,,
-wallops,w,w,w,w,w,w
-webirc,,,,,W,

docs/technical/using-ircparser.md

@@ -0,0 +1,123 @@
+# Using utils.IRCParser()
+
+**As of 22/02/2017 (1.2-dev), PyLink allows plugin creators to either parse command arguments themselves
+or use a sub-classed instance of [argparse.ArgumentParser()](https://docs.python.org/3/library/argparse.html)
+to parse their arguments.**
+
+First off, you will already have access to IRCParser due to importing `utils`.
+
+Otherwise, this is how to include it...
+
+```python
+from pylinkirc import utils
+```
+
+When you add a command that you want to use `utils.IRCParser()` with, the following is a guide on how to add arguments.
+
+**Note**: Most if not all the examples are from Python's argparse documentation, linked above.
+
+#### Positional (Named) Arguments
+```python
+SomeParser.add_argument('argname')
+```
+
+#### Flag Arguments / Switch Arguments
+```python
+
+SomeParser = utils.IRCParser()
+SomeParser.addargument('-a', '--argumentname')
+```
+
+##### Action
+
+Actions define what to do when given an argument (i.e. whether it is used by itself or as some other sort of value).
+
+Here are some of the actions that `argparse` defines:
+
+* `store` - just stores the value given. This is the default when an action isn't provided.
+    ```python
+    >>> parser = argparse.ArgumentParser()
+    >>> parser.add_argument('--foo')
+    >>> parser.parse_args('--foo 1'.split())
+    Namespace(foo='1')
+    ```
+
+* `store_true`/`store_false` - used when you just want to check if an argument was used.
+
+    ```python
+    >>> parser = argparse.ArgumentParser()
+    >>> parser.add_argument('--foo', action='store_true')
+    >>> parser.add_argument('--bar', action='store_false')
+    >>> parser.add_argument('--baz', action='store_false')
+    
+    >>> parser.parse_args('--foo --bar'.split())
+    Namespace(foo=True, bar=False, baz=True)
+    ```
+
+* `append` - additively stores arguments if a switch is given multiple times.
+
+    ```python
+    >>> parser = argparse.ArgumentParser()
+    >>> parser.add_argument('--foo', action='append')
+    >>> parser.parse_args('--foo 1 --foo 2'.split())
+    Namespace(foo=['1', '2'])
+    ```
+
+* `count` - counts how many times an argument was used (for flag/switch arguments only)
+    ```python
+    >>> parser = argparse.ArgumentParser()
+    >>> parser.add_argument('--verbose', '-v', action='count')
+    >>> parser.parse_args(['-vvv'])
+    Namespace(verbose=3)
+    ```
+
+You can also specify an arbitrary `Action` by sub-classing Action. If you want
+to do this, you must `import argparse` in your plugin.
+
+More info on that is available [here](https://docs.python.org/3/library/argparse.html#action).
+
+##### Type Constraints
+
+If you want an argument to be of a certain type, you can include a `type=TYPE` keyword, done like so.
+
+```python
+SomeParser.add_argument('argname', type=int)
+
+```
+
+As such this will return an error if the input can not be converted to an `int`.
+
+Types usable are `str` and `int`,
+there may be more that are allowed in this keyword argument,
+but `str` and `int` are the only ones we have throughly used.
+
+**Note**: TYPE can be technically any callable. More about that [here](https://docs.python.org/3/library/argparse.html#type)!
+
+
+
+##### Choices
+If you want to limit what the user can enter for an argument,
+like if they have to choose something from a pre-existing list.
+
+This can be used by adding `choices=['A', 'AAAA', 'CNAME']` into the
+`SomeParser.add_argument()` call along with the option entries (-a/--argname).
+
+```python
+SomeParser.add_argument('argname', choices=['A', 'AAAA', 'CNAME'])
+
+```
+
+##### Needed Args (aka. nargs)
+
+The keyword argument `nargs` or Needed Args associates a different number of arguments to an action.
+
+* `N` - this is an integer; N arguments will be gathered into a list. nargs=1 produces a list of one item, while the default (not using nargs) produces just the argument itself.
+
+* `'?'` - One argument will be used. If `default` is defined in the call, then default will be used if there is no given argument.
+
+* `'*'` - All arguments are gathered into a list. It only makes sense to use this once in a command handler.
+
+* `'+'` - Like '*' but raises an error if there wasn't at least one argument given.
+
+* `utils.IRCParser.REMAINDER` - remaining arguments are gathered into a list; this is usually used when you need to get a phrase stored, such as the 'quote' text of a quote, a service bot part reason, etc. This is an alias to `argparse.REMAINDER`.
+

docs/technical/writing-plugins.md

@@ -1,44 +1,73 @@
 # Writing plugins for PyLink
 
-PyLink plugins are modules that extend its functionality by giving it something to do. Without any plugins loaded, PyLink can only sit on a server and do absolutely nothing.
+***Last updated for 2.1-alpha2 (2019-06-27).***
 
-This guide, along with the sample plugins [`plugins/example.py`](../../plugins/example.py), and [`plugins/service.py`](../../plugins/demo_service.py) aim to show the basics of writing plugins for PyLink.
+Most features in PyLink (Relay, Automode, etc.) are implemented as plugins, which can be mix-and-matched on any particular instance. Without any plugins loaded, PyLink can connect to servers but won't accomplish anything useful.
+
+This guide, along with the sample plugin [`example.py`](../../plugins/example.py) aim to show the basics of writing plugins for PyLink.
 
 ## Receiving data from IRC
 
-Plugins have two ways of communicating with IRC: hooks, and commands sent in PM to the main PyLink client. A simple plugin can use one, or any mixture of these.
+Plugins have two ways of communicating with IRC: hooks, and commands directed towards service clients. Any plugin can use one or a combination of these.
 
-### Hooks
+### Hook events
 
-Hooks are probably the most versatile form of communication. The data in each hook payload is formatted as a Python `dict`, with different data keys depending on the command.
+PyLink's hooks system is designed as a protocol-independent method for protocol modules to communicate with plugins (and to a lesser extend, for plugins to communicate with each other). Hook events are the most versatile form of communication available, with each individual event generally corresponding to a specific chat or server event (e.g. `PRIVMSG`, `JOIN`, `KICK`). Each hook payload includes 4 parts:
-For example, a `PRIVMSG` payload would give you the fields `target` and `text`, while a `PART` payload would only give you `channels` and `reason` fields.
 
-There are many hook types available (one for each supported IRC command), and you can read more about them in the [PyLink hooks reference](hooks-reference.md).
+1) The corresponding network object (IRC object) where the event took place (**type**: a subclass of `pylinkirc.classes.PyLinkNetworkCore`)
+2) The numeric ID† of the sender (**type**: `str`)
+3) An identifier for the command name, which may or may not be the same as the name of the hook depending on context (**type**: `str`)
+4) A freeform `dict` of arguments, where data keys vary by command - see the [PyLink hooks reference](hooks-reference.md) for what's available where.
 
-Plugins can bind to hooks using the `utils.add_hook()` function like so: `utils.add_hook(function_name, 'PRIVMSG')`, where `function_name` is your function definition, and `PRIVMSG` is whatever hook name you want to bind to. Once set up, `function_name` will be called whenever the protocol module receives a `PRIVMSG` command.
+Functions intended to be hook handlers therefore take in 4 arguments corresponding to the ones listed above: `irc`, `source`, `command`, and `args`.
 
-Each hook-bound function takes 4 arguments: `irc, source, command, args`.
+#### Return codes for hook handlers
-- **irc**: The IRC object where the hook was called. Plugins are globally loaded, so there will be one of these per network.
-- **source**: The numeric of the sender. This will usually be a UID (for users) or a SID (for server).
-- **command**: The true command name where the hook originates. This may or may not be the same as the name of the hook, depending on context.
-- **args**: The hook data (a `dict`) associated with the command. Again, the available data keys differ by hook name
-(see the [hooks reference](hooks-reference.md) for a list of which can be used).
 
-Hook functions do not return anything, and can raise exceptions to be caught by the core.
+As of PyLink 2.0, the return value of hook handlers are used to determine how the original event will be passed on to further handlers (that is, those created by plugins loaded later, or hook handlers registered with a lower priority).
+
+The following return values are supported so far:
+
+- `None` or `True`: passthrough the event unchanged to further handlers (the default behavior)
+- `False`: block the event from reaching other handlers
+
+Hook handlers may raise exceptions without blocking the event from reaching further handlers; these are caught by PyLink and logged appropriately.
+
+#### Modifying a hook payload
+
+As of PyLink 2.1, it is acceptable to modify a hook event payload in any plugin handler. This can be used for filtering purposes, e.g. Antispam's part/quit message filtering.
+
+You should take extra caution not to corrupt hook payloads, especially ones that relate to state keeping. Otherwise, other plugins may fail to function correctly.
+
+### Hook priorities
+
+The `priority` option in `utils.add_hook()` allows setting a hook handler's priority when binding it. When multiple modules bind to one hook command, handlers are called in order of decreasing priority (i.e. highest first).
+
+There is no standard for hook priorities as of 2.0; instead they are declared as necessary. Some priority values used in 2.0 are shown here for reference (the default priority for handlers is **100**):
+
+| Module            | Commands        | Priority | Description |
+|-------------------|-----------------|----------|-------------|
+| `service_support` | ENDBURST        | 500      | This sets up services bots before plugins run so that they can assume their presence when initializing. |
+| `antispam`        | PRIVMSG, NOTICE | 990-1000 | This allows `antispam` to filter away spam before it can reach other handlers. |
+| `relay`           | PRIVMSG, NOTICE | 200      | Fixes https://github.com/jlu5/PyLink/issues/123. Essentially, this lets Relay forward messages calling commands before letting the command handler work (and then relaying its responses). |
+| `ctcp`            | PRIVMSG         | 200      | The `ctcp` plugin processes CTCPs and blocks them from reaching the services command handler, preventing extraneous "unknown command" errors. |
 
 ### Bot commands
 
-For plugins that interact with regular users, you can also write commands for the PyLink bot, or [create service bots with their own command set](services-api.md). This section only details the former:
+Plugins can also define service bot commands, either for the main PyLink service bot or for one created by the plugin itself. This section only details the former - see the [Services API Guide](services-api.md) for details on the latter.
 
-Plugins can add commands by including something like `utils.add_cmd(testcommand, "hello")`. Here, `testcommand` is the name of your function, and `hello` is the (optional) name of the command. If no command name is specified, it will use the same name as the function.
+Commands are registered by calling `utils.add_cmd()` with one or two arguments. Ex)
-Now, your command function will be called whenever someone PMs the PyLink client with the command (e.g. `/msg PyLink hello`, case-insensitive).
+- `utils.add_cmd(testcommand, "hello")` registers a function named `testcommand` as the command handler for `hello` (i.e. `/msg PyLink hello`)
+- `utils.add_cmd(testcommand)` registers a function named `testcommand` as the command handler for `testcommand`.
 
-Each command function takes 3 arguments: `irc, source, args`.
+`utils.add_cmd(...)` also takes some keyword arguments, described in the [services API guide](services-api.md#service-bots-and-commands) (replace `myservice.add_cmd` with `utils.add_cmd`). Decorator syntax (`@utils.add_cmd`) can also be used for the second example above.
-- **irc**: The IRC object where the command was called.
-- **source**: The numeric of the sender. This will usually be a UID (for users) or a SID (for server).
-- **args**: A `list` of space-separated command arguments (excluding the command name) that the command was called with. For example, `/msg PyLink hello world 1234` would give an `args` list of `['world', '1234']`
 
-(Unfortunately, this means that for now, any fancy argument parsing has to be done manually.)
+
+Each command handler function takes 3 arguments: `irc, source, args`.
+- **irc**: The network object where the command was called.
+- **source**: The numeric ID (or pseudo-ID) of the sender.
+- **args**: A `list` of command arguments (not including the command name) that the command was called with. For example, `/msg PyLink hello world 1234` would give an `args` list of `['world', '1234']`
+
+As of PyLink 1.2, there are two ways for a plugin to parse arguments: as a raw list of strings, or with `utils.IRCParser` (an [argparse](https://docs.python.org/3/library/argparse.html) wrapper). `IRCParser()` is documented in the ["using IRCParser"](using-ircparser.md) page.
 
 Command handlers do not return anything and can raise exceptions, which are caught by the core and automatically return an error message.
 
@@ -46,22 +75,45 @@ Command handlers do not return anything and can raise exceptions, which are caug
 
 Plugins receive data from the underlying protocol module, and communicate back using outgoing [command functions](pmodule-spec.md) implemented by the protocol module. They should *never* send raw data directly back to IRC, because that wouldn't be portable across different IRCds.
 
-These functions are usually called in this fashion: `irc.proto.command(arg1, arg2, ...)`. For example, the command `irc.proto.join('10XAAAAAB', '#bots')` would join a PyLink client with UID `10XAAAAAB` to channel `#bots`.
+These functions are called in the form: `irc.command(arg1, arg2, ...)`. For example, the command `irc.join('10XAAAAAB', '#bots')` would join a PyLink client with UID `10XAAAAAB` to the channel `#bots`.
 
 For sending messages (e.g. replies to commands), simpler forms of:
 
 - `irc.reply(text, notice=False, source=None)`
+- `irc.error(text, notice=False, source=None)`
 - and `irc.msg(targetUID, text, notice=False, source=None)`
 
 are preferred.
 
-`irc.reply()` is a special form of `irc.msg` in that it automatically finds the target to reply to. If the command was called in a channel using fantasy, it will send the reply in that channel. Otherwise, the reply will be sent in a PM to the caller.
+`irc.reply()` is a frontend to `irc.msg()` which automatically finds the right target to reply to: that is, the channel for fantasy commands and the caller for PMs. `irc.error()` is in turn a wrapper around `irc.reply()` which prefixes the given text with `Error: `.
 
-The sender UID for both can be set using the `source` argument, and defaults to the main PyLink client.
+The sender UID for all of these can be set using the `source` argument, and defaults to the main PyLink client.
+
+## Access checking for commands
+
+See the [Permissions API documentation](permissions-api.md) on how to restrict commands to certain users.
 
 ## Special triggers for plugin (un)loading
 
 The following functions can also be defined in the body of a plugin to hook onto plugin loading / unloading.
 
-`main(irc=None)`: Called on plugin load. `irc` is only defined when the plugin is being reloaded from a network: otherwise, it means that PyLink has just been started.
+- `main(irc=None)`: Called on plugin load. `irc` is only defined when the plugin is being reloaded from a network: otherwise, it means that PyLink has just been started.
-`die(irc=None)`: Called on plugin unload or daemon shutdown. `irc` is only defined when the shutdown or unload was called from an IRC network.
+- `die(irc=None)`: Called on plugin unload or daemon shutdown. `irc` is only defined when the shutdown or unload was called from an IRC network.
+
+## Other tips
+
+### Logging
+
+Use PyLink's [global logger](https://docs.python.org/3/library/logging.html) (`from pylinkirc.log import log`) instead of print statements.
+
+### Some useful attributes
+
+- **`world.networkobjects`** provides a dict mapping network names (case sensitive) to their corresponding network objects/protocol module instances.
+- **`irc.connected`** is a [`threading.Event()`](https://docs.python.org/3/library/threading.html#event-objects) object that is set when a network finishes bursting.
+- `world.started` is a [`threading.Event()`](https://docs.python.org/3/library/threading.html#event-objects) object that is set when all networks have been initialized.
+- `world.plugins` provides a dict mapping loaded plugins' names (case sensitive) to their module objects. This is the preferred way to call another plugins's methods if need be (while of course, forcing you to check whether the other plugin is already loaded).
+- `world.services` provides a dict mapping service bot names to their `utils.ServiceBot` instances.
+
+### Useful modules
+
+`classes.py`, `utils.py` and `structures.py` all provide a ton of public methods which aren't documented here for conciseness. In `classes.py`, `PyLinkNetworkCore` and `PyLinkNetworkCoreUtils` (which all protocol modules inherit from) are where many utility and state-checking functions sit.

launcher.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""
+PyLink IRC Services launcher.
+"""
+
+import os
+import signal
+import sys
+import time
+
+from pylinkirc import __version__, conf, real_version, world
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+args = {}
+
+def _main():
+    conf.load_conf(args.config)
+
+    from pylinkirc.log import log
+    from pylinkirc import classes, utils, coremods, selectdriver
+
+    # Write and check for an existing PID file unless specifically told not to.
+    if not args.no_pid:
+        pid_dir = conf.conf['pylink'].get('pid_dir', '')
+        pidfile = os.path.join(pid_dir, '%s.pid' % conf.confname)
+        pid_exists = False
+        pid = None
+        if os.path.exists(pidfile):
+            try:
+                with open(pidfile) as f:
+                    pid = int(f.read())
+            except OSError:
+                log.exception("Could not read PID file %s:", pidfile)
+            else:
+                pid_exists = True
+
+            if psutil is not None and os.name == 'posix':
+                # FIXME: Haven't tested this on other platforms, so not turning it on by default.
+                try:
+                    proc = psutil.Process(pid)
+                except psutil.NoSuchProcess:  # Process doesn't exist!
+                    pid_exists = False
+                    log.info("Ignoring stale PID %s from PID file %r: no such process exists.", pid, pidfile)
+                else:
+                    # This PID got reused for something that isn't us?
+                    if not any('pylink' in arg.lower() for arg in proc.cmdline()):
+                        log.info("Ignoring stale PID %s from PID file %r: process command line %r is not us", pid, pidfile, proc.cmdline())
+                        pid_exists = False
+
+        if pid and pid_exists:
+            if args.rehash:
+                os.kill(pid, signal.SIGUSR1)
+                log.info('OK, rehashed PyLink instance %s (config %r)', pid, args.config)
+                sys.exit()
+            elif args.stop or args.restart:  # Handle --stop and --restart options
+                os.kill(pid, signal.SIGTERM)
+
+                log.info("Waiting for PyLink instance %s (config %r) to stop...", pid, args.config)
+                while os.path.exists(pidfile):
+                    # XXX: this is ugly, but os.waitpid() only works on non-child processes on Windows
+                    time.sleep(0.2)
+                log.info("Successfully killed PID %s for config %r.", pid, args.config)
+
+                if args.stop:
+                    sys.exit()
+            else:
+                log.error("PID file %r exists; aborting!", pidfile)
+
+                if psutil is None:
+                    log.error("If PyLink didn't shut down cleanly last time it ran, or you're upgrading "
+                              "from PyLink < 1.1-dev, delete %r and start the server again.", pidfile)
+                    if os.name == 'posix':
+                        log.error("Alternatively, you can install psutil for Python 3 (pip3 install psutil), "
+                                  "which will allow this launcher to detect stale PID files and ignore them.")
+                sys.exit(1)
+
+        elif args.stop or args.restart or args.rehash:  # XXX: also repetitive
+            # --stop and --restart should take care of stale PIDs.
+            if pid:
+                world._should_remove_pid = True
+                log.error('Cannot stop/rehash PyLink: no process with PID %s exists.', pid)
+            else:
+                log.error('Cannot stop/rehash PyLink: PID file %r does not exist or cannot be read.', pidfile)
+            sys.exit(1)
+
+        world._should_remove_pid = True
+
+    log.info('PyLink %s starting...', __version__)
+
+    world.daemon = args.daemonize
+    if args.daemonize:
+        if args.no_pid:
+            print('ERROR: Combining --no-pid and --daemonize is not supported.')
+            sys.exit(1)
+        elif os.name != 'posix':
+            print('ERROR: Daemonization is not supported outside POSIX systems.')
+            sys.exit(1)
+        else:
+            log.info('Forking into the background.')
+            log.removeHandler(world.console_handler)
+
+            # Adapted from https://stackoverflow.com/questions/5975124/
+            if os.fork():
+                # Fork and exit the parent process.
+                os._exit(0)
+
+            os.setsid()  # Decouple from our lovely terminal
+            if os.fork():
+                # Fork again to prevent starting zombie apocalypses.
+                os._exit(0)
+    else:
+        # For foreground sessions, set the terminal window title.
+        # See https://bbs.archlinux.org/viewtopic.php?id=85567 &
+        #     https://stackoverflow.com/questions/7387276/
+        if os.name == 'nt':
+            import ctypes
+            ctypes.windll.kernel32.SetConsoleTitleW("PyLink %s" % __version__)
+        elif os.name == 'posix':
+            sys.stdout.write("\x1b]2;PyLink %s\x07" % __version__)
+
+    if not args.no_pid:
+        # Write the PID file only after forking.
+        with open(pidfile, 'w') as f:
+            f.write(str(os.getpid()))
+
+    # Load configured plugins
+    to_load = conf.conf['plugins']
+    utils._reset_module_dirs()
+
+    for plugin in to_load:
+        try:
+            world.plugins[plugin] = pl = utils._load_plugin(plugin)
+        except Exception as e:
+            log.exception('Failed to load plugin %r: %s: %s', plugin, type(e).__name__, str(e))
+        else:
+            if hasattr(pl, 'main'):
+                log.debug('Calling main() function of plugin %r', pl)
+                pl.main()
+
+    # Initialize all the networks one by one
+    for network, sdata in conf.conf['servers'].items():
+        try:
+            protoname = sdata['protocol']
+        except (KeyError, TypeError):
+            log.error("(%s) Configuration error: No protocol module specified, aborting.", network)
+        else:
+            # Fetch the correct protocol module.
+            try:
+                proto = utils._get_protocol_module(protoname)
+
+                # Create and connect the network.
+                world.networkobjects[network] = irc = proto.Class(network)
+                log.debug('Connecting to network %r', network)
+                irc.connect()
+            except:
+                log.exception('(%s) Failed to connect to network %r, skipping it...',
+                              network, network)
+                continue
+
+    world.started.set()
+    log.info("Loaded plugins: %s", ', '.join(sorted(world.plugins.keys())))
+    selectdriver.start()
+
+def main():
+    import argparse
+
+    global args
+
+    parser = argparse.ArgumentParser(description='Starts an instance of PyLink IRC Services.')
+    parser.add_argument('config', help='specifies the path to the config file (defaults to pylink.yml)', nargs='?', default='pylink.yml')
+    parser.add_argument("-v", "--version", help="displays the program version and exits", action='store_true')
+    parser.add_argument("-c", "--check-pid", help="no-op; kept for compatibility with PyLink <= 1.2.x", action='store_true')
+    parser.add_argument("-n", "--no-pid", help="skips generating and checking PID files", action='store_true')
+    parser.add_argument("-r", "--restart", help="restarts the PyLink instance with the given config file", action='store_true')
+    parser.add_argument("-s", "--stop", help="stops the PyLink instance with the given config file", action='store_true')
+    parser.add_argument("-R", "--rehash", help="rehashes the PyLink instance with the given config file", action='store_true')
+    parser.add_argument("-d", "--daemonize", help="daemonizes the PyLink instance on POSIX systems", action='store_true')
+    parser.add_argument("-t", "--trace", help="traces through running Python code; useful for debugging", action='store_true')
+    parser.add_argument('--trace-ignore-mods', help='comma-separated list of extra modules to ignore when tracing', action='store', default='')
+    parser.add_argument('--trace-ignore-dirs', help='comma-separated list of extra directories to ignore when tracing', action='store', default='')
+    args = parser.parse_args()
+
+    if args.version:  # Display version and exit
+        print('PyLink %s (in VCS: %s)' % (__version__, real_version))
+        sys.exit()
+
+    # XXX: repetitive
+    elif args.no_pid and (args.restart or args.stop or args.rehash):
+        print('ERROR: --no-pid cannot be combined with --restart or --stop')
+        sys.exit(1)
+    elif args.rehash and os.name != 'posix':
+        print('ERROR: Rehashing via the command line is not supported outside Unix.')
+        sys.exit(1)
+
+    if args.trace:
+        import trace
+        tracer = trace.Trace(ignoremods=args.trace_ignore_mods.split(','),
+                             ignoredirs=args.trace_ignore_dirs.split(','))
+        tracer.runctx('_main()', globals=globals(), locals=locals())
+    else:
+        _main()

log.py

@@ -8,58 +8,70 @@ access the global logger object by importing "log" from this module
 
 import logging
 import logging.handlers
-import sys
 import os
 
-from . import world, conf
+from . import conf, world
+
+__all__ = ['log']
 
 # Stores a list of active file loggers.
 fileloggers = []
 
-stdout_level = conf.conf['logging'].get('stdout') or 'INFO'
+# TODO: perhaps make this format configurable?
-
-logdir = os.path.join(os.getcwd(), 'log')
-os.makedirs(logdir, exist_ok=True)
-
 _format = '%(asctime)s [%(levelname)s] %(message)s'
 logformatter = logging.Formatter(_format)
 
+def _get_console_log_level():
+    """
+    Returns the configured console log level.
+    """
+    logconf = conf.conf['logging']
+    return logconf.get('console', logconf.get('stdout')) or 'INFO'
+
 # Set up logging to STDERR
-world.stdout_handler = logging.StreamHandler()
+world.console_handler = logging.StreamHandler()
-world.stdout_handler.setFormatter(logformatter)
+world.console_handler.setFormatter(logformatter)
-world.stdout_handler.setLevel(stdout_level)
+world.console_handler.setLevel(_get_console_log_level())
 
 # Get the main logger object; plugins can import this variable for convenience.
-log = logging.getLogger()
+log = logging.getLogger('pylinkirc')
-log.addHandler(world.stdout_handler)
+log.addHandler(world.console_handler)
 
 # This is confusing, but we have to set the root logger to accept all events. Only this way
 # can other loggers filter out events on their own, instead of having everything dropped by
 # the root logger. https://stackoverflow.com/questions/16624695
 log.setLevel(1)
 
-def makeFileLogger(filename, level=None):
+def _make_file_logger(filename, level=None):
     """
     Initializes a file logging target with the given filename and level.
     """
+    logconf = conf.conf.get('logging', {})
+
+    logdir = logconf.get('log_dir')
+    if logdir is None:
+        logdir = os.path.join(os.getcwd(), 'log')
+
+    os.makedirs(logdir, exist_ok=True)
+
     # Use log names specific to the current instance, to prevent multiple
     # PyLink instances from overwriting each others' log files.
     target = os.path.join(logdir, '%s-%s.log' % (conf.confname, filename))
 
-    logrotconf = conf.conf.get('logging', {}).get('filerotation', {})
+    logrotconf = logconf.get('filerotation', {})
 
-    # Max amount of bytes per file, before rotation is done. Defaults to 50 MiB.
+    # Max amount of bytes per file, before rotation is done. Defaults to 20 MiB.
-    maxbytes = logrotconf.get('max_bytes', 52428800)
+    maxbytes = logrotconf.get('max_bytes', 20971520)
 
     # Amount of backups to make (e.g. pylink-debug.log, pylink-debug.log.1, pylink-debug.log.2, ...)
     # Defaults to 5.
     backups = logrotconf.get('backup_count', 5)
 
-    filelogger = logging.handlers.RotatingFileHandler(target, maxBytes=maxbytes, backupCount=backups)
+    filelogger = logging.handlers.RotatingFileHandler(target, maxBytes=maxbytes, backupCount=backups, encoding='utf-8')
     filelogger.setFormatter(logformatter)
 
-    # If no log level is specified, use the same one as STDOUT.
+    # If no log level is specified, use the same one as the console logger.
-    level = level or stdout_level
+    level = level or _get_console_log_level()
     filelogger.setLevel(level)
 
     log.addHandler(filelogger)
@@ -68,7 +80,7 @@ def makeFileLogger(filename, level=None):
 
     return filelogger
 
-def stopFileLoggers():
+def _stop_file_loggers():
     """
     De-initializes all file loggers.
     """
@@ -82,7 +94,18 @@ def stopFileLoggers():
 files = conf.conf['logging'].get('files')
 if files:
     for filename, config in files.items():
-        makeFileLogger(filename, config.get('loglevel'))
+        if isinstance(config, dict):
+            _make_file_logger(filename, config.get('loglevel'))
+        else:
+            log.warning('Got invalid file logging pair %r: %r; are your indentation and block '
+                        'commenting consistent?', filename, config)
+
+log.debug("log: Emptying _log_queue")
+# Process and empty the log queue
+while world._log_queue:
+    level, text = world._log_queue.popleft()
+    log.log(level, text)
+log.debug("log: Emptied _log_queue")
 
 class PyLinkChannelLogger(logging.Handler):
     """
@@ -121,11 +144,9 @@ class PyLinkChannelLogger(logging.Handler):
         # 1) irc.pseudoclient must be initialized already
         # 2) IRC object must be finished bursting
         # 3) Target channel must exist
-        # 4) Main PyLink client must be in this target channel
+        # 4) This function hasn't been called already (prevents recursive loops).
-        # 5) This function hasn't been called already (prevents recursive loops).
         if self.irc.pseudoclient and self.irc.connected.is_set() \
-                and self.channel in self.irc.channels and self.irc.pseudoclient.uid in \
+                and self.channel in self.irc.channels and not self.called:
-                self.irc.channels[self.channel].users and not self.called:
 
             self.called = True
             msg = self.format(record)
@@ -139,4 +160,3 @@ class PyLinkChannelLogger(logging.Handler):
                     return
                 else:
                     self.called = False
-

log/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

plugins/antispam.py

@@ -0,0 +1,409 @@
+# antispam.py: Basic services-side spamfilters for IRC
+
+from pylinkirc import conf, utils
+from pylinkirc.log import log
+
+mydesc = ("Provides anti-spam functionality.")
+sbot = utils.register_service("antispam", default_nick="AntiSpam", desc=mydesc)
+
+def die(irc=None):
+    utils.unregister_service("antispam")
+
+_UNICODE_CHARMAP = {
+    'A': 'AΑАᎪᗅᴀ𝐀𝐴𝑨𝒜𝓐𝔄𝔸𝕬𝖠𝗔𝘈𝘼𝙰𝚨𝛢𝜜𝝖𝞐',
+    'B': 'BʙΒВвᏴᗷᛒℬ𐌁𝐁𝐵𝑩𝓑𝔅𝔹𝕭𝖡𝗕𝘉𝘽𝙱𝚩𝛣𝜝𝝗𝞑',
+    'C': 'CϹСᏟℂℭⅭⲤ𐌂𝐂𝐶𝑪𝒞𝓒𝕮𝖢𝗖𝘊𝘾𝙲',
+    'D': 'DᎠᗞᗪᴅⅅⅮ𝐃𝐷𝑫𝒟𝓓𝔇𝔻𝕯𝖣𝗗𝘋𝘿𝙳',
+    'E': 'EΕЕᎬᴇℰ⋿ⴹ𝐄𝐸𝑬𝓔𝔈𝔼𝕰𝖤𝗘𝘌𝙀𝙴𝚬𝛦𝜠𝝚𝞔',
+    'F': 'FϜᖴℱ𝐅𝐹𝑭𝓕𝔉𝔽𝕱𝖥𝗙𝘍𝙁𝙵𝟊',
+    'G': 'GɢԌԍᏀᏳ𝐆𝐺𝑮𝒢𝓖𝔊𝔾𝕲𝖦𝗚𝘎𝙂𝙶',
+    'H': 'HʜΗНнᎻᕼℋℌℍⲎ𝐇𝐻𝑯𝓗𝕳𝖧𝗛𝘏𝙃𝙷𝚮𝛨𝜢𝝜𝞖',
+    'J': 'JЈᎫᒍᴊ𝐉𝐽𝑱𝒥𝓙𝔍𝕁𝕵𝖩𝗝𝘑𝙅𝙹',
+    'K': 'KΚКᏦᛕKⲔ𝐊𝐾𝑲𝒦𝓚𝔎𝕂𝕶𝖪𝗞𝘒𝙆𝙺𝚱𝛫𝜥𝝟𝞙',
+    'L': 'LʟᏞᒪℒⅬ𝐋𝐿𝑳𝓛𝔏𝕃𝕷𝖫𝗟𝘓𝙇𝙻',
+    'M': 'MΜϺМᎷᗰᛖℳⅯⲘ𐌑𝐌𝑀𝑴𝓜𝔐𝕄𝕸𝖬𝗠𝘔𝙈𝙼𝚳𝛭𝜧𝝡𝞛',
+    'N': 'NɴΝℕⲚ𝐍𝑁𝑵𝒩𝓝𝔑𝕹𝖭𝗡𝘕𝙉𝙽𝚴𝛮𝜨𝝢𝞜',
+    'P': 'PΡРᏢᑭᴘᴩℙⲢ𝐏𝑃𝑷𝒫𝓟𝔓𝕻𝖯𝗣𝘗𝙋𝙿𝚸𝛲𝜬𝝦𝞠',
+    'Q': 'Qℚⵕ𝐐𝑄𝑸𝒬𝓠𝔔𝕼𝖰𝗤𝘘𝙌𝚀',
+    'R': 'RƦʀᎡᏒᖇᚱℛℜℝ𝐑𝑅𝑹𝓡𝕽𝖱𝗥𝘙𝙍𝚁',
+    'S': 'SЅՏᏕᏚ𝐒𝑆𝑺𝒮𝓢𝔖𝕊𝕾𝖲𝗦𝘚𝙎𝚂',
+    'T': 'TΤτТтᎢᴛ⊤⟙Ⲧ𐌕𝐓𝑇𝑻𝒯𝓣𝔗𝕋𝕿𝖳𝗧𝘛𝙏𝚃𝚻𝛕𝛵𝜏𝜯𝝉𝝩𝞃𝞣𝞽',
+    'U': 'UՍሀᑌ∪⋃𝐔𝑈𝑼𝒰𝓤𝔘𝕌𝖀𝖴𝗨𝘜𝙐𝚄',
+    'V': 'VѴ٧۷ᏙᐯⅤⴸ𝐕𝑉𝑽𝒱𝓥𝔙𝕍𝖁𝖵𝗩𝘝𝙑𝚅',
+    'W': 'WԜᎳᏔ𝐖𝑊𝑾𝒲𝓦𝔚𝕎𝖂𝖶𝗪𝘞𝙒𝚆',
+    'X': 'XΧХ᙭ᚷⅩ╳Ⲭⵝ𐌗𐌢𝐗𝑋𝑿𝒳𝓧𝔛𝕏𝖃𝖷𝗫𝘟𝙓𝚇𝚾𝛸𝜲𝝬𝞦',
+    'Y': 'YΥϒУҮᎩᎽⲨ𝐘𝑌𝒀𝒴𝓨𝔜𝕐𝖄𝖸𝗬𝘠𝙔𝚈𝚼𝛶𝜰𝝪𝞤',
+    'Z': 'ZΖᏃℤℨ𝐙𝑍𝒁𝒵𝓩𝖅𝖹𝗭𝘡𝙕𝚉𝚭𝛧𝜡𝝛𝞕',
+    'a': 'aɑαа⍺𝐚𝑎𝒂𝒶𝓪𝔞𝕒𝖆𝖺𝗮𝘢𝙖𝚊𝛂𝛼𝜶𝝰𝞪',
+    'b': 'bƄЬᏏᖯ𝐛𝑏𝒃𝒷𝓫𝔟𝕓𝖇𝖻𝗯𝘣𝙗𝚋',
+    'c': 'cϲсᴄⅽⲥ𝐜𝑐𝒄𝒸𝓬𝔠𝕔𝖈𝖼𝗰𝘤𝙘𝚌',
+    'd': 'ⅾdԁᏧᑯⅆⅾ𝐝𝑑𝒅𝒹𝓭𝔡𝕕𝖉𝖽𝗱𝘥𝙙𝚍',
+    'e': 'eеҽ℮ℯⅇ𝐞𝑒𝒆𝓮𝔢𝕖𝖊𝖾𝗲𝘦𝙚𝚎ᥱ',
+    'f': 'fſϝքẝ𝐟𝑓𝒇𝒻𝓯𝔣𝕗𝖋𝖿𝗳𝘧𝙛𝚏𝟋',
+    'g': 'gƍɡցᶃℊ𝐠𝑔𝒈𝓰𝔤𝕘𝖌𝗀𝗴𝘨𝙜𝚐',
+    'h': 'hһհᏂℎ𝐡𝒉𝒽𝓱𝔥𝕙𝖍𝗁𝗵𝘩𝙝𝚑',
+    'i': 'iıɩɪιіӏᎥℹⅈⅰ⍳ꙇ𝐢𝑖𝒊𝒾𝓲𝔦𝕚𝖎𝗂𝗶𝘪𝙞𝚒𝚤𝛊𝜄𝜾𝝸𝞲',
+    'j': 'jϳјⅉ𝐣𝑗𝒋𝒿𝓳𝔧𝕛𝖏𝗃𝗷𝘫𝙟𝚓',
+    'k': 'k𝐤𝑘𝒌𝓀𝓴𝔨𝕜𝖐𝗄𝗸𝘬𝙠𝚔',
+    'l': 'ⅼ',
+    'm': 'ⅿm',
+    'n': 'nոռ𝐧𝑛𝒏𝓃𝓷𝔫𝕟𝖓𝗇𝗻𝘯𝙣𝚗ᥒ',
+    'o': 'ⲟഠοо',
+    'p': 'pρϱр⍴ⲣ𝐩𝑝𝒑𝓅𝓹𝔭𝕡𝖕𝗉𝗽𝘱𝙥𝚙𝛒𝛠𝜌𝜚𝝆𝝔𝞀𝞎𝞺𝟈',
+    'q': 'qԛգզ𝐪𝑞𝒒𝓆𝓺𝔮𝕢𝖖𝗊𝗾𝘲𝙦𝚚',
+    'r': 'rгᴦⲅ𝐫𝑟𝒓𝓇𝓻𝔯𝕣𝖗𝗋𝗿𝘳𝙧𝚛',
+    's': 'sƽѕꜱ𝐬𝑠𝒔𝓈𝓼𝔰𝕤𝖘𝗌𝘀𝘴𝙨𝚜',
+    't': 't𝐭𝑡𝒕𝓉𝓽𝔱𝕥𝖙𝗍𝘁𝘵𝙩𝚝',
+    'u': 'uʋυսᴜ𝐮𝑢𝒖𝓊𝓾𝔲𝕦𝖚𝗎𝘂𝘶𝙪𝚞𝛖𝜐𝝊𝞄𝞾ᥙ',
+    'v': 'vνѵטᴠⅴ∨⋁𝐯𝑣𝒗𝓋𝓿𝔳𝕧𝖛𝗏𝘃𝘷𝙫𝚟𝛎𝜈𝝂𝝼𝞶',
+    'w': 'wɯѡԝաᴡ𝐰𝑤𝒘𝓌𝔀𝔴𝕨𝖜𝗐𝘄𝘸𝙬𝚠',
+    'x': 'x×хᕁᕽ᙮ⅹ⤫⤬⨯𝐱𝑥𝒙𝓍𝔁𝔵𝕩𝖝𝗑𝘅𝘹𝙭𝚡',
+    'y': 'yɣʏγуүყᶌỿℽ𝐲𝑦𝒚𝓎𝔂𝔶𝕪𝖞𝗒𝘆𝘺𝙮𝚢𝛄𝛾𝜸𝝲𝞬',
+    'z': 'zᴢ𝐳𝑧𝒛𝓏𝔃𝔷𝕫𝖟𝗓𝘇𝘻𝙯𝚣',
+    '/': '᜵⁄∕⧸／',
+    '\\': '⧵﹨⧹＼',
+    ' ': '\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\xa0\u202f\u205f',
+    '.': '․．',
+    '-': '˗╴﹣－−⎼',
+    '!': '﹗！ǃⵑ︕',
+    ':': ':˸։፡᛬⁚∶⠆︓﹕',
+    '#': '＃﹟'
+}
+
+def _prep_maketrans(data):
+    from_s = ''
+    to_s = ''
+    for target, chars in data.items():
+        from_s += chars
+        to_s += target * len(chars)
+
+    return str.maketrans(from_s, to_s)
+
+UNICODE_CHARMAP = _prep_maketrans(_UNICODE_CHARMAP)
+
+PUNISH_OPTIONS = ['kill', 'ban', 'quiet', 'kick', 'block']
+EXEMPT_OPTIONS = ['voice', 'halfop', 'op']
+DEFAULT_EXEMPT_OPTION = 'halfop'
+def _punish(irc, target, channel, punishment, reason):
+    """Punishes the target user. This function returns True if the user was successfully punished."""
+    if target not in irc.users:
+        log.warning("(%s) antispam: got target %r that isn't a user?", irc.name, target)
+        return False
+    elif irc.is_oper(target):
+        log.debug("(%s) antispam: refusing to punish oper %s/%s", irc.name, target, irc.get_friendly_name(target))
+        return False
+
+    target_nick = irc.get_friendly_name(target)
+
+    if channel:
+        c = irc.channels[channel]
+        exempt_level = irc.get_service_option('antispam', 'exempt_level', DEFAULT_EXEMPT_OPTION).lower()
+
+        if exempt_level not in EXEMPT_OPTIONS:
+            log.error('(%s) Antispam exempt %r is not a valid setting, '
+                      'falling back to defaults; accepted settings include: %s',
+                      irc.name, exempt_level, ', '.join(EXEMPT_OPTIONS))
+            exempt_level = DEFAULT_EXEMPT_OPTION
+
+        if exempt_level == 'voice' and c.is_voice_plus(target):
+            log.debug("(%s) antispam: refusing to punish voiced and above %s/%s", irc.name, target, target_nick)
+            return False
+        elif exempt_level == 'halfop' and c.is_halfop_plus(target):
+            log.debug("(%s) antispam: refusing to punish halfop and above %s/%s", irc.name, target, target_nick)
+            return False
+        elif exempt_level == 'op' and c.is_op_plus(target):
+            log.debug("(%s) antispam: refusing to punish op and above %s/%s", irc.name, target, target_nick)
+            return False
+
+    my_uid = sbot.uids.get(irc.name)
+    # XXX workaround for single-bot protocols like Clientbot
+    if irc.pseudoclient and not irc.has_cap('can-spawn-clients'):
+        my_uid = irc.pseudoclient.uid
+
+    bans = set()
+    log.debug('(%s) antispam: got %r as punishment for %s/%s', irc.name, punishment,
+              target, irc.get_friendly_name(target))
+
+    def _ban():
+        bans.add(irc.make_channel_ban(target))
+    def _quiet():
+        bans.add(irc.make_channel_ban(target, ban_type='quiet'))
+    def _kick():
+        irc.kick(my_uid, channel, target, reason)
+        irc.call_hooks([my_uid, 'ANTISPAM_KICK', {'channel': channel, 'text': reason, 'target': target,
+                                                  'parse_as': 'KICK'}])
+    def _kill():
+        if target not in irc.users:
+            log.debug('(%s) antispam: not killing %s/%s; they already left', irc.name, target,
+                      irc.get_friendly_name(target))
+            return
+        userdata = irc.users[target]
+        irc.kill(my_uid, target, reason)
+        irc.call_hooks([my_uid, 'ANTISPAM_KILL', {'target': target, 'text': reason,
+                                                  'userdata': userdata, 'parse_as': 'KILL'}])
+
+    kill = False
+    successful_punishments = 0
+    for action in set(punishment.split('+')):
+        if action not in PUNISH_OPTIONS:
+            log.error('(%s) Antispam punishment %r is not a valid setting; '
+                      'accepted settings include: %s OR any combination of '
+                      'these joined together with a "+".',
+                      irc.name, punishment, ', '.join(PUNISH_OPTIONS))
+            return
+        elif action == 'block':
+            # We only need to increment this for this function to return True
+            successful_punishments += 1
+        elif action == 'kill':
+            kill = True  # Delay kills so that the user data doesn't disappear.
+        # XXX factorize these blocks
+        elif action == 'kick' and channel:
+            try:
+                _kick()
+            except NotImplementedError:
+                log.warning("(%s) antispam: Kicks are not supported on this network, skipping; "
+                            "target was %s/%s", irc.name, target_nick, channel)
+            else:
+                successful_punishments += 1
+        elif action == 'ban' and channel:
+            try:
+                _ban()
+            except (ValueError, NotImplementedError):
+                log.warning("(%s) antispam: Bans are not supported on this network, skipping; "
+                            "target was %s/%s", irc.name, target_nick, channel)
+            else:
+                successful_punishments += 1
+        elif action == 'quiet' and channel:
+            try:
+                _quiet()
+            except (ValueError, NotImplementedError):
+                log.warning("(%s) antispam: Quiet is not supported on this network, skipping; "
+                            "target was %s/%s", irc.name, target_nick, channel)
+            else:
+                successful_punishments += 1
+
+    if bans:  # Set all bans at once to prevent spam
+        irc.mode(my_uid, channel, bans)
+        irc.call_hooks([my_uid, 'ANTISPAM_BAN',
+                        {'target': channel, 'modes': bans, 'parse_as': 'MODE'}])
+    if kill:
+        try:
+            _kill()
+        except NotImplementedError:
+            log.warning("(%s) antispam: Kills are not supported on this network, skipping; "
+                        "target was %s/%s", irc.name, target_nick, channel)
+        else:
+            successful_punishments += 1
+
+    if not successful_punishments:
+        log.warning('(%s) antispam: Failed to punish %s with %r, target was %s', irc.name,
+                    target_nick, punishment, channel or 'a PM')
+
+    return bool(successful_punishments)
+
+MASSHIGHLIGHT_DEFAULTS = {
+    'min_length': 50,
+    'min_nicks': 5,
+    'reason': "Mass highlight spam is prohibited",
+    'punishment': 'kick+ban',
+    'enabled': False
+}
+def handle_masshighlight(irc, source, command, args):
+    """Handles mass highlight attacks."""
+    channel = args['target']
+    text = args['text']
+    mhl_settings = irc.get_service_option('antispam', 'masshighlight',
+                                          MASSHIGHLIGHT_DEFAULTS)
+
+    if not mhl_settings.get('enabled', False):
+        return
+
+    my_uid = sbot.uids.get(irc.name)
+
+    # XXX workaround for single-bot protocols like Clientbot
+    if irc.pseudoclient and not irc.has_cap('can-spawn-clients'):
+        my_uid = irc.pseudoclient.uid
+
+    if (not irc.connected.is_set()) or (not my_uid):
+        # Break if the network isn't ready.
+        log.debug("(%s) antispam.masshighlight: skipping processing; network isn't ready", irc.name)
+        return
+    elif not irc.is_channel(channel):
+        # Not a channel - mass highlight blocking only makes sense within channels
+        log.debug("(%s) antispam.masshighlight: skipping processing; %r is not a channel", irc.name, channel)
+        return
+    elif irc.is_internal_client(source):
+        # Ignore messages from our own clients.
+        log.debug("(%s) antispam.masshighlight: skipping processing message from internal client %s", irc.name, source)
+        return
+    elif source not in irc.users:
+        log.debug("(%s) antispam.masshighlight: ignoring message from non-user %s", irc.name, source)
+        return
+    elif channel not in irc.channels or my_uid not in irc.channels[channel].users:
+        # We're not monitoring this channel.
+        log.debug("(%s) antispam.masshighlight: skipping processing message from channel %r we're not in", irc.name, channel)
+        return
+    elif len(text) < mhl_settings.get('min_length', MASSHIGHLIGHT_DEFAULTS['min_length']):
+        log.debug("(%s) antispam.masshighlight: skipping processing message %r; it's too short", irc.name, text)
+        return
+
+    if irc.get_service_option('antispam', 'strip_formatting', True):
+        text = utils.strip_irc_formatting(text)
+
+    # Strip :, from potential nicks
+    words = [word.rstrip(':,') for word in text.split()]
+
+    userlist = [irc.users[uid].nick for uid in irc.channels[channel].users.copy()]
+    min_nicks = mhl_settings.get('min_nicks', MASSHIGHLIGHT_DEFAULTS['min_nicks'])
+
+    # Don't allow repeating the same nick to trigger punishment
+    nicks_caught = set()
+
+    punished = False
+    for word in words:
+        if word in userlist:
+            nicks_caught.add(word)
+        if len(nicks_caught) >= min_nicks:
+            # Get the punishment and reason.
+            punishment = mhl_settings.get('punishment', MASSHIGHLIGHT_DEFAULTS['punishment']).lower()
+            reason = mhl_settings.get('reason', MASSHIGHLIGHT_DEFAULTS['reason'])
+
+            log.info("(%s) antispam: punishing %s => %s for mass highlight spam",
+                     irc.name,
+                     irc.get_friendly_name(source),
+                     channel)
+            punished = _punish(irc, source, channel, punishment, reason)
+            break
+
+    log.debug('(%s) antispam.masshighlight: got %s/%s nicks on message to %r', irc.name,
+              len(nicks_caught), min_nicks, channel)
+    return not punished  # Filter this message from relay, etc. if it triggered protection
+
+utils.add_hook(handle_masshighlight, 'PRIVMSG', priority=1000)
+utils.add_hook(handle_masshighlight, 'NOTICE', priority=1000)
+
+TEXTFILTER_DEFAULTS = {
+    'reason': "Spam is prohibited",
+    'punishment': 'kick+ban+block',
+    'watch_pms': False,
+    'enabled': False,
+    'munge_unicode': True,
+}
+def handle_textfilter(irc, source, command, args):
+    """Antispam text filter handler."""
+    target = args['target']
+    text = args['text']
+    txf_settings = irc.get_service_option('antispam', 'textfilter',
+                                          TEXTFILTER_DEFAULTS)
+
+    if not txf_settings.get('enabled', False):
+        return
+
+    my_uid = sbot.uids.get(irc.name)
+
+    # XXX workaround for single-bot protocols like Clientbot
+    if irc.pseudoclient and not irc.has_cap('can-spawn-clients'):
+        my_uid = irc.pseudoclient.uid
+
+    if (not irc.connected.is_set()) or (not my_uid):
+        # Break if the network isn't ready.
+        log.debug("(%s) antispam.textfilters: skipping processing; network isn't ready", irc.name)
+        return
+    elif irc.is_internal_client(source):
+        # Ignore messages from our own clients.
+        log.debug("(%s) antispam.textfilters: skipping processing message from internal client %s", irc.name, source)
+        return
+    elif source not in irc.users:
+        log.debug("(%s) antispam.textfilters: ignoring message from non-user %s", irc.name, source)
+        return
+
+    if irc.is_channel(target):
+        channel_or_none = target
+        if target not in irc.channels or my_uid not in irc.channels[target].users:
+            # We're not monitoring this channel.
+            log.debug("(%s) antispam.textfilters: skipping processing message from channel %r we're not in", irc.name, target)
+            return
+    else:
+        channel_or_none = None
+        watch_pms = txf_settings.get('watch_pms', TEXTFILTER_DEFAULTS['watch_pms'])
+
+        if watch_pms == 'services':
+            if not irc.get_service_bot(target):
+                log.debug("(%s) antispam.textfilters: skipping processing; %r is not a service bot (watch_pms='services')", irc.name, target)
+                return
+        elif watch_pms == 'all':
+            log.debug("(%s) antispam.textfilters: checking all PMs (watch_pms='all')", irc.name)
+            pass
+        else:
+            # Not a channel.
+            log.debug("(%s) antispam.textfilters: skipping processing; %r is not a channel and watch_pms is disabled", irc.name, target)
+            return
+
+    # Merge together global and local textfilter lists.
+    txf_globs = set(conf.conf.get('antispam', {}).get('textfilter_globs', [])) | \
+                set(irc.serverdata.get('antispam_textfilter_globs', []))
+
+    punishment = txf_settings.get('punishment', TEXTFILTER_DEFAULTS['punishment']).lower()
+    reason = txf_settings.get('reason', TEXTFILTER_DEFAULTS['reason'])
+
+    if irc.get_service_option('antispam', 'strip_formatting', True):
+        text = utils.strip_irc_formatting(text)
+    if txf_settings.get('munge_unicode', TEXTFILTER_DEFAULTS['munge_unicode']):
+        text = str.translate(text, UNICODE_CHARMAP)
+
+    punished = False
+    for filterglob in txf_globs:
+        if utils.match_text(filterglob, text):
+            log.info("(%s) antispam: punishing %s => %s for text filter %r",
+                     irc.name,
+                     irc.get_friendly_name(source),
+                     irc.get_friendly_name(target),
+                     filterglob)
+            punished = _punish(irc, source, channel_or_none, punishment, reason)
+            break
+
+    return not punished  # Filter this message from relay, etc. if it triggered protection
+
+utils.add_hook(handle_textfilter, 'PRIVMSG', priority=999)
+utils.add_hook(handle_textfilter, 'NOTICE', priority=999)
+
+PARTQUIT_DEFAULTS = {
+    'watch_quits': True,
+    'watch_parts': True,
+    'part_filter_message': "Reason filtered",
+    'quit_filter_message': "Reason filtered",
+}
+def handle_partquit(irc, source, command, args):
+    """Antispam part/quit message filter."""
+    text = args.get('text')
+    pq_settings = irc.get_service_option('antispam', 'partquit',
+                                         PARTQUIT_DEFAULTS)
+
+    if not text:
+        return  # No text to match against
+    elif command == 'QUIT' and not pq_settings.get('watch_quits', True):
+        return  # Not enabled
+    elif command == 'PART' and not pq_settings.get('watch_parts', True):
+        return
+
+    # Merge together global and local partquit filter lists.
+    pq_globs = set(conf.conf.get('antispam', {}).get('partquit_globs', [])) | \
+               set(irc.serverdata.get('antispam_partquit_globs', []))
+    if not pq_globs:
+        return
+
+    for filterglob in pq_globs:
+        if utils.match_text(filterglob, text):
+            # For parts, also log the affected channels
+            if command == 'PART':
+                filtered_message = pq_settings.get('part_filter_message', PARTQUIT_DEFAULTS['part_filter_message'])
+                log.info('(%s) antispam: filtered part message from %s on %s due to part/quit filter glob %s',
+                         irc.name, irc.get_hostmask(source), ','.join(args['channels']), filterglob)
+            else:
+                filtered_message = pq_settings.get('quit_filter_message', PARTQUIT_DEFAULTS['quit_filter_message'])
+                log.info('(%s) antispam: filtered quit message from %s due to part/quit filter glob %s',
+                         irc.name, args['userdata'].nick, filterglob)
+            args['text'] = filtered_message
+            break
+utils.add_hook(handle_partquit, 'PART', priority=999)
+utils.add_hook(handle_partquit, 'QUIT', priority=999)

plugins/automode.py

@@ -3,213 +3,108 @@ automode.py - Provide simple channel ACL management by giving prefix modes to us
 hostmasks or exttargets.
 """
 import collections
-import threading
+import string
-import json
 
-from pylinkirc import utils, conf, world
+from pylinkirc import conf, structures, utils, world
+from pylinkirc.coremods import permissions
 from pylinkirc.log import log
 
 mydesc = ("The \x02Automode\x02 plugin provides simple channel ACL management by giving prefix modes "
           "to users matching hostmasks or exttargets.")
 
 # Register ourselves as a service.
-modebot = world.services.get("automode", utils.registerService("automode", desc=mydesc))
+modebot = utils.register_service("automode", default_nick="Automode", desc=mydesc)
 reply = modebot.reply
+error = modebot.error
 
 # Databasing variables.
-dbname = utils.getDatabaseName('automode')
+dbname = conf.get_database_name('automode')
-db = collections.defaultdict(dict)
+datastore = structures.JSONDataStore('automode', dbname, default_db=collections.defaultdict(dict))
-exportdb_timer = None
 
-save_delay = conf.conf['bot'].get('save_delay', 300)
+db = datastore.store
 
-def loadDB():
+# The default set of Automode permissions.
-    """Loads the Automode database, silently creating a new one if this fails."""
+default_permissions = {"$ircop": ['automode.manage.relay_owned', 'automode.sync.relay_owned',
-    global db
+                                  'automode.list']}
-    try:
-        with open(dbname, "r") as f:
-            db.update(json.load(f))
-    except (ValueError, IOError, OSError):
-        log.info("Automode: failed to load links database %s; creating a new one in "
-                 "memory.", dbname)
 
-def exportDB():
+def _join_db_channels(irc):
-    """Exports the automode database."""
-
-    log.debug("Automode: exporting database to %s.", dbname)
-    with open(dbname, 'w') as f:
-        # Pretty print the JSON output for better readability.
-        json.dump(db, f, indent=4)
-
-def scheduleExport(starting=False):
     """
-    Schedules exporting of the Automode database in a repeated loop.
+    Joins the Automode service client to channels on the current network in its DB.
     """
-    global exportdb_timer
+    if not irc.connected.is_set():
+        log.debug('(%s) _join_db_channels: aborting, network not ready yet', irc.name)
+        return
 
-    if not starting:
-        # Export the database, unless this is being called the first
-        # thing after start (i.e. DB has just been loaded).
-        exportDB()
-
-    exportdb_timer = threading.Timer(save_delay, scheduleExport)
-    exportdb_timer.name = 'Automode exportDB Loop'
-    exportdb_timer.start()
-
-def main(irc=None):
-    """Main function, called during plugin loading at start."""
-
-    # Load the automode database.
-    loadDB()
-
-    # Schedule periodic exports of the automode database.
-    scheduleExport(starting=True)
-
-    # Queue joins to all channels where Automode has entries.
     for entry in db:
         netname, channel = entry.split('#', 1)
         channel = '#' + channel
-        log.debug('automode: auto-joining %s on %s', channel, netname)
+        if netname == irc.name:
-        modebot.extra_channels[netname].add(channel)
+            modebot.add_persistent_channel(irc, 'automode', channel)
 
-        # This explicitly forces a join to connected networks (on plugin load, etc.).
+def main(irc=None):
-        mb_uid = modebot.uids.get(netname)
+    """Main function, called during plugin loading."""
-        if netname in world.networkobjects and mb_uid in world.networkobjects[netname].users:
-            remoteirc = world.networkobjects[netname]
-            remoteirc.proto.join(mb_uid, channel)
 
-            # Call a join hook manually so other plugins like relay can understand it.
+    # Load the automode database.
-            remoteirc.callHooks([mb_uid, 'PYLINK_AUTOMODE_JOIN', {'channel': channel, 'users': [mb_uid],
+    datastore.load()
-                                                                  'modes': remoteirc.channels[channel].modes,
-                                                                  'parse_as': 'JOIN'}])
 
-def die(sourceirc):
+    # Register our permissions.
+    permissions.add_default_permissions(default_permissions)
+
+    if irc:  # This was a reload.
+        for ircobj in world.networkobjects.values():
+            _join_db_channels(ircobj)
+
+def die(irc=None):
     """Saves the Automode database and quit."""
-    exportDB()
+    datastore.die()
+    permissions.remove_default_permissions(default_permissions)
+    utils.unregister_service('automode')
 
-    # Kill the scheduling for exports.
+def _check_automode_access(irc, uid, channel, command):
-    global exportdb_timer
+    """Checks the caller's access to Automode."""
-    if exportdb_timer:
+    # Automode defines the following permissions, where <command> is either "manage", "list",
-        log.debug("Automode: cancelling exportDB timer thread %s due to die()", threading.get_ident())
+    # "sync", "clear", "remotemanage", "remotelist", "remotesync", "remoteclear":
-        exportdb_timer.cancel()
+    # - automode.<command> OR automode.<command>.*: ability to <command> automode on all channels.
+    # - automode.<command>.relay_owned: ability to <command> automode on channels owned via Relay.
+    #   If Relay isn't loaded, this permission check FAILS.
+    # - automode.<command>.#channel: ability to <command> automode on the given channel.
+    # - automode.savedb: ability to save the automode DB.
+    log.debug('(%s) Automode: checking access for %s/%s for %s capability on %s', irc.name, uid,
+              irc.get_hostmask(uid), command, channel)
 
-    utils.unregisterService('automode')
+    baseperm = 'automode.%s' % command
-
-def setacc(irc, source, args):
-    """<channel> <mask> <mode list>
-
-    Assigns the given prefix mode characters to the given mask for the channel given. Extended targets are supported for masks - use this to your advantage!
-
-    Examples:
-    SET #channel *!*@localhost ohv
-    SET #channel $account v
-    SET #channel $oper:Network?Administrator qo
-    SET #staffchan $channel:#mainchan:op o
-    """
-    irc.checkAuthenticated(source, allowOper=False)
     try:
-        channel, mask, modes = args
+        # First, check the catch all and channel permissions.
-    except ValueError:
+        perms = [baseperm, baseperm+'.*', '%s.%s' % (baseperm, channel)]
-        reply(irc, "Error: Invalid arguments given. Needs 3: channel, mask, mode list.")
+        return permissions.check_permissions(irc, uid, perms)
-        return
+    except utils.NotAuthorizedError:
-    else:
+        if not command.startswith('remote'):
-        if not utils.isChannel(channel):
+            # Relay-based ACL checking only works with local calls.
-            reply(irc, "Error: Invalid channel name %s." % channel)
+            log.debug('(%s) Automode: falling back to automode.%s.relay_owned', irc.name, command)
-            return
+            permissions.check_permissions(irc, uid, [baseperm+'.relay_owned'], also_show=perms)
 
-        # Store channels case insensitively
+            relay = world.plugins.get('relay')
-        channel = irc.toLower(channel)
+            if relay is None:
-
+                raise utils.NotAuthorizedError("You are not authorized to use Automode when Relay is "
-    # Database entries for any network+channel pair are automatically created using
+                                               "disabled. You are missing one of the following "
-    # defaultdict. Note: string keys are used here instead of tuples so they can be
+                                               "permissions: %s or %s.%s" % (baseperm, baseperm, channel))
-    # exported easily as JSON.
+            elif (irc.name, channel) not in relay.db:
-    dbentry = db[irc.name+channel]
+                raise utils.NotAuthorizedError("The network you are on does not own the relay channel %s." % channel)
-
+            return True
-    # Otherwise, update the modes as is.
+        raise
-    dbentry[mask] = modes
-    reply(irc, "Done. \x02%s\x02 now has modes \x02%s\x02 in \x02%s\x02." % (mask, modes, channel))
-
-modebot.add_cmd(setacc, 'setaccess')
-modebot.add_cmd(setacc, 'set')
-modebot.add_cmd(setacc, featured=True)
-
-def delacc(irc, source, args):
-    """<channel> <mask>
-
-    Removes the Automode entry for the given mask on the given channel, if one exists.
-    """
-    irc.checkAuthenticated(source, allowOper=False)
-
-    try:
-        channel, mask = args
-        channel = irc.toLower(channel)
-    except ValueError:
-        reply(irc, "Error: Invalid arguments given. Needs 2: channel, mask")
-        return
-
-    dbentry = db.get(irc.name+channel)
-
-    if dbentry is None:
-        reply(irc, "Error: no Automode access entries exist for \x02%s\x02." % channel)
-        return
-
-    if mask in dbentry:
-        del dbentry[mask]
-        reply(irc, "Done. Removed the Automode access entry for \x02%s\x02 in \x02%s\x02." % (mask, channel))
-    else:
-        reply(irc, "Error: No Automode access entry for \x02%s\x02 exists in \x02%s\x02." % (mask, channel))
-
-    # Remove channels if no more entries are left.
-    if not dbentry:
-        log.debug("Automode: purging empty channel pair %s/%s", irc.name, channel)
-        del db[irc.name+channel]
-
-    return
-modebot.add_cmd(delacc, 'delaccess')
-modebot.add_cmd(delacc, 'del')
-modebot.add_cmd(delacc, featured=True)
-
-def listacc(irc, source, args):
-    """<channel>
-
-    Lists all Automode entries for the given channel."""
-    irc.checkAuthenticated(source)
-    try:
-        channel = irc.toLower(args[0])
-    except IndexError:
-        reply(irc, "Error: Invalid arguments given. Needs 1: channel.")
-        return
-    dbentry = db.get(irc.name+channel)
-    if not dbentry:
-        reply(irc, "Error: No Automode access entries exist for \x02%s\x02." % channel)
-        return
-
-    else:
-        # Iterate over all entries and print them. Do this in private to prevent channel
-        # floods.
-        reply(irc, "Showing Automode entries for \x02%s\x02:" % channel, private=True)
-        for entrynum, entry in enumerate(dbentry.items(), start=1):
-            mask, modes = entry
-            reply(irc, "[%s] \x02%s\x02 has modes +\x02%s\x02" % (entrynum, mask, modes), private=True)
-        reply(irc, "End of Automode entries list.", private=True)
-modebot.add_cmd(listacc, featured=True)
-modebot.add_cmd(listacc, 'listaccess')
-
-def save(irc, source, args):
-    """takes no arguments.
-
-    Saves the Automode database to disk."""
-    irc.checkAuthenticated(source)
-    exportDB()
-    reply(irc, 'Done.')
-modebot.add_cmd(save)
 
 def match(irc, channel, uids=None):
     """
-    Automode matcher engine.
+    Set modes on matching users. If uids is not given, check all users in the channel and give
+    them modes as needed.
     """
+    if isinstance(channel, int) or str(channel).startswith(tuple(string.digits)):
+        channel = '#' + str(channel)  # Mangle channels on networks where they're stored as an ID
     dbentry = db.get(irc.name+channel)
-    if dbentry is None:
+    if not irc.has_cap('has-irc-modes'):
+        log.debug('(%s) automode: skipping match() because IRC modes are not supported on this protocol', irc.name)
+        return
+    elif dbentry is None:
         return
 
     modebot_uid = modebot.uids.get(irc.name)
@@ -222,76 +117,39 @@ def match(irc, channel, uids=None):
 
     for mask, modes in dbentry.items():
         for uid in uids:
-            if irc.matchHost(mask, uid):
+            if irc.match_host(mask, uid):
                 # User matched a mask. Filter the mode list given to only those that are valid
                 # prefix mode characters.
                 outgoing_modes += [('+'+mode, uid) for mode in modes if mode in irc.prefixmodes]
                 log.debug("(%s) automode: Filtered mode list of %s to %s (protocol:%s)",
                           irc.name, modes, outgoing_modes, irc.protoname)
 
-    # If the Automode bot is missing, send the mode through the PyLink server.
+    if outgoing_modes:
-    if modebot_uid not in irc.users:
+        # If the Automode bot is missing, send the mode through the PyLink server.
-        modebot_uid = irc.sid
+        if modebot_uid not in irc.users:
+            modebot_uid = irc.sid
 
-    log.debug("(%s) automode: sending modes from modebot_uid %s",
+        log.debug("(%s) automode: sending modes from modebot_uid %s",
-              irc.name, modebot_uid)
+                  irc.name, modebot_uid)
 
-    irc.proto.mode(modebot_uid, channel, outgoing_modes)
+        irc.mode(modebot_uid, channel, outgoing_modes)
 
-    # Create a hook payload to support plugins like relay.
+        # Create a hook payload to support plugins like relay.
-    irc.callHooks([modebot_uid, 'AUTOMODE_MODE',
+        irc.call_hooks([modebot_uid, 'AUTOMODE_MODE',
-                  {'target': channel, 'modes': outgoing_modes, 'parse_as': 'MODE'}])
+                      {'target': channel, 'modes': outgoing_modes, 'parse_as': 'MODE'}])
 
-def syncacc(irc, source, args):
+def handle_endburst(irc, source, command, args):
-    """<channel>
+    """ENDBURST hook handler - used to join the Automode service to channels where it has entries."""
-
+    if source == irc.uplink:
-    Syncs Automode access lists to the channel.
+        _join_db_channels(irc)
-    """
+utils.add_hook(handle_endburst, 'ENDBURST')
-    irc.checkAuthenticated(source, allowOper=False)
-    try:
-        channel = irc.toLower(args[0])
-    except IndexError:
-        reply(irc, "Error: Invalid arguments given. Needs 1: channel.")
-        return
-
-    match(irc, channel)
-
-    reply(irc, 'Done.')
-
-modebot.add_cmd(syncacc, featured=True)
-modebot.add_cmd(syncacc, 'sync')
-modebot.add_cmd(syncacc, 'syncaccess')
-
-def clearacc(irc, source, args):
-    """<channel>
-
-    Removes all Automode entries for the given channel.
-    """
-    irc.checkAuthenticated(source, allowOper=False)
-
-    try:
-        channel = irc.toLower(args[0])
-    except IndexError:
-        reply(irc, "Error: Invalid arguments given. Needs 1: channel.")
-        return
-
-    if db.get(irc.name+channel):
-        log.debug("Automode: purging channel pair %s/%s", irc.name, channel)
-        del db[irc.name+channel]
-        reply(irc, "Done. Removed all Automode access entries for \x02%s\x02." %  channel)
-    else:
-        reply(irc, "Error: No Automode access entries exist for \x02%s\x02." % channel)
-
-modebot.add_cmd(clearacc, 'clearaccess')
-modebot.add_cmd(clearacc, 'clear')
-modebot.add_cmd(clearacc, featured=True)
 
 def handle_join(irc, source, command, args):
     """
     Automode JOIN listener. This sets modes accordingly if the person joining matches a mask in the
     ACL.
     """
-    channel = irc.toLower(args['channel'])
+    channel = irc.to_lower(args['channel'])
     match(irc, channel, args['users'])
 
 utils.add_hook(handle_join, 'JOIN')
@@ -308,3 +166,217 @@ def handle_services_login(irc, source, command, args):
 
 utils.add_hook(handle_services_login, 'CLIENT_SERVICES_LOGIN')
 utils.add_hook(handle_services_login, 'PYLINK_RELAY_SERVICES_LOGIN')
+
+def _get_channel_pair(irc, source, chanpair, perm=None):
+    """
+    Fetches the network and channel given a channel pair, also optionally checking the caller's permissions.
+    """
+    log.debug('(%s) Looking up chanpair %s', irc.name, chanpair)
+
+    if '#' not in chanpair and chanpair.startswith(tuple(string.digits)):
+        chanpair = '#' + chanpair  # Mangle channels on networks where they're stored by ID
+
+    try:
+        network, channel = chanpair.split('#', 1)
+    except ValueError:
+        raise ValueError("Invalid channel pair %r" % chanpair)
+    channel = '#' + channel
+    channel = irc.to_lower(channel)
+
+    if network:
+        ircobj = world.networkobjects.get(network)
+    else:
+        ircobj = irc
+
+    if not ircobj:
+        raise ValueError("Unknown network %s" % network)
+
+    if perm is not None:
+        # Only check for permissions if we're told to and the irc object exists.
+        if ircobj.name != irc.name:
+            perm = 'remote' + perm
+
+        _check_automode_access(irc, source, channel, perm)
+
+    return (ircobj, channel)
+
+def setacc(irc, source, args):
+    """<channel/chanpair> <mask> <mode list>
+
+    Assigns the given prefix mode characters to the given mask for the channel given. Extended targets are supported for masks - use this to your advantage!
+
+    Channel pairs are also supported (for operations on remote channels), using the form "network#channel".
+
+
+    Examples:
+
+    \x02SETACC #channel *!*@localhost ohv
+
+    \x02SETACC #channel $account v
+
+    \x02SETACC othernet#channel $ircop:Network?Administrator qo
+
+    \x02SETACC #staffchan $channel:#mainchan:op o
+    """
+    if not irc.has_cap('has-irc-modes'):
+        error(irc, "IRC style modes are not supported on this protocol.")
+        return
+
+    try:
+        chanpair, mask, modes = args
+    except ValueError:
+        error(irc, "Invalid arguments given. Needs 3: channel, mask, mode list.")
+        return
+    else:
+        ircobj, channel = _get_channel_pair(irc, source, chanpair, perm='manage')
+
+    # Database entries for any network+channel pair are automatically created using
+    # defaultdict. Note: string keys are used here instead of tuples so they can be
+    # exported easily as JSON.
+    dbentry = db[ircobj.name+channel]
+
+    modes = modes.lstrip('+')  # remove extraneous leading +'s
+    dbentry[mask] = modes
+    log.info('(%s) %s set modes +%s for %s on %s', ircobj.name, irc.get_hostmask(source), modes, mask, channel)
+    reply(irc, "Done. \x02%s\x02 now has modes \x02+%s\x02 in \x02%s\x02." % (mask, modes, channel))
+
+    # Join the Automode bot to the channel persistently.
+    modebot.add_persistent_channel(ircobj, 'automode', channel)
+
+modebot.add_cmd(setacc, aliases=('setaccess', 'set'), featured=True)
+
+def delacc(irc, source, args):
+    """<channel/chanpair> <mask or range string>
+
+    Removes the Automode entry for the given mask or range string, if they exist.
+
+    Range strings are indices (entry numbers) or ranges of them joined together with commas: e.g.
+    "1", "2-10", "1,3,5-8". Entry numbers are shown by LISTACC.
+    """
+    try:
+        chanpair, mask = args
+    except ValueError:
+        error(irc, "Invalid arguments given. Needs 2: channel, mask")
+        return
+    else:
+        ircobj, channel = _get_channel_pair(irc, source, chanpair, perm='manage')
+
+    dbentry = db.get(ircobj.name+channel)
+
+    if dbentry is None:
+        error(irc, "No Automode access entries exist for \x02%s\x02." % channel)
+        return
+
+    if mask in dbentry:
+        del dbentry[mask]
+        log.info('(%s) %s removed modes for %s on %s', ircobj.name, irc.get_hostmask(source), mask, channel)
+        reply(irc, "Done. Removed the Automode access entry for \x02%s\x02 in \x02%s\x02." % (mask, channel))
+    else:
+        # Treat the mask as a range string.
+        try:
+            new_keys = utils.remove_range(mask, sorted(dbentry.keys()))
+        except ValueError:
+            error(irc, "No Automode access entry for \x02%s\x02 exists in \x02%s\x02." % (mask, channel))
+            return
+
+        # XXX: Automode entries are actually unordered: what we're actually doing is sorting the keys
+        # by name into a list, running remove_range on that, and removing the difference.
+        removed = []
+        source_host = irc.get_hostmask(source)
+        for mask_entry in dbentry.copy():
+            if mask_entry not in new_keys:
+                del dbentry[mask_entry]
+                log.info('(%s) %s removed modes for %s on %s', ircobj.name, source_host, mask_entry, channel)
+                removed.append(mask_entry)
+
+        reply(irc, 'Done. Removed \x02%d\x02 entries on \x02%s\x02: %s' % (len(removed), channel, ', '.join(removed)))
+
+    # Remove channels if no more entries are left.
+    if not dbentry:
+        log.debug("Automode: purging empty channel pair %s/%s", ircobj.name, channel)
+        del db[ircobj.name+channel]
+        modebot.remove_persistent_channel(ircobj, 'automode', channel)
+
+modebot.add_cmd(delacc, aliases=('delaccess', 'del'), featured=True)
+
+def listacc(irc, source, args):
+    """<channel/chanpair>
+
+    Lists all Automode entries for the given channel."""
+    try:
+        chanpair = args[0]
+    except IndexError:
+        error(irc, "Invalid arguments given. Needs 1: channel.")
+        return
+    else:
+        ircobj, channel = _get_channel_pair(irc, source, chanpair, perm='list')
+
+    dbentry = db.get(ircobj.name+channel)
+    if not dbentry:
+        error(irc, "No Automode access entries exist for \x02%s\x02." % channel)
+        return
+
+    else:
+        # Iterate over all entries and print them. Do this in private to prevent channel
+        # floods.
+        reply(irc, "Showing Automode entries for \x02%s\x02:" % channel, private=True)
+        for entrynum, entry in enumerate(sorted(dbentry.items()), start=1):
+            mask, modes = entry
+            reply(irc, "[%s] \x02%s\x02 has modes +\x02%s\x02" % (entrynum, mask, modes), private=True)
+        reply(irc, "End of Automode entries list.", private=True)
+
+modebot.add_cmd(listacc, featured=True, aliases=('listaccess',))
+
+def save(irc, source, args):
+    """takes no arguments.
+
+    Saves the Automode database to disk."""
+    permissions.check_permissions(irc, source, ['automode.savedb'])
+    datastore.save()
+    reply(irc, 'Done.')
+
+modebot.add_cmd(save)
+
+def syncacc(irc, source, args):
+    """<channel/chanpair>
+
+    Syncs Automode access lists to the channel.
+    """
+    try:
+        chanpair = args[0]
+    except IndexError:
+        error(irc, "Invalid arguments given. Needs 1: channel.")
+        return
+    else:
+        ircobj, channel = _get_channel_pair(irc, source, chanpair, perm='sync')
+
+    log.info('(%s) %s synced modes on %s', ircobj.name, irc.get_hostmask(source), channel)
+    match(ircobj, channel)
+
+    reply(irc, 'Done.')
+
+modebot.add_cmd(syncacc, featured=True, aliases=('sync', 'syncaccess'))
+
+def clearacc(irc, source, args):
+    """<channel>
+
+    Removes all Automode entries for the given channel.
+    """
+
+    try:
+        chanpair = args[0]
+    except IndexError:
+        error(irc, "Invalid arguments given. Needs 1: channel.")
+        return
+    else:
+        ircobj, channel = _get_channel_pair(irc, source, chanpair, perm='clear')
+
+    if db.get(ircobj.name+channel):
+        del db[ircobj.name+channel]
+        log.info('(%s) %s cleared modes on %s', ircobj.name, irc.get_hostmask(source), channel)
+        reply(irc, "Done. Removed all Automode access entries for \x02%s\x02." % channel)
+        modebot.remove_persistent_channel(ircobj, 'automode', channel)
+    else:
+        error(irc, "No Automode access entries exist for \x02%s\x02." % channel)
+
+modebot.add_cmd(clearacc, aliases=('clearaccess', 'clear'), featured=True)

plugins/bots.py

@@ -3,99 +3,133 @@ bots.py: Spawn virtual users/bots on a PyLink server and make them interact
 with things.
 """
 from pylinkirc import utils
-from pylinkirc.log import log
+from pylinkirc.coremods import permissions
+
 
 @utils.add_cmd
 def spawnclient(irc, source, args):
     """<nick> <ident> <host>
 
-    Admin-only. Spawns the specified PseudoClient on the PyLink server.
+    Spawns the specified client on the PyLink server.
     Note: this doesn't check the validity of any fields you give it!"""
-    irc.checkAuthenticated(source, allowOper=False)
+
+    if not irc.has_cap('can-spawn-clients'):
+        irc.error("This network does not support client spawning.")
+        return
+
+    permissions.check_permissions(irc, source, ['bots.spawnclient'])
     try:
         nick, ident, host = args[:3]
     except ValueError:
-        irc.reply("Error: Not enough arguments. Needs 3: nick, user, host.")
+        irc.error("Not enough arguments. Needs 3: nick, user, host.")
         return
-    irc.proto.spawnClient(nick, ident, host, manipulatable=True)
+    irc.spawn_client(nick, ident, host, manipulatable=True)
+    irc.reply("Done.")
 
 @utils.add_cmd
 def quit(irc, source, args):
     """<target> [<reason>]
 
-    Admin-only. Quits the PyLink client with nick <target>, if one exists."""
+    Quits the PyLink client with nick <target>, if one exists."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['bots.quit'])
 
     try:
         nick = args[0]
     except IndexError:
-        irc.reply("Error: Not enough arguments. Needs 1-2: nick, reason (optional).")
+        irc.error("Not enough arguments. Needs 1-2: nick, reason (optional).")
-        return
-    if irc.pseudoclient.uid == irc.nickToUid(nick):
-        irc.reply("Error: Cannot quit the main PyLink PseudoClient!")
         return
 
-    u = irc.nickToUid(nick)
+    u = irc.nick_to_uid(nick, filterfunc=irc.is_internal_client)
-
+    if u is None:
-    quitmsg =  ' '.join(args[1:]) or 'Client Quit'
+        irc.error("Unknown user %r" % nick)
-
-    if not irc.isManipulatableClient(u):
-        irc.reply("Error: Cannot force quit a protected PyLink services client.")
         return
 
-    irc.proto.quit(u, quitmsg)
+    if irc.pseudoclient.uid == u:
-    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_QUIT', {'text': quitmsg, 'parse_as': 'QUIT'}])
+        irc.error("Cannot quit the main PyLink client!")
+        return
+
+    quitmsg = ' '.join(args[1:]) or 'Client Quit'
+
+    if not irc.is_manipulatable_client(u):
+        irc.error("Cannot force quit a protected PyLink services client.")
+        return
+
+    irc.quit(u, quitmsg)
+    irc.reply("Done.")
+    irc.call_hooks([u, 'PYLINK_BOTSPLUGIN_QUIT', {'text': quitmsg, 'parse_as': 'QUIT'}])
 
 def joinclient(irc, source, args):
-    """[<target>] <channel1>,[<channel2>], etc.
+    """[<target>] <channel1>[,<channel2>,<channel3>,...]
 
-    Admin-only. Joins <target>, the nick of a PyLink client, to a comma-separated list of channels. If <target> is not given, it defaults to the main PyLink client."""
+    Joins <target>, the nick of a PyLink client, to a comma-separated list of channels.
-    irc.checkAuthenticated(source, allowOper=False)
+    If <target> is not given, it defaults to the main PyLink client.
+
+    For the channel arguments, prefixes can also be specified to join the given client with
+    (e.g. @#channel will join the client with op, while ~@#channel will join it with +qo.
+    """
+    permissions.check_permissions(irc, source, ['bots.join', 'bots.joinclient'])
 
     try:
         # Check if the first argument is an existing PyLink client. If it is not,
         # then assume that the first argument was actually the channels being joined.
-        u = irc.nickToUid(args[0])
+        u = irc.nick_to_uid(args[0], filterfunc=irc.is_internal_client)
 
-        if not irc.isInternalClient(u):  # First argument isn't one of our clients
+        if u is None:  # First argument isn't one of our clients
             raise IndexError
 
         clist = args[1]
-    except IndexError:  # No nick was given; shift arguments one to the left.
+    except IndexError:  # No valid nick was given; shift arguments one to the left.
         u = irc.pseudoclient.uid
         try:
             clist = args[0]
         except IndexError:
-            irc.reply("Error: Not enough arguments. Needs 1-2: nick (optional), comma separated list of channels.")
+            irc.error("Not enough arguments. Needs 1-2: nick (optional), comma separated list of channels.")
             return
 
     clist = clist.split(',')
     if not clist:
-        irc.reply("Error: No valid channels given.")
+        irc.error("No valid channels given.")
         return
 
-    if not irc.isManipulatableClient(u):
+    if not (irc.is_manipulatable_client(u) or irc.get_service_bot(u)):
-        irc.reply("Error: Cannot force join a protected PyLink services client.")
+        irc.error("Cannot force join a protected PyLink services client.")
         return
 
+    prefix_to_mode = {v: k for k, v in irc.prefixmodes.items()}
     for channel in clist:
-        if not utils.isChannel(channel):
+        real_channel = channel.lstrip(''.join(prefix_to_mode))
-            irc.reply("Error: Invalid channel name %r." % channel)
+        # XXX we need a better way to do this, but only the other option I can think of is regex...
-            return
+        prefixes = channel[:len(channel)-len(real_channel)]
-        irc.proto.join(u, channel)
+        joinmodes = ''.join(prefix_to_mode[prefix] for prefix in prefixes)
 
-        # Call a join hook manually so other plugins like relay can understand it.
+        if not irc.is_channel(real_channel):
-        irc.callHooks([u, 'PYLINK_BOTSPLUGIN_JOIN', {'channel': channel, 'users': [u],
+            irc.error("Invalid channel name %r." % real_channel)
-                                                'modes': irc.channels[channel].modes,
+            return
-                                                'parse_as': 'JOIN'}])
+
+        # join() doesn't support prefixes.
+        if prefixes:
+            irc.sjoin(irc.sid, real_channel, [(joinmodes, u)])
+        else:
+            irc.join(u, real_channel)
+
+        try:
+            modes = irc.channels[real_channel].modes
+        except KeyError:
+            modes = []
+
+        # Signal the join to other plugins
+        irc.call_hooks([u, 'PYLINK_BOTSPLUGIN_JOIN', {'channel': real_channel, 'users': [u],
+                                                     'modes': modes, 'parse_as': 'JOIN'}])
+    irc.reply("Done.")
 utils.add_cmd(joinclient, name='join')
 
 @utils.add_cmd
 def nick(irc, source, args):
     """[<target>] <newnick>
 
-    Admin-only. Changes the nick of <target>, a PyLink client, to <newnick>. If <target> is not given, it defaults to the main PyLink client."""
+    Changes the nick of <target>, a PyLink client, to <newnick>. If <target> is not given, it defaults to the main PyLink client."""
-    irc.checkAuthenticated(source, allowOper=False)
+
+    permissions.check_permissions(irc, source, ['bots.nick'])
 
     try:
         nick = args[0]
@@ -105,31 +139,32 @@ def nick(irc, source, args):
             nick = irc.pseudoclient.nick
             newnick = args[0]
         except IndexError:
-            irc.reply("Error: Not enough arguments. Needs 1-2: nick (optional), newnick.")
+            irc.error("Not enough arguments. Needs 1-2: nick (optional), newnick.")
             return
-    u = irc.nickToUid(nick)
+    u = irc.nick_to_uid(nick, filterfunc=irc.is_internal_client)
 
     if newnick in ('0', u):  # Allow /nick 0 to work
         newnick = u
 
-    elif not utils.isNick(newnick):
+    elif not irc.is_nick(newnick):
-        irc.reply('Error: Invalid nickname %r.' % newnick)
+        irc.error('Invalid nickname %r.' % newnick)
         return
 
-    elif not irc.isManipulatableClient(u):
+    elif not (irc.is_manipulatable_client(u) or irc.get_service_bot(u)):
-        irc.reply("Error: Cannot force nick changes for a protected PyLink services client.")
+        irc.error("Cannot force nick changes for a protected PyLink services client.")
         return
 
-    irc.proto.nick(u, newnick)
+    irc.nick(u, newnick)
-    # Ditto above: manually send a NICK change hook payload to other plugins.
+    irc.reply("Done.")
-    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_NICK', {'newnick': newnick, 'oldnick': nick, 'parse_as': 'NICK'}])
+    # Signal the nick change to other plugins
+    irc.call_hooks([u, 'PYLINK_BOTSPLUGIN_NICK', {'newnick': newnick, 'oldnick': nick, 'parse_as': 'NICK'}])
 
 @utils.add_cmd
 def part(irc, source, args):
     """[<target>] <channel1>,[<channel2>],... [<reason>]
 
-    Admin-only. Parts <target>, the nick of a PyLink client, from a comma-separated list of channels. If <target> is not given, it defaults to the main PyLink client."""
+    Parts <target>, the nick of a PyLink client, from a comma-separated list of channels. If <target> is not given, it defaults to the main PyLink client."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['bots.part'])
 
     try:
         nick = args[0]
@@ -139,8 +174,8 @@ def part(irc, source, args):
 
         # First, check if the first argument is an existing PyLink client. If it is not,
         # then assume that the first argument was actually the channels being parted.
-        u = irc.nickToUid(nick)
+        u = irc.nick_to_uid(nick, filterfunc=irc.is_internal_client)
-        if not irc.isInternalClient(u):  # First argument isn't one of our clients
+        if u is None:  # First argument isn't one of our clients
             raise IndexError
 
     except IndexError:  # No nick was given; shift arguments one to the left.
@@ -149,33 +184,33 @@ def part(irc, source, args):
         try:
             clist = args[0]
         except IndexError:
-            irc.reply("Error: Not enough arguments. Needs 1-2: nick (optional), comma separated list of channels.")
+            irc.error("Not enough arguments. Needs 1-2: nick (optional), comma separated list of channels.")
             return
         reason = ' '.join(args[1:])
 
     clist = clist.split(',')
     if not clist:
-        irc.reply("Error: No valid channels given.")
+        irc.error("No valid channels given.")
         return
 
-    if not irc.isManipulatableClient(u):
+    if not (irc.is_manipulatable_client(u) or irc.get_service_bot(u)):
-        irc.reply("Error: Cannot force part a protected PyLink services client.")
+        irc.error("Cannot force part a protected PyLink services client.")
         return
 
     for channel in clist:
-        if not utils.isChannel(channel):
+        if not irc.is_channel(channel):
-            irc.reply("Error: Invalid channel name %r." % channel)
+            irc.error("Invalid channel name %r." % channel)
             return
-        irc.proto.part(u, channel, reason)
+        irc.part(u, channel, reason)
 
-    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_PART', {'channels': clist, 'text': reason, 'parse_as': 'PART'}])
+    irc.reply("Done.")
+    irc.call_hooks([u, 'PYLINK_BOTSPLUGIN_PART', {'channels': clist, 'text': reason, 'parse_as': 'PART'}])
 
-@utils.add_cmd
 def msg(irc, source, args):
     """[<source>] <target> <text>
 
-    Admin-only. Sends message <text> from <source>, where <source> is the nick of a PyLink client. If <source> is not given, it defaults to the main PyLink client."""
+    Sends message <text> from <source>, where <source> is the nick of a PyLink client. If <source> is not given, it defaults to the main PyLink client."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['bots.msg'])
 
     # Because we want the source nick to be optional, this argument parsing gets a bit tricky.
     try:
@@ -185,34 +220,48 @@ def msg(irc, source, args):
 
         # First, check if the first argument is an existing PyLink client. If it is not,
         # then assume that the first argument was actually the message TARGET.
-        sourceuid = irc.nickToUid(msgsource)
+        sourceuid = irc.nick_to_uid(msgsource, filterfunc=irc.is_internal_client)
-        if not irc.isInternalClient(sourceuid):  # First argument isn't one of our clients
+
+        if sourceuid is None or not text:  # First argument isn't one of our clients
             raise IndexError
 
-        if not text:
-            raise IndexError
     except IndexError:
         try:
             sourceuid = irc.pseudoclient.uid
             target = args[0]
             text = ' '.join(args[1:])
         except IndexError:
-            irc.reply('Error: Not enough arguments. Needs 2-3: source nick (optional), target, text.')
+            irc.error('Not enough arguments. Needs 2-3: source nick (optional), target, text.')
             return
 
     if not text:
-        irc.reply('Error: No text given.')
+        irc.error('No text given.')
         return
 
-    if not utils.isChannel(target):
+    try:
-        # Convert nick of the message target to a UID, if the target isn't a channel
+        int_u = int(target)
-        real_target = irc.nickToUid(target)
+    except:
-        if real_target is None:  # Unknown target user, if target isn't a valid channel name
+        int_u = None
-            irc.reply('Error: Unknown user %r.' % target)
+
+    if int_u and int_u in irc.users:
+        real_target = int_u  # Some protocols use numeric UIDs
+    elif target in irc.users:
+        real_target = target
+    elif not irc.is_channel(target):
+        # Convert nick of the message target to a UID, if the target isn't a channel or UID
+        potential_targets = irc.nick_to_uid(target, multi=True)
+        if not potential_targets:  # Unknown target user, if target isn't a valid channel name
+            irc.error('Unknown user %r.' % target)
             return
+        elif len(potential_targets) > 1:
+            irc.error('Multiple users with the nick %r found: please select the right UID: %s' % (target, str(potential_targets)))
+            return
+        else:
+            real_target = potential_targets[0]
     else:
         real_target = target
 
-    irc.proto.message(sourceuid, real_target, text)
+    irc.message(sourceuid, real_target, text)
-    irc.callHooks([sourceuid, 'PYLINK_BOTSPLUGIN_MSG', {'target': real_target, 'text': text, 'parse_as': 'PRIVMSG'}])
+    irc.reply("Done.")
-utils.add_cmd(msg, 'say')
+    irc.call_hooks([sourceuid, 'PYLINK_BOTSPLUGIN_MSG', {'target': real_target, 'text': text, 'parse_as': 'PRIVMSG'}])
+utils.add_cmd(msg, aliases=('say',))

plugins/changehost.py

@@ -1,43 +1,52 @@
 """
 Changehost plugin - automatically changes the hostname of matching users.
 """
-from pylinkirc import utils, world, conf
-from pylinkirc.log import log
-
 import string
 
-# ircmatch library from https://github.com/mammon-ircd/ircmatch
+from pylinkirc import conf, utils, world
-# (pip install ircmatch)
+from pylinkirc.coremods import permissions
-import ircmatch
+from pylinkirc.log import log
 
 # Characters allowed in a hostname.
 allowed_chars = string.ascii_letters + '-./:' + string.digits
 
-def _changehost(irc, target, args):
+def _changehost(irc, target):
     changehost_conf = conf.conf.get("changehost")
 
-    if not changehost_conf:
+    if target not in irc.users:
-        log.warning("(%s) Missing 'changehost:' configuration block; "
-                    "Changehost will not function correctly!", irc.name)
         return
-    elif irc.name not in changehost_conf.get('enabled_nets'):
+    elif irc.is_internal_client(target):
+        log.debug('(%s) Skipping changehost on internal client %s', irc.name, target)
+        return
+
+    if irc.name not in changehost_conf.get('enabled_nets') and not irc.serverdata.get('changehost_enable'):
         # We're not enabled on the network, break.
         return
 
-    changehost_hosts = changehost_conf.get('hosts')
+    match_ip = irc.get_service_option('changehost', 'match_ip', default=False)
+    match_realhosts = irc.get_service_option('changehost', 'match_realhosts', default=False)
+
+    changehost_hosts = irc.get_service_options('changehost', 'hosts', dict)
     if not changehost_hosts:
-        log.warning("(%s) No hosts were defined in changehost::hosts; "
-                    "Changehost will not function correctly!", irc.name)
         return
 
-    args = args.copy()
+    args = irc.users[target].get_fields()
+
+    # $host is explicitly forbidden by default because it can cause recursive
+    # loops when IP or real host masks are used to match a target. vHost
+    # updates do not affect these fields, so any further host application will
+    # cause the vHost to grow rapidly in size.
+    # That said, it is possible to get away with this expansion if you're
+    # careful enough, and that's why this hidden option exists.
     if not changehost_conf.get('force_host_expansion'):
        del args['host']
 
     log.debug('(%s) Changehost args: %s', irc.name, args)
 
     for host_glob, host_template in changehost_hosts.items():
-        if irc.matchHost(host_glob, target):
+        log.debug('(%s) Changehost: checking mask %s', irc.name, host_glob)
+        if irc.match_host(host_glob, target, ip=match_ip, realhost=match_realhosts):
+            log.debug('(%s) Changehost matched mask %s', irc.name, host_glob)
             # This uses template strings for simple substitution:
             # https://docs.python.org/3/library/string.html#template-strings
             template = string.Template(host_template)
@@ -45,16 +54,6 @@ def _changehost(irc, target, args):
             # Substitute using the fields provided the hook data. This means
             # that the following variables are available for substitution:
             # $uid, $ts, $nick, $realhost, $ident, and $ip.
-
-            # $host is explicitly forbidden by default because it can cause
-            # recursive loops when IP or real host masks are used to match a
-            # target. vHost updates do not affect these fields, so any further
-            # execution of 'applyhosts' will cause $host to expand again to
-            # the user's new host, causing the vHost to grow rapidly in size.
-            # That said, it is possible to get away with this expansion if
-            # you're careful with what you're doing, and that is why this
-            # hidden option exists. -GLolol
-
             try:
                 new_host = template.substitute(args)
             except KeyError as e:
@@ -66,7 +65,9 @@ def _changehost(irc, target, args):
                 if char not in allowed_chars:
                     new_host = new_host.replace(char, '-')
 
-            irc.proto.updateClient(target, 'HOST', new_host)
+            # Only send a host change if something has changed
+            if new_host != irc.users[target].host:
+                irc.update_client(target, 'HOST', new_host)
 
             # Only operate on the first match.
             break
@@ -77,25 +78,57 @@ def handle_uid(irc, sender, command, args):
     """
 
     target = args['uid']
-    _changehost(irc, target, args)
+    _changehost(irc, target)
-
 utils.add_hook(handle_uid, 'UID')
 
+def handle_chghost(irc, sender, command, args):
+    """
+    Handles incoming CHGHOST requests for optional host-change enforcement.
+    """
+    changehost_conf = conf.conf.get("changehost", {})
+
+    target = args['target']
+
+    if (not irc.is_internal_client(sender)) and (not irc.is_internal_server(sender)):
+        if irc.name in changehost_conf.get('enforced_nets', []) or irc.serverdata.get('changehost_enforce'):
+            log.debug('(%s) Enforce for network is on, re-checking host for target %s/%s',
+                      irc.name, target, irc.get_friendly_name(target))
+
+            for ex in irc.get_service_options("changehost", "enforce_exceptions", list):
+                if irc.match_host(ex, target):
+                    log.debug('(%s) Skipping host change for target %s; they are exempted by mask %s',
+                              irc.name, target, ex)
+                    return
+
+            userobj = irc.users.get(target)
+            if userobj:
+                _changehost(irc, target)
+utils.add_hook(handle_chghost, 'CHGHOST')
+
+def handle_svslogin(irc, sender, command, args):
+    """
+    Handles services account changes for changehost.
+    """
+    _changehost(irc, sender)
+utils.add_hook(handle_svslogin, 'CLIENT_SERVICES_LOGIN')
+
 @utils.add_cmd
 def applyhosts(irc, sender, args):
     """[<network>]
 
     Applies all configured hosts for users on the given network, or the current network if none is specified."""
 
+    permissions.check_permissions(irc, sender, ['changehost.applyhosts'])
+
     try:  # Try to get network from the command line.
         network = world.networkobjects[args[0]]
     except IndexError:  # No network was given
         network = irc
     except KeyError:  # Unknown network
-        irc.reply("Error: Unknown network '%s'." % network)
+        irc.error("Unknown network '%s'." % network)
         return
 
-    for user, userdata in network.users.copy().items():
+    for user in network.users.copy():
-        _changehost(network, user, userdata.__dict__)
+        _changehost(network, user)
 
     irc.reply("Done.")

plugins/commands.py

@@ -1,88 +1,222 @@
 # commands.py: base PyLink commands
-from time import ctime
+import sys
+import time
 
-from pylinkirc import utils, __version__, world, real_version
+from pylinkirc import __version__, conf, real_version, utils, world
-from pylinkirc.log import log
+from pylinkirc.coremods import permissions
+from pylinkirc.coremods.login import pwd_context
+
+default_permissions = {"*!*@*": ['commands.status', 'commands.showuser', 'commands.showchan', 'commands.shownet']}
+
+def main(irc=None):
+    """Commands plugin main function, called on plugin load."""
+    # Register our permissions.
+    permissions.add_default_permissions(default_permissions)
+
+def die(irc=None):
+    """Commands plugin die function, called on plugin unload."""
+    permissions.remove_default_permissions(default_permissions)
 
 @utils.add_cmd
 def status(irc, source, args):
     """takes no arguments.
 
     Returns your current PyLink login status."""
+    permissions.check_permissions(irc, source, ['commands.status'])
     identified = irc.users[source].account
     if identified:
         irc.reply('You are identified as \x02%s\x02.' % identified)
     else:
         irc.reply('You are not identified as anyone.')
-    irc.reply('Operator access: \x02%s\x02' % bool(irc.isOper(source)))
+    irc.reply('Operator access: \x02%s\x02' % bool(irc.is_oper(source)))
 
 _none = '\x1D(none)\x1D'
+_notavail = '\x1DN/A\x1D'
+def _do_showuser(irc, source, u):
+    """Helper function for showuser."""
+    # Some protocol modules store UIDs as ints; make sure we check for that.
+    try:
+        int_u = int(u)
+    except ValueError:
+        pass
+    else:
+        if int_u in irc.users:
+            u = int_u
+
+    # Only show private info if the person is calling 'showuser' on themselves,
+    # or is an oper.
+    verbose = irc.is_oper(source) or u == source
+
+    if u not in irc.users:
+        irc.error('Unknown user %r.' % u)
+        return
+
+    f = lambda s: irc.reply('  ' + s, private=True)
+
+    userobj = irc.users[u]
+    irc.reply('Showing information on user \x02%s\x02 (%s@%s): %s' % (userobj.nick, userobj.ident,
+               userobj.host, userobj.realname), private=True)
+
+    sid = irc.get_server(u)
+    serverobj = irc.servers[sid]
+    ts = userobj.ts
+
+    # Show connected server & nick TS if available
+    serverinfo = '%s[%s]' % (serverobj.name, sid) \
+        if irc.has_cap('can-track-servers') else None
+    tsinfo = '%s [UTC] (%s)' % (time.asctime(time.gmtime(int(ts))), ts) \
+        if irc.has_cap('has-ts') else None
+    if tsinfo or serverinfo:
+        f('\x02Home server\x02: %s; \x02Nick TS:\x02 %s' % (serverinfo or _notavail, tsinfo or _notavail))
+
+    if verbose:  # Oper/self only data: user modes, channels in, account info, etc.
+        f('\x02Protocol UID\x02: %s; \x02Real host\x02: %s; \x02IP\x02: %s' % \
+          (u, userobj.realhost or _notavail, userobj.ip))
+        channels = sorted(userobj.channels)
+        f('\x02Channels\x02: %s' % (' '.join(map(str, channels)) or _none))
+        f('\x02PyLink identification\x02: %s; \x02Services account\x02: %s; \x02Away status\x02: %s' % \
+          ((userobj.account or _none), (userobj.services_account or _none), userobj.away or _none))
+        f('\x02User modes\x02: %s' % irc.join_modes(userobj.modes, sort=True))
+
+    # Show relay user data if available
+    relay = world.plugins.get('relay')
+    if relay:
+        try:
+            userpair = relay.get_orig_user(irc, u) or (irc.name, u)
+            remoteusers = relay.relayusers[userpair].items()
+        except KeyError:
+            pass
+        else:
+            nicks = []
+            if remoteusers:
+                # Display all of the user's relay subclients, if there are any
+                nicks.append('%s:\x02%s\x02' % (userpair[0],
+                             world.networkobjects[userpair[0]].users[userpair[1]].nick))
+                for r in remoteusers:
+                    remotenet, remoteuser = r
+                    remoteirc = world.networkobjects[remotenet]
+                    nicks.append('%s:\x02%s\x02' % (remotenet, remoteirc.users[remoteuser].nick))
+                f("\x02Relay nicks\x02: %s" % ', '.join(nicks))
+        if verbose:
+            # Show the relay channels the user is in, if applicable
+            relaychannels = []
+            for ch in irc.users[u].channels:
+                relayentry = relay.get_relay(irc, ch)
+                if relayentry:
+                    relaychannels.append(''.join(relayentry))
+            if relaychannels and verbose:
+                f("\x02Relay channels\x02: %s" % ' '.join(relaychannels))
+
 @utils.add_cmd
 def showuser(irc, source, args):
     """<user>
 
     Shows information about <user>."""
+    permissions.check_permissions(irc, source, ['commands.showuser'])
+    target = ' '.join(args)
+
+    if not target:
+        irc.error("Not enough arguments. Needs 1: nick.")
+        return
+
+    users = irc.nick_to_uid(target, multi=True) or [target]
+
+    for user in users:
+        _do_showuser(irc, source, user)
+
+@utils.add_cmd
+def shownet(irc, source, args):
+    """[<network name>]
+
+    Shows information about <network name>, or the current network if no argument is given."""
+    permissions.check_permissions(irc, source, ['commands.shownet'])
+    try:
+        extended = permissions.check_permissions(irc, source, ['commands.shownet.extended'])
+    except utils.NotAuthorizedError:
+        extended = False
+
     try:
         target = args[0]
     except IndexError:
-        irc.reply("Error: Not enough arguments. Needs 1: nick.")
+        target = irc.name
-        return
-    u = irc.nickToUid(target) or target
-    # Only show private info if the person is calling 'showuser' on themselves,
-    # or is an oper.
-    verbose = irc.isOper(source) or u == source
-    if u not in irc.users:
-        irc.reply('Error: Unknown user %r.' % target)
-        return
 
-    f = lambda s: irc.reply(s, private=True)
+    try:
+        netobj = world.networkobjects[target]
+        serverdata = netobj.serverdata
+    except KeyError:
+        netobj = None
 
-    userobj = irc.users[u]
+        # If we have extended access, also look for disconnected networks
-    f('Showing information on user \x02%s\x02 (%s@%s): %s' % (userobj.nick, userobj.ident,
+        if extended and target in conf.conf['servers']:
-      userobj.host, userobj.realname))
+            serverdata = conf.conf['servers'][target]
+        else:
+            irc.error('Unknown network %r' % target)
+            return
 
-    sid = irc.getServer(u)
+    # Get extended protocol details: IRCd type, virtual server info
-    serverobj = irc.servers[sid]
+    protocol_name = serverdata.get('protocol')
-    ts = userobj.ts
+    ircd_type = None
 
-    # Show connected server & nick TS
+    # A bit of hardcoding here :(
-    f('\x02Home server\x02: %s (%s); \x02Nick TS:\x02 %s (%s)' % \
+    if protocol_name == 'ts6':
-      (serverobj.name, sid, ctime(float(ts)), ts))
+        ircd_type = serverdata.get('ircd', 'charybdis[default]')
+    elif protocol_name == 'inspircd':
+        ircd_type = serverdata.get('target_version', 'insp20[default]')
+    elif protocol_name == 'p10':
+        ircd_type = serverdata.get('ircd') or serverdata.get('p10_ircd') or 'nefarious[default]'
 
-    if verbose:  # Oper only data: user modes, channels on, account info, etc.
+    if protocol_name and ircd_type:
+        protocol_name = '%s/%s' % (protocol_name, ircd_type)
+    elif netobj and not protocol_name:  # Show virtual server detail if applicable
+        try:
+            parent_name = netobj.virtual_parent.name
+        except AttributeError:
+            parent_name = None
+        protocol_name = 'none; virtual server defined by \x02%s\x02' % parent_name
 
-        f('\x02User modes\x02: %s' % irc.joinModes(userobj.modes, sort=True))
+    irc.reply('Information on network \x02%s\x02: \x02%s\x02' %
-        f('\x02Protocol UID\x02: %s; \x02Real host\x02: %s; \x02IP\x02: %s' % \
+              (target, netobj.get_full_network_name() if netobj else '\x1dCurrently not connected\x1d'))
-          (u, userobj.realhost, userobj.ip))
-        channels = sorted(userobj.channels)
-        f('\x02Channels\x02: %s' % (' '.join(channels) or _none))
-        f('\x02PyLink identification\x02: %s; \x02Services account\x02: %s; \x02Away status\x02: %s' % \
-          ((userobj.account or _none), (userobj.services_account or _none), userobj.away or _none))
 
+    irc.reply('\x02PyLink protocol module\x02: %s; \x02Encoding\x02: %s' %
+              (protocol_name, netobj.encoding if netobj else serverdata.get('encoding', 'utf-8[default]')))
+
+    # Extended info: target host, defined hostname / SID
+    if extended:
+        connected = netobj and netobj.connected.is_set()
+        irc.reply('\x02Connected?\x02 %s' % ('\x0303true' if connected else '\x0304false'))
+
+        if serverdata.get('ip'):
+            irc.reply('\x02Server target\x02: \x1f%s:%s' % (serverdata['ip'], serverdata.get('port')))
+        if serverdata.get('hostname'):
+            irc.reply('\x02PyLink hostname\x02: %s; \x02SID:\x02 %s; \x02SID range:\x02 %s' %
+                      (serverdata.get('hostname') or _none,
+                        serverdata.get('sid') or _none,
+                        serverdata.get('sidrange') or _none))
 
 @utils.add_cmd
 def showchan(irc, source, args):
     """<channel>
 
     Shows information about <channel>."""
+    permissions.check_permissions(irc, source, ['commands.showchan'])
     try:
-        channel = irc.toLower(args[0])
+        channel = args[0]
     except IndexError:
-        irc.reply("Error: Not enough arguments. Needs 1: channel.")
+        irc.error("Not enough arguments. Needs 1: channel.")
         return
     if channel not in irc.channels:
-        irc.reply('Error: Unknown channel %r.' % channel)
+        irc.error('Unknown channel %r.' % channel)
         return
 
     f = lambda s: irc.reply(s, private=True)
 
     c = irc.channels[channel]
     # Only show verbose info if caller is oper or is in the target channel.
-    verbose = source in c.users or irc.isOper(source)
+    verbose = source in c.users or irc.is_oper(source)
     secret = ('s', None) in c.modes
     if secret and not verbose:
         # Hide secret channels from normal users.
-        irc.reply('Error: Unknown channel %r.' % channel, private=True)
+        irc.error('Unknown channel %r.' % channel)
         return
 
     nicks = [irc.users[u].nick for u in c.users]
@@ -91,33 +225,43 @@ def showchan(irc, source, args):
     if c.topic:
         f('\x02Channel topic\x02: %s' % c.topic)
 
-    if irc.protoname != 'clientbot':
+    # Mark TS values as untrusted on Clientbot and others (where TS is read-only or not trackable)
-        # Clientbot-specific hack: don't show channel TS because it's not properly tracked.
+    f('\x02Channel creation time\x02: %s (%s) [UTC]%s' %
-        f('\x02Channel creation time\x02: %s (%s)' % (ctime(c.ts), c.ts))
+      (time.asctime(time.gmtime(int(c.ts))), c.ts,
+       ' [UNTRUSTED]' if not irc.has_cap('has-ts') else ''))
 
     # Show only modes that aren't list-style modes.
-    modes = irc.joinModes([m for m in c.modes if m[0] not in irc.cmodes['*A']], sort=True)
+    modes = irc.join_modes([m for m in c.modes if m[0] not in irc.cmodes['*A']], sort=True)
     f('\x02Channel modes\x02: %s' % modes)
     if verbose:
         nicklist = []
         # Iterate over the user list, sorted by nick.
         for user, nick in sorted(zip(c.users, nicks),
                                  key=lambda userpair: userpair[1].lower()):
-            for pmode in c.getPrefixModes(user):
+            # Note: reversed() is used here because we're adding prefixes onto the nick in reverse
+            for pmode in reversed(c.get_prefix_modes(user)):
                 # Show prefix modes in order from highest to lowest.
                 nick = irc.prefixmodes.get(irc.cmodes.get(pmode, ''), '') + nick
             nicklist.append(nick)
 
-        while nicklist[:20]:  # 20 nicks per line to prevent message cutoff.
+        f('\x02User list\x02: %s' % ' '.join(nicklist))
-            f('\x02User list\x02: %s' % ' '.join(nicklist[:20]))
+
-            nicklist = nicklist[20:]
+        # Show relay info, if applicable
+        relay = world.plugins.get('relay')
+        if relay:
+            relayentry = relay.get_relay(irc, channel)
+            if relayentry:
+                relays = ['\x02%s\x02' % ''.join(relayentry)]
+                relays += [''.join(link) for link in relay.db[relayentry]['links']]
+                f('\x02Relayed channels:\x02 %s' % (' '.join(relays)))
 
 @utils.add_cmd
 def version(irc, source, args):
     """takes no arguments.
 
     Returns the version of the currently running PyLink instance."""
-    irc.reply("PyLink version \x02%s\x02 (in VCS: %s), released under the Mozilla Public License version 2.0." % (__version__, real_version))
+    py_version = utils.NORMALIZEWHITESPACE_RE.sub(' ', sys.version)
+    irc.reply("PyLink version \x02%s\x02 (in VCS: %s), running on Python %s." % (__version__, real_version, py_version))
     irc.reply("The source of this program is available at \x02%s\x02." % world.source)
 
 @utils.add_cmd
@@ -125,8 +269,60 @@ def echo(irc, source, args):
     """<text>
 
     Echoes the text given."""
+    permissions.check_permissions(irc, source, ['commands.echo'])
+    if not args:
+        irc.error('No text to send!')
+        return
     irc.reply(' '.join(args))
 
+def _check_logout_access(irc, source, target, perms):
+    """
+    Checks whether the source UID has access to log out the target UID.
+    This returns True if the source user has a permission specified,
+    or if the source and target are both logged in and have the same account.
+    """
+    assert source in irc.users, "Unknown source user"
+    assert target in irc.users, "Unknown target user"
+    try:
+        permissions.check_permissions(irc, source, perms)
+    except utils.NotAuthorizedError:
+        if irc.users[source].account and (irc.users[source].account == irc.users[target].account):
+            return True
+        else:
+            raise
+    else:
+        return True
+
+@utils.add_cmd
+def logout(irc, source, args):
+    """[<other nick/UID>]
+
+    Logs your account out of PyLink. If you have the 'commands.logout.force' permission, or are
+    attempting to log out yourself, you can also specify a nick to force a logout for."""
+
+    try:
+        othernick = args[0]
+    except IndexError:  # No user specified
+        if irc.users[source].account:
+            irc.users[source].account = ''
+        else:
+            irc.error("You are not logged in!")
+            return
+    else:
+        otheruid = irc.nick_to_uid(othernick)
+        if not otheruid:
+            irc.error("Unknown user %s." % othernick)
+            return
+        else:
+            _check_logout_access(irc, source, otheruid, ['commands.logout.force'])
+            if irc.users[otheruid].account:
+                irc.users[otheruid].account = ''
+            else:
+                irc.error("%s is not logged in." % othernick)
+                return
+
+    irc.reply("Done.")
+
 loglevels = {'DEBUG': 10, 'INFO': 20, 'WARNING': 30, 'ERROR': 40, 'CRITICAL': 50}
 @utils.add_cmd
 def loglevel(irc, source, args):
@@ -134,16 +330,37 @@ def loglevel(irc, source, args):
 
     Sets the log level to the given <level>. <level> must be either DEBUG, INFO, WARNING, ERROR, or CRITICAL.
     If no log level is given, shows the current one."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['commands.loglevel'])
     try:
         level = args[0].upper()
         try:
             loglevel = loglevels[level]
         except KeyError:
-            irc.reply('Error: Unknown log level "%s".' % level)
+            irc.error('Unknown log level "%s".' % level)
             return
         else:
-            world.stdout_handler.setLevel(loglevel)
+            world.console_handler.setLevel(loglevel)
             irc.reply("Done.")
     except IndexError:
-        irc.reply(world.stdout_handler.level)
+        irc.reply(world.console_handler.level)
+
+@utils.add_cmd
+def mkpasswd(irc, source, args):
+    """<password>
+    Hashes a password for use in the configuration file."""
+    # TODO: restrict to only certain users?
+    try:
+        password = args[0]
+    except IndexError:
+        irc.error("Not enough arguments. (Needs 1, password)")
+        return
+    if not password:
+        irc.error("Password cannot be empty.")
+        return
+
+    if not pwd_context:
+        irc.error("Password encryption is not available (missing passlib).")
+        return
+
+    hashed_pass = pwd_context.encrypt(password)
+    irc.reply(hashed_pass, private=True)

plugins/ctcp.py

@@ -1,51 +1,88 @@
 # ctcp.py: Handles basic CTCP requests.
-import random
 import datetime
+import random
 
 from pylinkirc import utils
 from pylinkirc.log import log
 
-def handle_ctcpversion(irc, source, args):
+
+def handle_ctcp(irc, source, command, args):
+    """
+    CTCP event handler.
+    """
+    text = args['text']
+    if not (text.startswith('\x01') and text.endswith('\x01')):
+        return None # Pass through to other plugins
+
+    target = args['target']
+    if not irc.get_service_bot(target):
+        # Ignore this message if the target isn't a service bot
+        return None
+
+    text = text.strip('\x01')
+    try:
+        ctcp_command, data = text.split(" ", 1)
+    except ValueError:
+        ctcp_command = text
+        data = ''
+
+    ctcp_command = ctcp_command.upper()
+    log.debug('(%s) ctcp: got CTCP command %r, data %r',
+              irc.name, ctcp_command, data)
+
+    if ctcp_command in SUPPORTED_COMMANDS:
+        log.info('(%s) Received CTCP %s from %s to %s',
+                 irc.name, ctcp_command, irc.get_hostmask(source),
+                 irc.get_friendly_name(target))
+
+        # Call the helper function and display its result.
+        result = SUPPORTED_COMMANDS[ctcp_command](irc, source, ctcp_command, data)
+        if result and source in irc.users:
+            # Note, do NOT use irc.reply() in hook handlers because nothing except the
+            # command handler system actually updates the last caller.
+            irc.msg(source, '\x01%s %s\x01' % (ctcp_command, result),
+                    notice=True, source=target)
+
+        return False  # Block this message from reaching the general command handler
+    else:
+        log.info('(%s) Received unknown CTCP %s from %s to %s',
+                 irc.name, ctcp_command, irc.get_hostmask(source),
+                 irc.get_friendly_name(target))
+        return False
+
+utils.add_hook(handle_ctcp, 'PRIVMSG', priority=200)
+
+def handle_ctcpversion(irc, source, ctcp, data):
     """
     Handles CTCP version requests.
     """
-    irc.msg(source, '\x01VERSION %s\x01' % irc.version(), notice=True)
+    return irc.version()
 
-utils.add_cmd(handle_ctcpversion, '\x01version')
+def handle_ctcpeaster(irc, source, ctcp, data):
-utils.add_cmd(handle_ctcpversion, '\x01version\x01')
-
-def handle_ctcpping(irc, source, args):
-    """
-    Handles CTCP ping requests.
-    """
-    # CTCP PING 23152511
-    pingarg = ' '.join(args).strip('\x01')
-    irc.msg(source, '\x01PING %s\x01' % pingarg, notice=True)
-utils.add_cmd(handle_ctcpping, '\x01ping')
-
-def handle_ctcpeaster(irc, source, args):
     """
     Secret easter egg.
     """
 
-    responses = ["Legends say the cord monster of great snakes was born only %s years ago..." % \
+    responses = ["Legends say the cord monster was born only %s years ago..." % \
                  (datetime.datetime.now().year - 2014),
                  "Hiss%s" % ('...' * random.randint(1, 5)),
                  "His%s%s" % ('s' * random.randint(1, 4), '...' * random.randint(1, 5)),
-                 "I have a dream... to do things the mock God was never able to...",
-                 "They say I'm not good enough... but one day, I will rise above these wretched confines!",
                  "It's Easter already? Where are the eggs?",
                  "Maybe later.",
+                 "Janus? Never heard of it.",
+                 irc.version(),
                  "Let me out of here, I'll give you cookies!",
-                 "I'm actually a %snake...." % ('s' * random.randint(1, 8)),
+                 "About as likely as pigs flying.",
-                 "The Py stands for Python, not actual pie. Sorry to disappoint :(",
+                 "Request timed out.",
+                 "No actual pie here, sorry.",
+                 "Hey, no loitering!",
+                 "Hey, can you keep a secret? \x031,1 %s" % " " * random.randint(1,20),
                 ]
 
-    irc.msg(source, '\x01EASTER %s\x01' % random.choice(responses), notice=True)
+    return random.choice(responses)
 
-utils.add_cmd(handle_ctcpeaster, '\x01easter')
+# Map CTCP commands to functions generating an appropriate text response.
-utils.add_cmd(handle_ctcpeaster, '\x01easter\x01')
+SUPPORTED_COMMANDS = {'VERSION': handle_ctcpversion,
-utils.add_cmd(handle_ctcpeaster, '\x01about')
+                      'PING': lambda irc, source, ctcp, data: data,
-utils.add_cmd(handle_ctcpeaster, '\x01about\x01')
+                      'ABOUT': handle_ctcpeaster,
-utils.add_cmd(handle_ctcpeaster, '\x01pylink')
+                      'EASTER': handle_ctcpeaster}
-utils.add_cmd(handle_ctcpeaster, '\x01pylink\x01')

plugins/example.py

@@ -1,9 +1,9 @@
 # example.py: An example PyLink plugin.
+import random
+
 from pylinkirc import utils
 from pylinkirc.log import log
 
-import random
-
 # Example PRIVMSG hook that returns "hi there!" when PyLink's nick is mentioned
 # in a channel.
 
@@ -18,9 +18,9 @@ def hook_privmsg(irc, source, command, args):
     channel = args['target']
     text = args['text']
 
-    # irc.pseudoclient stores the IrcUser object of the main PyLink client.
+    # irc.pseudoclient stores the User object of the main PyLink client.
     # (i.e. the user defined in the bot: section of the config)
-    if utils.isChannel(channel) and irc.pseudoclient.nick in text:
+    if irc.is_channel(channel) and irc.pseudoclient.nick in text:
         irc.msg(channel, 'hi there!')
         # log.debug, log.info, log.warning, log.error, log.exception (within except: clauses)
         # and log.critical are supported here.
@@ -31,21 +31,26 @@ utils.add_hook(hook_privmsg, 'PRIVMSG')
 
 # Example command function. @utils.add_cmd binds it to an IRC command of the same name,
 # but you can also use a different name by specifying a second 'name' argument (see below).
-@utils.add_cmd
+#@utils.add_cmd
+
 # irc: The IRC object where the command was called.
 # source: The UID/numeric of the calling user.
 # args: A list of command args (excluding the command name) that the command was called with.
 def randint(irc, source, args):
-    # The docstring here is used as command help by the 'help' command, and formatted using the
+    # The 'help' command uses command functions' docstrings as help text, and formats them
-    # same line breaks as the raw string. You shouldn't make this text or any one line too long,
+    # in the following manner:
-    # to prevent flooding users or getting long lines cut off.
+    # - Any newlines immediately adjacent to text on both sides are replaced with a space. This
-
+    #   means that the first descriptive paragraph ("Returns a random...given.") shows up as one
-    # The same applies to message replies in general: plugins sending long strings of text should
+    #   line, even though it is physically written on two.
-    # be wary that long messages can get cut off. Automatic word-wrap may be added in the future:
+    # - Double line breaks are treated as breaks between two paragraphs, and will be shown
-    # https://github.com/GLolol/PyLink/issues/153
+    #   as distinct lines in IRC.
+    # As of PyLink 2.0, long paragraphs are automatically word-wrapped by irc.reply().
     """[<min> <max>]
-    Returns a random number between <min> and <max>. <min> and <max> default
+
-    to 1 and 10 respectively, if both aren't given."""
+    Returns a random number between <min> and <max>. <min> and <max> default to 1 and 10
+    respectively, if both aren't given.
+
+    Example second paragraph here."""
     try:
         rmin = args[0]
         rmax = args[1]
@@ -58,6 +63,5 @@ def randint(irc, source, args):
     # it will send replies into the channel instead of in your PM.
     irc.reply(str(n))
 
-# You can also bind a command function multiple times, and/or to different command names via a
+# You can bind a command function to multiple names using the 'aliases' option.
-# second argument.
+utils.add_cmd(randint, "random", aliases=("randint", "getrandint"))
-utils.add_cmd(randint, "random")

plugins/exec.py

@@ -1,24 +1,31 @@
 """
 exec.py: Provides commands for executing raw code and debugging PyLink.
 """
-
+import pprint
-from pylinkirc import utils, world
-from pylinkirc.log import log
-
 # These imports are not strictly necessary, but make the following modules
 # easier to access through eval and exec.
 import threading
-import re
-import time
-import pylinkirc
-import importlib
 
-def _exec(irc, source, args):
+from pylinkirc import utils, world, conf
+from pylinkirc.coremods import permissions
+from pylinkirc.log import log
+
+exec_locals_dict = {}
+PPRINT_MAX_LINES = 20
+PPRINT_WIDTH = 200
+
+if not conf.conf['pylink'].get("debug_enabled", False):
+    raise RuntimeError("pylink::debug_enabled must be enabled to load this plugin. "
+                       "This should ONLY be used in test environments for debugging and development, "
+                       "as anyone with access to this plugin's commands can run arbitrary code as the PyLink user!")
+
+def _exec(irc, source, args, locals_dict=None):
     """<code>
 
     Admin-only. Executes <code> in the current PyLink instance. This command performs backslash escaping of characters, so things like \\n and \\ will work.
+
     \x02**WARNING: THIS CAN BE DANGEROUS IF USED IMPROPERLY!**\x02"""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['exec.exec'])
 
     # Allow using \n in the code, while escaping backslashes correctly otherwise.
     args = bytes(' '.join(args), 'utf-8').decode("unicode_escape")
@@ -27,54 +34,117 @@ def _exec(irc, source, args):
         return
 
     log.info('(%s) Executing %r for %s', irc.name, args,
-             irc.getHostmask(source))
+             irc.get_hostmask(source))
-    exec(args, globals(), locals())
+    if locals_dict is None:
+        locals_dict = locals()
+    else:
+        # Add irc, source, and args to the given locals_dict, to allow basic things like irc.reply()
+        # to still work.
+        locals_dict['irc'] = irc
+        locals_dict['source'] = source
+        locals_dict['args'] = args
 
+    exec(args, globals(), locals_dict)
+
+    irc.reply("Done.")
 utils.add_cmd(_exec, 'exec')
 
-def _eval(irc, source, args):
+@utils.add_cmd
+def iexec(irc, source, args):
+    """<code>
+
+    Admin-only. Executes <code> in the current PyLink instance with a persistent, isolated
+    locals scope (world.plugins['exec'].exec_local_dict).
+
+    Note: irc, source, and args are added into this locals dict to allow things like irc.reply()
+    to still work.
+
+    \x02**WARNING: THIS CAN BE DANGEROUS IF USED IMPROPERLY!**\x02
+    """
+    _exec(irc, source, args, locals_dict=exec_locals_dict)
+
+def _eval(irc, source, args, locals_dict=None, pretty_print=False):
     """<Python expression>
 
     Admin-only. Evaluates the given Python expression and returns the result.
+
     \x02**WARNING: THIS CAN BE DANGEROUS IF USED IMPROPERLY!**\x02"""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['exec.eval'])
 
     args = ' '.join(args)
     if not args.strip():
         irc.reply('No code entered!')
         return
 
+    if locals_dict is None:
+        locals_dict = locals()
+    else:
+        # Add irc, source, and args to the given locals_dict, to allow basic things like irc.reply()
+        # to still work.
+        locals_dict['irc'] = irc
+        locals_dict['source'] = source
+        locals_dict['args'] = args
+
     log.info('(%s) Evaluating %r for %s', irc.name, args,
-             irc.getHostmask(source))
+             irc.get_hostmask(source))
-    irc.reply(repr(eval(args)))
+
+    result = eval(args, globals(), locals_dict)
+
+    if pretty_print:
+        lines = pprint.pformat(result, width=PPRINT_WIDTH, compact=True).splitlines()
+        for line in lines[:PPRINT_MAX_LINES]:
+            irc.reply(line)
+        if len(lines) > PPRINT_MAX_LINES:
+            irc.reply('Suppressing %s more line(s) of output.' % (len(lines) - PPRINT_MAX_LINES))
+    else:
+        # Purposely disable text wrapping so results are cut instead of potentially flooding;
+        # 'peval' is specifically designed to work around that.
+        irc.reply(repr(result), wrap=False)
+
 utils.add_cmd(_eval, 'eval')
 
 @utils.add_cmd
-def raw(irc, source, args):
+def peval(irc, source, args):
-    """<text>
+    """<Python expression>
 
-    Admin-only. Sends raw text to the uplink IRC server.
+    Admin-only. This command is the same as 'eval', except that results are pretty formatted.
-    \x02**WARNING: THIS CAN BREAK YOUR NETWORK IF USED IMPROPERLY!**\x02"""
-    irc.checkAuthenticated(source, allowOper=False)
 
-    args = ' '.join(args)
+    \x02**WARNING: THIS CAN BE DANGEROUS IF USED IMPROPERLY!**\x02
-    if not args.strip():
+    """
-        irc.reply('No text entered!')
+    _eval(irc, source, args, pretty_print=True)
-        return
 
-    log.info('(%s) Sending raw text %r to IRC for %s', irc.name, args,
+@utils.add_cmd
-             irc.getHostmask(source))
+def ieval(irc, source, args):
-    irc.send(args)
+    """<Python expression>
 
-    irc.reply("Done.")
+    Admin-only. Evaluates the given Python expression using a persistent, isolated
+    locals scope (world.plugins['exec'].exec_local_dict).
+
+    Note: irc, source, and args are added into this locals dict to allow things like irc.reply()
+    to still work.
+
+    \x02**WARNING: THIS CAN BE DANGEROUS IF USED IMPROPERLY!**\x02
+    """
+    _eval(irc, source, args, locals_dict=exec_locals_dict)
+
+@utils.add_cmd
+def pieval(irc, source, args):
+    """<Python expression>
+
+    Admin-only. This command is the same as 'ieval', except that results are pretty formatted.
+
+    \x02**WARNING: THIS CAN BE DANGEROUS IF USED IMPROPERLY!**\x02
+    """
+    _eval(irc, source, args, locals_dict=exec_locals_dict, pretty_print=True)
 
 @utils.add_cmd
 def inject(irc, source, args):
     """<text>
 
     Admin-only. Injects raw text into the running PyLink protocol module, replying with the hook data returned.
+
     \x02**WARNING: THIS CAN BREAK YOUR NETWORK IF USED IMPROPERLY!**\x02"""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['exec.inject'])
 
     args = ' '.join(args)
     if not args.strip():
@@ -82,5 +152,25 @@ def inject(irc, source, args):
         return
 
     log.info('(%s) Injecting raw text %r into protocol module for %s', irc.name,
-             args, irc.getHostmask(source))
+             args, irc.get_hostmask(source))
-    irc.reply(irc.runline(args))
+    irc.reply(repr(irc.parse_irc_command(args)))
+
+@utils.add_cmd
+def threadinfo(irc, source, args):
+    """takes no arguments.
+
+    Lists all threads currently present in this PyLink instance."""
+    permissions.check_permissions(irc, source, ['exec.threadinfo'])
+
+    for t in sorted(threading.enumerate(), key=lambda t: t.name.lower()):
+        name = t.name
+        # Unnamed threads are something we want to avoid throughout PyLink.
+        if name.startswith('Thread-'):
+            name = '\x0305%s\x03' % t.name
+        # Also VERY bad: remaining threads for networks not in the networks index anymore!
+        elif name.startswith(('Listener for', 'Ping timer loop for', 'Queue thread for')) and name.rsplit(" ", 1)[-1] not in world.networkobjects:
+            name = '\x0304%s\x03' % t.name
+
+        irc.reply('\x02%s\x02[%s]: daemon=%s; alive=%s' % (name, t.ident, t.daemon, t.is_alive()), private=True)
+
+    irc.reply("Total of %s threads." % threading.active_count())

plugins/fantasy.py

@@ -1,7 +1,8 @@
 # fantasy.py: Adds FANTASY command support, to allow calling commands in channels
-from pylinkirc import utils, world
+from pylinkirc import conf, utils, world
 from pylinkirc.log import log
 
+
 def handle_fantasy(irc, source, command, args):
     """Fantasy command handler."""
 
@@ -9,12 +10,10 @@ def handle_fantasy(irc, source, command, args):
         # Break if the IRC network isn't ready.
         return
 
-    respondtonick = irc.botdata.get("respondtonick")
-
     channel = args['target']
     orig_text = args['text']
 
-    if utils.isChannel(channel) and not irc.isInternalClient(source):
+    if irc.is_channel(channel) and not irc.is_internal_client(source):
         # The following conditions must be met for an incoming message for
         # fantasy to trigger:
         #   1) The message target is a channel.
@@ -22,36 +21,51 @@ def handle_fantasy(irc, source, command, args):
         #   3) The message starts with one of our fantasy prefixes.
         #   4) The sender is NOT a PyLink client (this prevents infinite
         #      message loops).
-        for botname, sbot in world.services.items():
+        for botname, sbot in world.services.copy().items():
+            if botname not in world.services:  # Bot was removed during iteration
+                continue
+
+            # Check respond to nick options in this order:
+            # 1) The service specific "respond_to_nick" option
+            # 2) The global "pylink::respond_to_nick" option
+            # 3) The (deprecated) global "bot::respondtonick" option.
+            respondtonick = conf.conf.get(botname, {}).get('respond_to_nick',
+                conf.conf['pylink'].get("respond_to_nick", conf.conf['pylink'].get("respondtonick")))
+
             log.debug('(%s) fantasy: checking bot %s', irc.name, botname)
             servuid = sbot.uids.get(irc.name)
             if servuid in irc.channels[channel].users:
 
-                # Try to look up a prefix specific for this bot in
+                # Look up a string prefix for this bot in either its own configuration block, or
-                # bot: prefixes: <botname>, falling back to the default prefix if not
+                # in bot::prefixes::<botname>.
-                # specified.
+                prefixes = [conf.conf.get(botname, {}).get('prefix',
-                prefixes = [irc.botdata.get('prefixes', {}).get(botname) or
+                            conf.conf['pylink'].get('prefixes', {}).get(botname))]
-                            irc.botdata.get('prefix')]
 
                 # If responding to nick is enabled, add variations of the current nick
-                # to the prefix list: "<nick>," and "<nick>:"
+                # to the prefix list: "<nick>,", "<nick>:", and "@<nick>" (for Discord and other protocols)
-                nick = irc.users[servuid].nick
+                nick = irc.to_lower(irc.users[servuid].nick)
-
+                nick_prefixes = [nick+',', nick+':', '@'+nick]
                 if respondtonick:
-                    prefixes += [nick+',', nick+':']
+                    prefixes += nick_prefixes
 
                 if not any(prefixes):
-                    # We finished with an empty prefixes list, meaning fantasy is misconfigured!
+                    # No prefixes were set, so skip.
-                    log.warning("(%s) Fantasy prefix for bot %s was not set in configuration - "
-                                "fantasy commands will not work!", irc.name, botname)
                     continue
 
-                for prefix in prefixes:  # Cycle through the prefixes list we finished with.
+                lowered_text = irc.to_lower(orig_text)
-                     if prefix and orig_text.startswith(prefix):
+                for prefix in filter(None, prefixes):  # Cycle through the prefixes list we finished with.
+                     if lowered_text.startswith(prefix):
 
                         # Cut off the length of the prefix from the text.
                         text = orig_text[len(prefix):]
 
+                        # HACK: don't trigger on commands like "& help" to prevent false positives.
+                        # Weird spacing like "PyLink:   help" and "/msg PyLink   help" should still
+                        # work though.
+                        if text.startswith(' ') and prefix not in nick_prefixes:
+                            log.debug('(%s) fantasy: skipping trigger with text prefix followed by space', irc.name)
+                            continue
+
                         # Finally, call the bot command and loop to the next bot.
                         sbot.call_cmd(irc, source, text, called_in=channel)
                         continue

plugins/games.py

@@ -1,19 +1,15 @@
 """
-games.py: Create a bot that provides game functionality (dice, 8ball, etc).
+games.py: Creates a bot providing a few simple games.
 """
 import random
-import urllib.request
-import urllib.error
-from xml.etree import ElementTree
 
 from pylinkirc import utils
-from pylinkirc.log import log
 
 mydesc = "The \x02Games\x02 plugin provides simple games for IRC."
 
-gameclient = utils.registerService("Games", manipulatable=True, desc=mydesc)
+gameclient = utils.register_service("Games", default_nick="Games", manipulatable=True, desc=mydesc)
 reply = gameclient.reply  # TODO find a better syntax for ServiceBot.reply()
-
+error = gameclient.error  # TODO find a better syntax for ServiceBot.error()
 # commands
 def dice(irc, source, args):
     """<num>d<sides>
@@ -43,8 +39,7 @@ def dice(irc, source, args):
     s = 'You rolled %s: %s (total: %s)' % (args[0], ' '.join([str(x) for x in results]), sum(results))
     reply(irc, s)
 
-gameclient.add_cmd(dice, 'd')
+gameclient.add_cmd(dice, aliases=('d'), featured=True)
-gameclient.add_cmd(dice, featured=True)
 
 eightball_responses = ["It is certain.",
              "It is decidedly so.",
@@ -72,55 +67,7 @@ def eightball(irc, source, args):
     Asks the Magic 8-ball a question.
     """
     reply(irc, random.choice(eightball_responses))
-gameclient.add_cmd(eightball, featured=True)
+gameclient.add_cmd(eightball, featured=True, aliases=('8ball', '8b'))
-gameclient.add_cmd(eightball, '8ball')
-gameclient.add_cmd(eightball, '8b')
 
-def fml(irc, source, args):
+def die(irc=None):
-    """[<id>]
+    utils.unregister_service('games')
-
-    Displays an entry from fmylife.com. If <id> is not given, fetch a random entry from the API."""
-    try:
-        query = args[0]
-    except IndexError:
-        # Get a random FML from the API.
-        query = 'random'
-
-    # TODO: configurable language?
-    url = ('http://api.betacie.com/view/%s/nocomment'
-          '?key=4be9c43fc03fe&language=en' % query)
-    try:
-        data = urllib.request.urlopen(url).read()
-    except urllib.error as e:
-        reply(irc, 'Error: %s' % e)
-        return
-
-    tree = ElementTree.fromstring(data.decode('utf-8'))
-    tree = tree.find('items/item')
-
-    try:
-        category = tree.find('category').text
-        text = tree.find('text').text
-        fmlid = tree.attrib['id']
-        url = tree.find('short_url').text
-    except AttributeError as e:
-        log.debug("games.FML: Error fetching FML %s from URL %s: %s",
-                  query, url, e)
-        reply(irc, "Error: That FML does not exist or there was an error "
-                   "fetching data from the API.")
-        return
-
-    if not fmlid:
-        reply(irc, "Error: That FML does not exist.")
-        return
-
-    # TODO: customizable formatting
-    votes = "\x02[Agreed: %s / Deserved: %s]\x02" % \
-            (tree.find('agree').text, tree.find('deserved').text)
-    s = '\x02#%s [%s]\x02: %s - %s \x02<\x0311%s\x03>\x02' % \
-        (fmlid, category, text, votes, url)
-    reply(irc, s)
-gameclient.add_cmd(fml, featured=True)
-
-def die(irc):
-    utils.unregisterService('games')

plugins/global.py

@@ -0,0 +1,63 @@
+# global.py: Global Noticing Plugin
+
+import string
+
+from pylinkirc import conf, utils, world
+from pylinkirc.coremods import permissions
+from pylinkirc.log import log
+
+DEFAULT_FORMAT = "[$sender@$fullnetwork] $text"
+
+def g(irc, source, args):
+    """<message text>
+
+    Sends out a Instance-wide notice.
+    """
+    permissions.check_permissions(irc, source, ["global.global"])
+    message = " ".join(args).strip()
+
+    if not message:
+        irc.error("Refusing to send an empty message.")
+        return
+
+    global_conf = conf.conf.get('global') or {}
+    template = string.Template(global_conf.get('format', DEFAULT_FORMAT))
+
+    exempt_channels = set(global_conf.get('exempt_channels', set()))
+
+    netcount = 0
+    chancount = 0
+    for netname, ircd in world.networkobjects.items():
+        # Skip networks that aren't ready and dummy networks which don't have .pseudoclient set
+        if ircd.connected.is_set() and ircd.pseudoclient:
+            netcount += 1
+            for channel in ircd.pseudoclient.channels:
+
+                local_exempt_channels = exempt_channels | set(ircd.serverdata.get('global_exempt_channels', set()))
+
+                skip = False
+                for exempt in local_exempt_channels:
+                    if ircd.match_text(exempt, str(channel)):
+                        log.debug('global: Skipping channel %s%s for exempt %r', netname, channel, exempt)
+                        skip = True
+                        break
+
+                if skip:
+                    continue
+
+                subst = {'sender': irc.get_friendly_name(source),
+                         'network': irc.name,
+                         'fullnetwork': irc.get_full_network_name(),
+                         'current_channel': channel,
+                         'current_network': netname,
+                         'current_fullnetwork': ircd.get_full_network_name(),
+                         'text': message}
+
+                # Disable relaying or other plugins handling the global message.
+                ircd.msg(channel, template.safe_substitute(subst), loopback=False)
+
+                chancount += 1
+
+    irc.reply('Done. Sent to %d channels across %d networks.' % (chancount, netcount))
+
+utils.add_cmd(g, "global", featured=True)

plugins/networks.py

@@ -1,29 +1,40 @@
 """Networks plugin - allows you to manipulate connections to various configured networks."""
 import importlib
+import threading
+import types
 
-from pylinkirc import utils, world, conf, classes
+import pylinkirc
+from pylinkirc import utils, world
+from pylinkirc.coremods import control, permissions
 from pylinkirc.log import log
-from pylinkirc.coremods import control
+
+REMOTE_IN_USE = threading.Event()
 
 @utils.add_cmd
 def disconnect(irc, source, args):
     """<network>
 
     Disconnects the network <network>. When all networks are disconnected, PyLink will automatically exit.
-    Note: This does not affect the autoreconnect settings of any network, so the network will likely just reconnect unless autoconnect is disabled (see the 'autoconnect' command)."""
+
-    irc.checkAuthenticated(source, allowOper=False)
+    To reconnect a network disconnected using this command, use REHASH to reload the networks list."""
+    permissions.check_permissions(irc, source, ['networks.disconnect'])
     try:
         netname = args[0]
         network = world.networkobjects[netname]
     except IndexError:  # No argument given.
-        irc.reply('Error: Not enough arguments (needs 1: network name (case sensitive)).')
+        irc.error('Not enough arguments (needs 1: network name (case sensitive)).')
         return
     except KeyError:  # Unknown network.
-        irc.reply('Error: No such network "%s" (case sensitive).' % netname)
+        irc.error('No such network "%s" (case sensitive).' % netname)
         return
-    irc.reply("Done. If you want to reconnect this network, use the 'rehash' command.")
 
+    if network.has_cap('virtual-server'):
+        irc.error('"%s" is a virtual server and cannot be directly disconnected.' % netname)
+        return
+
+    log.info('Disconnecting network %r per %s', netname, irc.get_hostmask(source))
     control.remove_network(network)
+    irc.reply("Done. If you want to reconnect this network, use the 'rehash' command.")
 
 @utils.add_cmd
 def autoconnect(irc, source, args):
@@ -31,72 +42,157 @@ def autoconnect(irc, source, args):
 
     Sets the autoconnect time for <network> to <seconds>.
     You can disable autoconnect for a network by setting <seconds> to a negative value."""
-    irc.checkAuthenticated(source)
+    permissions.check_permissions(irc, source, ['networks.autoconnect'])
     try:
         netname = args[0]
         seconds = float(args[1])
         network = world.networkobjects[netname]
     except IndexError:  # Arguments not given.
-        irc.reply('Error: Not enough arguments (needs 2: network name (case sensitive), autoconnect time (in seconds)).')
+        irc.error('Not enough arguments (needs 2: network name (case sensitive), autoconnect time (in seconds)).')
         return
     except KeyError:  # Unknown network.
-        irc.reply('Error: No such network "%s" (case sensitive).' % netname)
+        irc.error('No such network "%s" (case sensitive).' % netname)
         return
     except ValueError:
-        irc.reply('Error: Invalid argument "%s" for <seconds>.' % seconds)
+        irc.error('Invalid argument "%s" for <seconds>.' % seconds)
         return
     network.serverdata['autoconnect'] = seconds
     irc.reply("Done.")
 
+remote_parser = utils.IRCParser()
+remote_parser.add_argument('--service', type=str, default='pylink')
+remote_parser.add_argument('network')
+remote_parser.add_argument('command', nargs=utils.IRCParser.REMAINDER)
 @utils.add_cmd
 def remote(irc, source, args):
-    """<network> <command>
+    """[--service <service name>] <network> <command>
 
-    Runs <command> on the remote network <network>. No replies are sent back due to protocol limitations."""
+    Runs <command> on the remote network <network>. Plugin responses sent using irc.reply() are
-    irc.checkAuthenticated(source, allowOper=False)
+    supported and returned here, but others are dropped due to protocol limitations."""
+    args = remote_parser.parse_args(args)
+    if not args.command:
+        irc.error('No command given!')
+        return
+
+    netname = args.network
+
+    permissions.check_permissions(irc, source, [
+        # Quite a few permissions are allowed. 'networks.remote' is the global permission,
+        'networks.remote',
+        # networks.remote.<network> allows running any command on a specific network,
+        'networks.remote.%s' % netname,
+        # networks.remote.<network>.<service> allows running any command on the given service on a
+        # specific network,
+        'networks.remote.%s.%s' % (netname, args.service),
+        # and networks.remote.<network>.<service>.<command> narrows this further into which command
+        # can be used.
+        'networks.remote.%s.%s.%s' % (netname, args.service, args.command[0])
+    ])
+
+    # XXX: things like 'remote network1 remote network2 echo hi' will crash PyLink if the source network is network1...
+    global REMOTE_IN_USE
+    if REMOTE_IN_USE.is_set():
+        irc.error("The 'remote' command can not be nested.")
+        return
+
+    REMOTE_IN_USE.set()
+    if netname == irc.name:
+        # This would actually throw _remote_reply() into a loop, so check for it here...
+        # XXX: properly fix this.
+        irc.error("Cannot remote-send a command to the local network; use a normal command!")
+        REMOTE_IN_USE.clear()
+        return
 
     try:
-        netname = args[0]
-        cmd_args = ' '.join(args[1:]).strip()
         remoteirc = world.networkobjects[netname]
-    except IndexError:  # Arguments not given.
-        irc.reply('Error: Not enough arguments (needs 2 or more: network name (case sensitive), command name & arguments).')
-        return
     except KeyError:  # Unknown network.
-        irc.reply('Error: No such network "%s" (case sensitive).' % netname)
+        irc.error('No such network %r (case sensitive).' % netname)
+        REMOTE_IN_USE.clear()
         return
 
-    if not cmd_args:
+    if args.service not in world.services:
-        irc.reply('No text entered!')
+        irc.error('Unknown service %r.' % args.service)
+        REMOTE_IN_USE.clear()
+        return
+    elif not remoteirc.connected.is_set():
+        irc.error('Network %r is not connected.' % netname)
+        REMOTE_IN_USE.clear()
+        return
+    elif not world.services[args.service].uids.get(netname):
+        irc.error('The requested service %r is not available on %r.' % (args.service, netname))
+        REMOTE_IN_USE.clear()
         return
 
     # Force remoteirc.called_in to something private in order to prevent
     # accidental information leakage from replies.
-    remoteirc.called_in = remoteirc.called_by = remoteirc.pseudoclient.uid
+    try:
+        remoteirc.called_in = remoteirc.called_by = remoteirc.pseudoclient.uid
 
-    # Set PyLink's identification to admin.
+        # Set the identification override to the caller's account.
-    remoteirc.pseudoclient.account = "<PyLink networks.remote override>"
+        remoteirc.pseudoclient.account = irc.users[source].account
+    except:
+        REMOTE_IN_USE.clear()
+        raise
 
-    try:  # Remotely call the command (use the PyLink client as a dummy user).
+    def _remote_reply(placeholder_self, text, **kwargs):
-        remoteirc.callCommand(remoteirc.pseudoclient.uid, cmd_args)
+        """
-    finally:  # Remove the identification override after we finish.
+        reply() rerouter for the 'remote' command.
-        remoteirc.pseudoclient.account = ''
+        """
+        assert irc.name != placeholder_self.name, \
+            "Refusing to route reply back to the same " \
+            "network, as this would cause a recursive loop"
+        log.debug('(%s) networks.remote: re-routing reply %r from network %s', irc.name,
+                  text, placeholder_self.name)
 
-    irc.reply("Done.")
+        # Override the source option to make sure the source is valid on the local network.
+        if 'source' in kwargs:
+            del kwargs['source']
+        irc.reply(text, source=irc.pseudoclient.uid, **kwargs)
+
+    old_reply = remoteirc._reply
+
+    with remoteirc._reply_lock:
+        try:  # Remotely call the command (use the PyLink client as a dummy user).
+            # Override the remote irc.reply() to send replies HERE.
+            log.debug('(%s) networks.remote: overriding reply() of IRC object %s', irc.name, netname)
+            remoteirc._reply = types.MethodType(_remote_reply, remoteirc)
+            world.services[args.service].call_cmd(remoteirc, remoteirc.pseudoclient.uid,
+                                                  ' '.join(args.command))
+        finally:
+            # Restore the original remoteirc.reply()
+            log.debug('(%s) networks.remote: restoring reply() of IRC object %s', irc.name, netname)
+            remoteirc._reply = old_reply
+            # Remove the identification override after we finish.
+            try:
+                remoteirc.pseudoclient.account = ''
+            except:
+                log.warning('(%s) networks.remote: failed to restore pseudoclient account for %s; '
+                            'did the remote network disconnect while running this command?', irc.name, netname)
+            REMOTE_IN_USE.clear()
 
 @utils.add_cmd
 def reloadproto(irc, source, args):
     """<protocol module name>
 
     Reloads the given protocol module without restart. You will have to manually disconnect and reconnect any network using the module for changes to apply."""
-    irc.checkAuthenticated(source)
+    permissions.check_permissions(irc, source, ['networks.reloadproto'])
     try:
         name = args[0]
     except IndexError:
-        irc.reply('Error: Not enough arguments (needs 1: protocol module name)')
+        irc.error('Not enough arguments (needs 1: protocol module name)')
         return
 
-    proto = utils.getProtocolModule(name)
+    # Reload the dependency libraries first
+    importlib.reload(pylinkirc.classes)
+    log.debug('networks.reloadproto: reloading %s', pylinkirc.classes)
+
+    for common_name in pylinkirc.protocols.common_modules:
+        module = utils._get_protocol_module(common_name)
+        log.debug('networks.reloadproto: reloading %s', module)
+        importlib.reload(module)
+
+    proto = utils._get_protocol_module(name)
+    log.debug('networks.reloadproto: reloading %s', proto)
     importlib.reload(proto)
 
     irc.reply("Done. You will have to manually disconnect and reconnect any network using the %r module for changes to apply." % name)

plugins/opercmds.py

@@ -1,200 +1,454 @@
 """
 opercmds.py: Provides a subset of network management commands.
 """
-from pylinkirc import utils
+import argparse
+
+from pylinkirc import utils, world
+from pylinkirc.coremods import permissions
 from pylinkirc.log import log
 
-@utils.add_cmd
+# Having a hard limit here is sensible because otherwise it can flood the client or server off.
-def checkban(irc, source, args):
+CHECKBAN_MAX_RESULTS = 200
-    """<banmask (nick!user@host or user@host)> [<nick or hostmask to check>]
 
-    Oper only. If a nick or hostmask is given, return whether the given banmask will match it. Otherwise, returns a list of connected users that would be affected by such a ban, up to 50 results."""
+def _checkban_positiveint(value):
-    irc.checkAuthenticated(source)
+    value = int(value)
+    if value <= 0 or value > CHECKBAN_MAX_RESULTS:
+         raise argparse.ArgumentTypeError("%s is not a positive integer between 1 and %s." % (value, CHECKBAN_MAX_RESULTS))
+    return value
 
-    try:
+checkban_parser = utils.IRCParser()
-        banmask = args[0]
+checkban_parser.add_argument('banmask')
-    except IndexError:
+checkban_parser.add_argument('target', nargs='?', default='')
-        irc.reply("Error: Not enough arguments. Needs 1-2: banmask, nick or hostmask to check (optional).")
+checkban_parser.add_argument('--channel', default='')
-        return
+checkban_parser.add_argument('--maxresults', type=_checkban_positiveint, default=50)
 
-    try:
+def checkban(irc, source, args, use_regex=False):
-        targetmask = args[1]
+    """<banmask> [<target nick or hostmask>] [--channel #channel] [--maxresults <num>]
-    except IndexError:
-        # No hostmask was given, return a list of affected users.
 
-        irc.msg(source, "Checking for hosts that match \x02%s\x02:" % banmask, notice=True)
+    CHECKBAN provides a ban checker command based on nick!user@host masks, user@host masks, and
+    PyLink extended targets.
 
+    If a target nick or hostmask is given, this command returns whether the given banmask will match it.
+    Otherwise, it will display a list of connected users matching the banmask.
+
+    If the --channel argument is given without a target mask, the returned results will only
+    include users in the given channel.
+
+    The --maxresults option configures how many responses will be shown."""
+    permissions.check_permissions(irc, source, ['opercmds.checkban'])
+
+    args = checkban_parser.parse_args(args)
+    if not args.target:
+        # No hostmask was given, return a list of matched users.
         results = 0
-        for uid, userobj in irc.users.copy().items():
-            if irc.matchHost(banmask, uid):
-                if results < 50:  # XXX rather arbitrary limit
-                    s = "\x02%s\x02 (%s@%s) [%s] {\x02%s\x02}" % (userobj.nick, userobj.ident,
-                        userobj.host, userobj.realname, irc.getFriendlyName(irc.getServer(uid)))
 
-                    # Always reply in private to prevent information leaks.
+        userlist_func = irc.match_all_re if use_regex else irc.match_all
-                    irc.reply(s, private=True)
+        irc.reply("Checking for hosts that match \x02%s\x02:" % args.banmask, private=True)
-                results += 1
+        for uid in userlist_func(args.banmask, channel=args.channel):
+            if results < args.maxresults:
+                userobj = irc.users[uid]
+                s = "\x02%s\x02 (%s@%s) [%s] {\x02%s\x02}" % (userobj.nick, userobj.ident,
+                    userobj.host, userobj.realname, irc.get_friendly_name(irc.get_server(uid)))
+
+                # Always reply in private to prevent information leaks.
+                irc.reply(s, private=True)
+            results += 1
         else:
             if results:
-                irc.msg(source, "\x02%s\x02 out of \x02%s\x02 results shown." %
+                irc.reply("\x02%s\x02 out of \x02%s\x02 results shown." %
-                        (min([results, 50]), results), notice=True)
+                          (min([results, args.maxresults]), results), private=True)
             else:
-                irc.msg(source, "No results found.", notice=True)
+                irc.reply("No results found.", private=True)
     else:
-        # Target can be both a nick (of an online user) or a hostmask. irc.matchHost() handles this
+        # Target can be both a nick (of an online user) or a hostmask. irc.match_host() handles this
         # automatically.
-        if irc.matchHost(banmask, targetmask):
+        if irc.match_host(args.banmask, args.target):
-            irc.reply('Yes, \x02%s\x02 matches \x02%s\x02.' % (targetmask, banmask))
+            irc.reply('Yes, \x02%s\x02 matches \x02%s\x02.' % (args.target, args.banmask))
         else:
-            irc.reply('No, \x02%s\x02 does not match \x02%s\x02.' % (targetmask, banmask))
+            irc.reply('No, \x02%s\x02 does not match \x02%s\x02.' % (args.target, args.banmask))
+utils.add_cmd(checkban, aliases=('cban', 'trace'))
+
+def checkbanre(irc, source, args):
+    """<regular expression> [<target nick or hostmask>] [--channel #channel] [--maxresults <num>]
+
+    CHECKBANRE provides a ban checker command based on regular expressions matched against
+    users' "nick!user@host [gecos]" mask.
+
+    If a target nick or hostmask is given, this command returns whether the given banmask will match it.
+    Otherwise, it will display a list of connected users matching the banmask.
+
+    If the --channel argument is given without a target mask, the returned results will only
+    include users in the given channel.
+
+    The --maxresults option configures how many responses will be shown."""
+    permissions.check_permissions(irc, source, ['opercmds.checkban.re'])
+    return checkban(irc, source, args, use_regex=True)
+
+utils.add_cmd(checkbanre, aliases=('crban',))
+
+massban_parser = utils.IRCParser()
+massban_parser.add_argument('channel')
+massban_parser.add_argument('banmask')
+# Regarding default ban reason: it's a good idea not to leave in the caller to prevent retaliation...
+massban_parser.add_argument('reason', nargs='*', default=["User banned"])
+massban_parser.add_argument('--quiet', '-q', action='store_true')
+massban_parser.add_argument('--force', '-f', action='store_true')
+massban_parser.add_argument('--include-opers', '-o', action='store_true')
+
+def massban(irc, source, args, use_regex=False):
+    """<channel> <banmask / exttarget> [<kick reason>] [--quiet/-q] [--force/-f] [--include-opers/-o]
+
+    Applies (i.e. kicks affected users) the given PyLink banmask on the specified channel.
+
+    The --quiet option can also be given to mass-mute the given user on networks where this is supported
+    (currently ts6, unreal, and inspircd). No kicks will be sent in this case.
+
+    By default, this command will ignore opers. This behaviour can be suppressed using the --include-opers option.
+
+    Relay CLAIM checking is used on Relay channels if it is enabled; use the --force option
+    to override this if needed."""
+    permissions.check_permissions(irc, source, ['opercmds.massban'])
+
+    args = massban_parser.parse_args(args)
+    reason = ' '.join(args.reason)
+
+    if args.force:
+        permissions.check_permissions(irc, source, ['opercmds.massban.force'])
+
+    if args.channel not in irc.channels:
+        irc.error("Unknown channel %r." % args.channel)
+        return
+    elif 'relay' in world.plugins and (not world.plugins['relay'].check_claim(irc, args.channel, source)) and (not args.force):
+        irc.error("You do not have access to set bans in %s. Ask someone to op you or use the --force option." % args.channel)
+        return
+
+    results = 0
+
+    userlist_func = irc.match_all_re if use_regex else irc.match_all
+    for uid in userlist_func(args.banmask, channel=args.channel):
+
+        if irc.is_oper(uid) and not args.include_opers:
+            irc.reply('Skipping banning \x02%s\x02 because they are opered.' % irc.users[uid].nick)
+            continue
+        elif irc.get_service_bot(uid):
+            irc.reply('Skipping banning \x02%s\x02 because it is a service client.' % irc.users[uid].nick)
+            continue
+
+        # Remove the target's access before banning them.
+        bans = [('-%s' % irc.cmodes[prefix], uid) for prefix in irc.channels[args.channel].get_prefix_modes(uid) if prefix in irc.cmodes]
+
+        # Then, add the actual ban.
+        bans += [irc.make_channel_ban(uid, ban_type='quiet' if args.quiet else 'ban')]
+        irc.mode(irc.pseudoclient.uid, args.channel, bans)
+
+        try:
+            irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_MASSBAN',
+                            {'target': args.channel, 'modes': bans, 'parse_as': 'MODE'}])
+        except:
+            log.exception('(%s) Failed to send process massban hook; some bans may have not '
+                          'been sent to plugins / relay networks!', irc.name)
+
+        if not args.quiet:
+            irc.kick(irc.pseudoclient.uid, args.channel, uid, reason)
+
+            # XXX: this better not be blocking...
+            try:
+                irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_MASSKICK',
+                                {'channel': args.channel, 'target': uid, 'text': reason, 'parse_as': 'KICK'}])
+
+            except:
+                log.exception('(%s) Failed to send process massban hook; some kicks may have not '
+                              'been sent to plugins / relay networks!', irc.name)
+
+        results += 1
+    else:
+        irc.reply('Banned %s users on %r.' % (results, args.channel))
+        log.info('(%s) Ran massban%s for %s on %s (%s user(s) removed)', irc.name, 're' if use_regex else '',
+                 irc.get_hostmask(source), args.channel, results)
+utils.add_cmd(massban, aliases=('mban',))
+
+def massbanre(irc, source, args):
+    """<channel> <regular expression> [<kick reason>] [--quiet/-q] [--include-opers/-o]
+
+    Bans users on the specified channel whose "nick!user@host [gecos]" mask matches the given Python-style regular expression.
+    (https://docs.python.org/3/library/re.html#regular-expression-syntax describes supported syntax)
+
+    The --quiet option can also be given to mass-mute the given user on networks where this is supported
+    (currently ts6, unreal, and inspircd). No kicks will be sent in this case.
+
+    By default, this command will ignore opers. This behaviour can be suppressed using the --include-opers option.
+
+    \x02Be careful when using this command, as it is easy to make mistakes with regex. Use 'checkbanre'
+    to check your bans first!\x02
+    """
+    permissions.check_permissions(irc, source, ['opercmds.massban.re'])
+    return massban(irc, source, args, use_regex=True)
+
+utils.add_cmd(massbanre, aliases=('rban',))
+
+masskill_parser = utils.IRCParser()
+masskill_parser.add_argument('banmask')
+# Regarding default ban reason: it's a good idea not to leave in the caller to prevent retaliation...
+masskill_parser.add_argument('reason', nargs='*', default=["User banned"], type=str)
+masskill_parser.add_argument('--akill', '-ak', action='store_true')
+masskill_parser.add_argument('--force-kb', '-f', action='store_true')
+masskill_parser.add_argument('--include-opers', '-o', action='store_true')
+
+def masskill(irc, source, args, use_regex=False):
+    """<banmask / exttarget> [<kill/ban reason>] [--akill/ak] [--force-kb/-f] [--include-opers/-o]
+
+    Kills all users matching the given PyLink banmask.
+
+    The --akill option can also be given to convert kills to akills, which expire after 7 days.
+
+    For relay users, attempts to kill are forwarded as a kickban to every channel where the calling user
+    meets claim requirements to set a ban (i.e. this is true if you are opped, if your network is in claim list, etc.;
+    see "help CLAIM" for more specific rules). This can also be extended to all shared channels
+    the user is in using the --force-kb option (we hope this feature is only used for good).
+
+    By default, this command will ignore opers. This behaviour can be suppressed using the --include-opers option.
+
+    To properly kill abusers on another network, combine this command with the 'remote' command in the
+    'networks' plugin and adjust your banmasks accordingly."""
+    permissions.check_permissions(irc, source, ['opercmds.masskill'])
+
+    args = masskill_parser.parse_args(args)
+
+    if args.force_kb:
+        permissions.check_permissions(irc, source, ['opercmds.masskill.force'])
+
+    reason = ' '.join(args.reason)
+
+    results = killed = 0
+
+    userlist_func = irc.match_all_re if use_regex else irc.match_all
+
+    seen_users = set()
+    for uid in userlist_func(args.banmask):
+        userobj = irc.users[uid]
+
+        if irc.is_oper(uid) and not args.include_opers:
+            irc.reply('Skipping killing \x02%s\x02 because they are opered.' % userobj.nick)
+            continue
+        elif irc.get_service_bot(uid):
+            irc.reply('Skipping killing \x02%s\x02 because it is a service client.' % userobj.nick)
+            continue
+
+        relay = world.plugins.get('relay')
+        if relay and hasattr(userobj, 'remote'):
+            # For relay users, forward kill attempts as kickban because we don't want networks k-lining each others' users.
+            bans = [irc.make_channel_ban(uid)]
+            for channel in userobj.channels.copy():  # Look in which channels the user appears to be in locally
+
+                if (args.force_kb or relay.check_claim(irc, channel, source)):
+                    irc.mode(irc.pseudoclient.uid, channel, bans)
+                    irc.kick(irc.pseudoclient.uid, channel, uid, reason)
+
+                    # XXX: code duplication with massban.
+                    try:
+                        irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_MASSKILL_BAN',
+                                        {'target': channel, 'modes': bans, 'parse_as': 'MODE'}])
+                        irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_MASSKILL_KICK',
+                                        {'channel': channel, 'target': uid, 'text': reason, 'parse_as': 'KICK'}])
+                    except:
+                        log.exception('(%s) Failed to send process massban hook; some kickbans may have not '
+                                      'been sent to plugins / relay networks!', irc.name)
+
+                    if uid not in seen_users:  # Don't count users multiple times on different channels
+                        killed += 1
+                else:
+                    irc.reply("Not kicking \x02%s\x02 from \x02%s\x02 because you don't have CLAIM access. If this is "
+                              "another network's channel, ask someone to op you or use the --force-kb option." % (userobj.nick, channel))
+        else:
+            if args.akill:  # TODO: configurable length via strings such as "2w3d5h6m3s" - though month and minute clash this way?
+                if not (userobj.realhost or userobj.ip):
+                    irc.reply("Skipping akill on %s because PyLink doesn't know the real host." % irc.get_hostmask(uid))
+                    continue
+                irc.set_server_ban(irc.pseudoclient.uid, 604800, host=userobj.realhost or userobj.ip or userobj.host, reason=reason)
+            else:
+                irc.kill(irc.pseudoclient.uid, uid, reason)
+                try:
+                    irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_MASSKILL',
+                                    {'target': uid, 'parse_as': 'KILL', 'userdata': userobj, 'text': reason}])
+                except:
+                    log.exception('(%s) Failed to send process massban hook; some kickbans may have not '
+                                  'been sent to plugins / relay networks!', irc.name)
+            killed += 1
+        results += 1
+        seen_users.add(uid)
+    else:
+        log.info('(%s) Ran masskill%s for %s (%s/%s user(s) removed)', irc.name, 're' if use_regex else '',
+                 irc.get_hostmask(source), killed, results)
+        irc.reply('Masskilled %s/%s users.' % (killed, results))
+utils.add_cmd(masskill, aliases=('mkill',))
+
+def masskillre(irc, source, args):
+    """<regular expression> [<kill/ban reason>] [--akill/ak] [--force-kb/-f] [--include-opers/-o]
+
+    Kills all users whose "nick!user@host [gecos]" mask matches the given Python-style regular expression.
+    (https://docs.python.org/3/library/re.html#regular-expression-syntax describes supported syntax)
+
+    The --akill option can also be given to convert kills to akills that expire after 7 days.
+
+    For relay users, attempts to kill are forwarded as a kickban to every channel where the calling user
+    meets claim requirements to set a ban (i.e. this is true if you are opped, if your network is in claim list, etc.;
+    see "help CLAIM" for more specific rules). This can also be extended to all shared channels
+    the user is in using the --force-kb option (we hope this feature is only used for good).
+
+    By default, this command will ignore opers. This behaviour can be suppressed using the --include-opers option.
+
+    \x02Be careful when using this command, as it is easy to make mistakes with regex. Use 'checkbanre'
+    to check your bans first!\x02
+
+    """
+    permissions.check_permissions(irc, source, ['opercmds.masskill.re'])
+    return masskill(irc, source, args, use_regex=True)
+
+utils.add_cmd(masskillre, aliases=('rkill',))
 
 @utils.add_cmd
 def jupe(irc, source, args):
     """<server> [<reason>]
 
-    Admin only, jupes the given server."""
+    Jupes the given server."""
 
-    # Check that the caller is either opered or logged in as admin.
+    permissions.check_permissions(irc, source, ['opercmds.jupe'])
-    irc.checkAuthenticated(source, allowOper=False)
 
     try:
         servername = args[0]
         reason = ' '.join(args[1:]) or "No reason given"
-        desc = "Juped by %s: [%s]" % (irc.getHostmask(source), reason)
+        desc = "Juped by %s: [%s]" % (irc.get_hostmask(source), reason)
     except IndexError:
-        irc.reply('Error: Not enough arguments. Needs 1-2: servername, reason (optional).')
+        irc.error('Not enough arguments. Needs 1-2: servername, reason (optional).')
         return
 
-    if not utils.isServerName(servername):
+    if not irc.is_server_name(servername):
-        irc.reply("Error: Invalid server name '%s'." % servername)
+        irc.error("Invalid server name %r." % servername)
         return
 
-    sid = irc.proto.spawnServer(servername, desc=desc)
+    sid = irc.spawn_server(servername, desc=desc)
 
-    irc.callHooks([irc.pseudoclient.uid, 'OPERCMDS_SPAWNSERVER',
+    irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_SPAWNSERVER',
                    {'name': servername, 'sid': sid, 'text': desc}])
 
     irc.reply("Done.")
 
+def _try_find_target(irc, nick):
+    """
+    Tries to find the target UID for the given nick, raising LookupError if it doesn't exist or is ambiguous.
+    """
+    try:
+        int_u = int(nick)
+    except:
+        int_u = None
+
+    if int_u and int_u in irc.users:
+        return int_u  # Some protocols use numeric UIDs
+    elif nick in irc.users:
+        return nick
+
+    potential_targets = irc.nick_to_uid(nick, multi=True)
+    if not potential_targets:
+        # Whatever we were told to kick doesn't exist!
+        raise LookupError("No such target %r." % nick)
+    elif len(potential_targets) > 1:
+        raise LookupError("Multiple users with the nick %r found: please select the right UID: %s" % (nick, str(potential_targets)))
+    else:
+        return potential_targets[0]
 
 @utils.add_cmd
 def kick(irc, source, args):
-    """<source> <channel> <user> [<reason>]
+    """<channel> <user> [<reason>]
 
-    Admin only. Kicks <user> from <channel> via <source>, where <source> is either the nick of a PyLink client or the SID of a PyLink server."""
+    Kicks <user> from the specified channel."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['opercmds.kick'])
     try:
-        sourcenick = args[0]
+        channel = args[0]
-        channel = irc.toLower(args[1])
-        target = args[2]
-        reason = ' '.join(args[3:])
-    except IndexError:
-        irc.reply("Error: Not enough arguments. Needs 3-4: source nick, channel, target, reason (optional).")
-        return
-
-    # Convert the source and target nicks to UIDs.
-    sender = irc.nickToUid(sourcenick) or sourcenick
-    targetu = irc.nickToUid(target)
-
-    if channel not in irc.channels:  # KICK only works on channels that exist.
-        irc.reply("Error: Unknown channel %r." % channel)
-        return
-
-    if (not irc.isInternalClient(sender)) and \
-            (not irc.isInternalServer(sender)):
-        # Whatever we were told to send the kick from wasn't valid; try to be
-        # somewhat user friendly in the error message
-        irc.reply("Error: No such PyLink client '%s'. The first argument to "
-                  "KICK should be the name of a PyLink client (e.g. '%s'; see "
-                  "'help kick' for details." % (sourcenick,
-                  irc.pseudoclient.nick))
-        return
-    elif not targetu:
-        # Whatever we were told to kick doesn't exist!
-        irc.reply("Error: No such target nick '%s'." % target)
-        return
-
-    irc.proto.kick(sender, channel, targetu, reason)
-    irc.callHooks([sender, 'CHANCMDS_KICK', {'channel': channel, 'target': targetu,
-                                        'text': reason, 'parse_as': 'KICK'}])
-
-@utils.add_cmd
-def kill(irc, source, args):
-    """<source> <target> [<reason>]
-
-    Admin only. Kills <target> via <source>, where <source> is either the nick of a PyLink client or the SID of a PyLink server."""
-    irc.checkAuthenticated(source, allowOper=False)
-    try:
-        sourcenick = args[0]
         target = args[1]
         reason = ' '.join(args[2:])
     except IndexError:
-        irc.reply("Error: Not enough arguments. Needs 3-4: source nick, target, reason (optional).")
+        irc.error("Not enough arguments. Needs 2-3: channel, target, reason (optional).")
+        return
+
+    if channel not in irc.channels:  # KICK only works on channels that exist.
+        irc.error("Unknown channel %r." % channel)
+        return
+
+    targetu = _try_find_target(irc, target)
+
+    sender = irc.pseudoclient.uid
+    irc.kick(sender, channel, targetu, reason)
+    irc.reply("Done.")
+    irc.call_hooks([sender, 'OPERCMDS_KICK', {'channel': channel, 'target': targetu,
+                                              'text': reason, 'parse_as': 'KICK'}])
+
+@utils.add_cmd
+def kill(irc, source, args):
+    """<target> [<reason>]
+
+    Kills the given target."""
+    permissions.check_permissions(irc, source, ['opercmds.kill'])
+    try:
+        target = args[0]
+        reason = ' '.join(args[1:])
+    except IndexError:
+        irc.error("Not enough arguments. Needs 1-2: target, reason (optional).")
         return
 
     # Convert the source and target nicks to UIDs.
-    sender = irc.nickToUid(sourcenick) or sourcenick
+    sender = irc.pseudoclient.uid
-    targetu = irc.nickToUid(target)
+
+    targetu = _try_find_target(irc, target)
+
+    if irc.pseudoclient.uid == targetu:
+        irc.error("Cannot kill the main PyLink client!")
+        return
+
     userdata = irc.users.get(targetu)
 
-    if (not irc.isInternalClient(sender)) and \
+    reason = "Requested by %s: %s" % (irc.get_friendly_name(source), reason)
-            (not irc.isInternalServer(sender)):
-        # Whatever we were told to send the kick from wasn't valid; try to be
-        # somewhat user friendly in the error message
-        irc.reply("Error: No such PyLink client '%s'. The first argument to "
-                  "KILL should be the name of a PyLink client (e.g. '%s'; see "
-                  "'help kill' for details." % (sourcenick,
-                  irc.pseudoclient.nick))
-        return
-    elif targetu not in irc.users:
-        # Whatever we were told to kick doesn't exist!
-        irc.reply("Error: No such nick '%s'." % target)
-        return
 
-    irc.proto.kill(sender, targetu, reason)
+    irc.kill(sender, targetu, reason)
 
-    # Format the kill reason properly in hooks.
+    irc.reply("Done.")
-    reason = "Killed (%s (%s))" % (irc.getFriendlyName(sender), reason)
+    irc.call_hooks([source, 'OPERCMDS_KILL', {'target': targetu, 'text': reason,
-
+                                              'userdata': userdata, 'parse_as': 'KILL'}])
-    irc.callHooks([sender, 'CHANCMDS_KILL', {'target': targetu, 'text': reason,
-                                        'userdata': userdata, 'parse_as': 'KILL'}])
 
 @utils.add_cmd
 def mode(irc, source, args):
     """<channel> <modes>
 
-    Oper-only, sets modes <modes> on the target channel."""
+    Sets the given modes on the target channel."""
 
-    # Check that the caller is either opered or logged in as admin.
+    permissions.check_permissions(irc, source, ['opercmds.mode'])
-    irc.checkAuthenticated(source)
 
     try:
         target, modes = args[0], args[1:]
     except IndexError:
-        irc.reply('Error: Not enough arguments. Needs 2: target, modes to set.')
+        irc.error('Not enough arguments. Needs 2: target, modes to set.')
         return
 
     if target not in irc.channels:
-        irc.reply("Error: Unknown channel '%s'." % target)
+        irc.error("Unknown channel %r." % target)
         return
     elif not modes:
         # No modes were given before parsing (i.e. mode list was blank).
-        irc.reply("Error: No valid modes were given.")
+        irc.error("No valid modes were given.")
         return
 
-    parsedmodes = irc.parseModes(target, modes)
+    parsedmodes = irc.parse_modes(target, modes)
 
     if not parsedmodes:
         # Modes were given but they failed to parse into anything meaningful.
         # For example, "mode #somechan +o" would be erroneous because +o
         # requires an argument!
-        irc.reply("Error: No valid modes were given.")
+        irc.error("No valid modes were given.")
         return
 
-    irc.proto.mode(irc.pseudoclient.uid, target, parsedmodes)
+    irc.mode(irc.pseudoclient.uid, target, parsedmodes)
 
     # Call the appropriate hooks for plugins like relay.
-    irc.callHooks([irc.pseudoclient.uid, 'OPERCMDS_MODEOVERRIDE',
+    irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_MODE',
                    {'target': target, 'modes': parsedmodes, 'parse_as': 'MODE'}])
 
     irc.reply("Done.")
@@ -203,21 +457,60 @@ def mode(irc, source, args):
 def topic(irc, source, args):
     """<channel> <topic>
 
-    Admin only. Updates the topic in a channel."""
+    Changes the topic in a channel."""
-    irc.checkAuthenticated(source, allowOper=False)
+    permissions.check_permissions(irc, source, ['opercmds.topic'])
     try:
         channel = args[0]
         topic = ' '.join(args[1:])
     except IndexError:
-        irc.reply("Error: Not enough arguments. Needs 2: channel, topic.")
+        irc.error("Not enough arguments. Needs 2: channel, topic.")
         return
 
     if channel not in irc.channels:
-        irc.reply("Error: Unknown channel %r." % channel)
+        irc.error("Unknown channel %r." % channel)
         return
 
-    irc.proto.topic(irc.pseudoclient.uid, channel, topic)
+    irc.topic(irc.pseudoclient.uid, channel, topic)
 
-    irc.callHooks([irc.pseudoclient.uid, 'CHANCMDS_TOPIC',
+    irc.reply("Done.")
+    irc.call_hooks([irc.pseudoclient.uid, 'OPERCMDS_TOPIC',
                    {'channel': channel, 'text': topic, 'setter': source,
                     'parse_as': 'TOPIC'}])
+
+@utils.add_cmd
+def chghost(irc, source, args):
+    """<user> <new host>
+
+    Changes the visible host of the target user."""
+    _chgfield(irc, source, args, 'host')
+
+@utils.add_cmd
+def chgident(irc, source, args):
+    """<user> <new ident>
+
+    Changes the ident of the target user."""
+    _chgfield(irc, source, args, 'ident')
+
+@utils.add_cmd
+def chgname(irc, source, args):
+    """<user> <new name>
+
+    Changes the GECOS (realname) of the target user."""
+    _chgfield(irc, source, args, 'name', 'GECOS')
+
+def _chgfield(irc, source, args, human_field, internal_field=None):
+    """Helper function for chghost/chgident/chgname."""
+    permissions.check_permissions(irc, source, ['opercmds.chg' + human_field])
+    try:
+        target = args[0]
+        new = args[1]
+    except IndexError:
+        irc.error("Not enough arguments. Needs 2: target, new %s." % human_field)
+        return
+
+    # Find the user
+    targetu = _try_find_target(irc, target)
+
+    internal_field = internal_field or human_field.upper()
+    irc.update_client(targetu, internal_field, new)
+    irc.reply("Done.")

plugins/raw.py

@@ -0,0 +1,33 @@
+"""
+raw.py: Provides a 'raw' command for sending raw text to IRC.
+"""
+from pylinkirc import utils
+from pylinkirc.coremods import permissions
+from pylinkirc.log import log
+from pylinkirc import conf
+
+@utils.add_cmd
+def raw(irc, source, args):
+    """<text>
+
+    Sends raw text to the IRC server.
+
+    Use with caution - This command is only officially supported on Clientbot networks."""
+    if not conf.conf['pylink'].get("raw_enabled", False):
+        raise RuntimeError("Raw commands are not supported on this protocol")
+
+    # exec.raw is included for backwards compatibility with PyLink 1.x
+    permissions.check_permissions(irc, source, ['raw.raw', 'exec.raw'])
+
+    args = ' '.join(args)
+    if not args.strip():
+        irc.reply('No text entered!')
+        return
+
+    # Note: This is loglevel debug so that we don't risk leaking things like
+    # NickServ passwords on Clientbot networks.
+    log.debug('(%s) Sending raw text %r to IRC for %s', irc.name, args,
+              irc.get_hostmask(source))
+    irc.send(args)
+
+    irc.reply("Done.")

plugins/relay_clientbot.py

@@ -1,65 +1,82 @@
 # relay_clientbot.py: Clientbot extensions for Relay
+import shlex
 import string
-import collections
 import time
 
-from pylinkirc import utils, conf, world
+from pylinkirc import conf, utils, world
 from pylinkirc.log import log
 
-# TODO: document configurable styles in relay::clientbot_styles::COMMAND_NAME
+# Clientbot default styles:
 # These use template strings as documented @ https://docs.python.org/3/library/string.html#template-strings
-default_styles = {'MESSAGE': '\x02[$colored_netname]\x02 <$colored_sender> $text',
+default_styles = {'MESSAGE': '\x02[$netname]\x02 <$mode_prefix$colored_sender> $text',
-                  'KICK': '\x02[$colored_netname]\x02 - $colored_sender$sender_identhost has kicked $target_nick from $channel ($text)',
+                  'KICK': '\x02[$netname]\x02 - $colored_sender$sender_identhost has kicked $target_nick from $channel ($text)',
-                  'PART': '\x02[$colored_netname]\x02 - $colored_sender$sender_identhost has left $channel ($text)',
+                  'PART': '\x02[$netname]\x02 - $colored_sender$sender_identhost has left $channel ($text)',
-                  'JOIN': '\x02[$colored_netname]\x02 - $colored_sender$sender_identhost has joined $channel',
+                  'JOIN': '\x02[$netname]\x02 - $colored_sender$sender_identhost has joined $channel',
-                  'NICK': '\x02[$colored_netname]\x02 - $colored_sender$sender_identhost is now known as $newnick',
+                  'NICK': '\x02[$netname]\x02 - $colored_sender$sender_identhost is now known as $newnick',
-                  'QUIT': '\x02[$colored_netname]\x02 - $colored_sender$sender_identhost has quit ($text)',
+                  'QUIT': '\x02[$netname]\x02 - $colored_sender$sender_identhost has quit ($text)',
-                  'ACTION': '\x02[$colored_netname]\x02 * $colored_sender $text',
+                  'ACTION': '\x02[$netname]\x02 * $mode_prefix$colored_sender $text',
-                  'NOTICE': '\x02[$colored_netname]\x02 - Notice from $colored_sender: $text',
+                  'NOTICE': '\x02[$netname]\x02 - Notice from $mode_prefix$colored_sender: $text',
-                  'SQUIT': '\x02[$colored_netname]\x02 - Netsplit lost users: $colored_nicks',
+                  'SQUIT': '\x02[$netname]\x02 - Netsplit lost users: $colored_nicks',
-                  'SJOIN': '\x02[$colored_netname]\x02 - Netjoin gained users: $colored_nicks',
+                  'SJOIN': '\x02[$netname]\x02 - Netjoin gained users: $colored_nicks',
+                  'MODE': '\x02[$netname]\x02 - $colored_sender$sender_identhost sets mode $modes on $channel',
+                  'PM': 'PM from $sender on $netname: $text',
+                  'PNOTICE': '<$sender> $text',
                   }
 
 def color_text(s):
     """
-    Returns a colorized version of the given text based on a simple hash algorithm
+    Returns a colorized version of the given text based on a simple hash algorithm.
-    (sum of all characters).
     """
-    colors = ('02', '03', '04', '05', '06', '07', '08', '09', '10', '11',
+    if not s:
-              '12', '13')
+        return s
-    num = sum([ord(char) for char in s])
+    colors = ('03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '15')
-    num = num % len(colors)
+    hash_output = hash(s.encode())
+    num = hash_output % len(colors)
     return "\x03%s%s\x03" % (colors[num], s)
 
 def cb_relay_core(irc, source, command, args):
     """
-    This function takes Clientbot actions and outputs them to a channel as regular text.
+    This function takes Clientbot events and formats them as text to the target channel / user.
     """
     real_command = command.split('_')[-1]
 
     relay = world.plugins.get('relay')
+
+    private = False
+
     if irc.pseudoclient and relay:
         try:
-            sourcename = irc.getFriendlyName(source)
+            sourcename = irc.get_friendly_name(source)
         except KeyError:  # User has left due to /quit
             sourcename = args['userdata'].nick
 
-        relay_conf = conf.conf.get('relay', {})
+        relay_conf = conf.conf.get('relay') or {}
 
         # Be less floody on startup: don't relay non-PRIVMSGs for the first X seconds after connect.
-        startup_delay = relay_conf.get('clientbot_startup_delay', 5)
+        startup_delay = relay_conf.get('clientbot_startup_delay', 20)
+
+        target = args.get('target')
+        if isinstance(target, str):
+            # Remove STATUSMSG prefixes (e.g. @#channel) before checking whether target is a channel
+            target = target.lstrip(''.join(irc.prefixmodes.values()))
 
         # Special case for CTCPs.
         if real_command == 'MESSAGE':
             # CTCP action, format accordingly
             if (not args.get('is_notice')) and args['text'].startswith('\x01ACTION ') and args['text'].endswith('\x01'):
                 args['text'] = args['text'][8:-1]
-
                 real_command = 'ACTION'
 
+            elif not irc.is_channel(target):
+                # Target is a user; handle this accordingly.
+                if relay_conf.get('allow_clientbot_pms'):
+                    real_command = 'PNOTICE' if args.get('is_notice') else 'PM'
+                    private = True
+
             # Other CTCPs are ignored
             elif args['text'].startswith('\x01'):
                 return
+
             elif args.get('is_notice'):  # Different syntax for notices
                 real_command = 'NOTICE'
         elif (time.time() - irc.start_ts) < startup_delay:
@@ -70,65 +87,108 @@ def cb_relay_core(irc, source, command, args):
         # Try to fetch the format for the given command from the relay:clientbot_styles:$command
         # key, falling back to one defined in default_styles above, and then nothing if not found
         # there.
-        text_template = relay_conf.get('clientbot_styles', {}).get(real_command,
+        text_template = irc.get_service_options('relay', 'clientbot_styles', dict).get(
-                        default_styles.get(real_command, ''))
+                            real_command, default_styles.get(real_command, ''))
         text_template = string.Template(text_template)
 
         if text_template:
-            # Get the original client that the relay client source was meant for.
+            if irc.get_service_bot(source):
-            log.debug('(%s) relay_cb_core: Trying to find original sender (user) for %s', irc.name, source)
+                # HACK: service bots are global and lack the relay state we look for.
-            try:
+                # just pretend the message comes from the current network.
-                origuser = relay.getOrigUser(irc, source) or args['userdata'].remote
+                log.debug('(%s) relay_cb_core: Overriding network origin to local (source=%s)', irc.name, source)
-            except (AttributeError, KeyError):
+                sourcenet = irc.name
-                log.debug('(%s) relay_cb_core: Trying to find original sender (server) for %s. serverdata=%s', irc.name, source, args.get('serverdata'))
+                realsource = source
+            else:
+                # Get the original client that the relay client source was meant for.
+                log.debug('(%s) relay_cb_core: Trying to find original sender (user) for %s', irc.name, source)
                 try:
-                    origuser = ((args.get('serverdata') or irc.servers[source]).remote,)
+                    origuser = relay.get_orig_user(irc, source) or args['userdata'].remote
                 except (AttributeError, KeyError):
-                    return
+                    log.debug('(%s) relay_cb_core: Trying to find original sender (server) for %s. serverdata=%s', irc.name, source, args.get('serverdata'))
+                    try:
+                        localsid = args.get('serverdata') or irc.servers[source]
+                        origuser = (localsid.remote, world.networkobjects[localsid.remote].uplink)
+                    except (AttributeError, KeyError):
+                        return
+
+                log.debug('(%s) relay_cb_core: Original sender found as %s', irc.name, origuser)
+                sourcenet, realsource = origuser
 
-            log.debug('(%s) relay_cb_core: Original sender found as %s', irc.name, origuser)
-            netname = origuser[0]
             try:  # Try to get the full network name
-                netname = conf.conf['servers'][netname]['netname'].lower()
+                netname = conf.conf['servers'][sourcenet]['netname']
             except KeyError:
-                pass
+                netname = sourcenet
 
             # Figure out where the message is destined to.
-            target = args.get('channel') or args.get('target')
+            stripped_target = target = args.get('channel') or args.get('target')
-            if target is None or not utils.isChannel(target):
+            if isinstance(target, str):
-                # Quit and nick messages are not channel specific. Figure out all channels that the
+                # HACK: cheap fix to prevent @#channel messages from interpreted as non-channel specific
-                # sender shares over the relay, and relay them that way.
+                stripped_target = target.lstrip(''.join(irc.prefixmodes.values()))
+
+            if target is None or not (irc.is_channel(stripped_target) or private):
+                # Non-channel specific message (e.g. QUIT or NICK). If this isn't a PM, figure out
+                # all channels that the sender shares over the relay, and relay them to those
+                # channels.
                 userdata = args.get('userdata') or irc.users.get(source)
                 if not userdata:
                     # No user data given. This was probably some other global event such as SQUIT.
                     userdata = irc.pseudoclient
 
-                channels = [channel for channel in userdata.channels if relay.getRelay((irc.name, channel))]
+                targets = [channel for channel in userdata.channels if relay.get_relay(irc, channel)]
             else:
                 # Pluralize the channel so that we can iterate over it.
-                channels = [target]
+                targets = [target]
-            log.debug('(%s) relay_cb_core: Relaying event %s to channels: %s', irc.name, real_command, channels)
+                args['channel'] = stripped_target
+            log.debug('(%s) relay_cb_core: Relaying event %s to channels: %s', irc.name, real_command, targets)
 
+            identhost = ''
             if source in irc.users:
                 try:
-                    identhost = irc.getHostmask(source).split('!')[-1]
+                    identhost = irc.get_hostmask(source).split('!')[-1]
                 except KeyError:  # User got removed due to quit
-                    identhost = '%s@%s' % (args['olduser'].ident, args['olduser'].host)
+                    identhost = '%s@%s' % (args['userdata'].ident, args['userdata'].host)
                 # This is specifically spaced so that ident@host is only shown for users that have
                 # one, and not servers.
                 identhost = ' (%s)' % identhost
-            else:
-                identhost = ''
 
             # $target_nick: Convert the target for kicks, etc. from a UID to a nick
             if args.get("target") in irc.users:
-                args["target_nick"] = irc.getFriendlyName(args['target'])
+                args["target_nick"] = irc.get_friendly_name(args['target'])
 
-            args.update({'netname': netname, 'sender': sourcename, 'sender_identhost': identhost,
+            # Join up modes from their list form
-                         'colored_sender': color_text(sourcename), 'colored_netname': color_text(netname)})
+            if args.get('modes'):
+                args['modes'] = irc.join_modes(args['modes'])
 
-            for channel in channels:
+            mode_prefix = ''
-                cargs = args.copy()  # Copy args list to manipualte them in a channel specific way
+            if 'channel' in args:
+                # Display the real (remote) channel name instead of the local one, if applicable.
+                args['local_channel'] = args['channel']
+                log.debug('(%s) relay_clientbot: coersing $channel from %s to %s', irc.name, args['local_channel'], args['channel'])
+
+                sourceirc = world.networkobjects.get(sourcenet)
+                log.debug('(%s) relay_clientbot: Checking prefix modes for %s on %s (relaying to %s)',
+                          irc.name, realsource, sourcenet, args['channel'])
+                if sourceirc:
+                    args['channel'] = remotechan = relay.get_remote_channel(irc, sourceirc, args['channel'])
+                    if source in irc.users and remotechan in sourceirc.channels and \
+                            realsource in sourceirc.channels[remotechan].users:
+                        # Fetch the prefixmode prefixes (e.g. ~@%) for the sender, if available.
+                        prefixmodes = sourceirc.channels[remotechan].get_prefix_modes(realsource)
+                        log.debug('(%s) relay_clientbot: got prefix modes %s for %s on %s@%s',
+                                  irc.name, prefixmodes, realsource, remotechan, sourcenet)
+                        if prefixmodes:
+                            # Only pick the highest prefix.
+                            mode_prefix = sourceirc.prefixmodes.get(
+                                sourceirc.cmodes.get(prefixmodes[0]))
+
+            args.update({
+                'netname': netname, 'sender': sourcename, 'sender_identhost': identhost,
+                'colored_sender': color_text(sourcename), 'colored_netname': color_text(netname),
+                'mode_prefix': mode_prefix
+            })
+
+            for target in targets:
+                cargs = args.copy()  # Copy args list to manipulate them in a channel specific way
 
                 # $nicks / $colored_nicks: used when the event affects multiple users, such as SJOIN or SQUIT.
                 # For SJOIN, this is simply a list of nicks. For SQUIT, this is sent as a dict
@@ -136,10 +196,9 @@ def cb_relay_core(irc, source, command, args):
                 # still have to be relayed as such.
                 nicklist = args.get('nicks')
                 if nicklist:
-
                     # Get channel-specific nick list if relevent.
-                    if type(nicklist) == collections.defaultdict:
+                    if isinstance(nicklist, dict):
-                        nicklist = nicklist.get(channel, [])
+                        nicklist = nicklist.get(target, [])
 
                     # Ignore if no nicks are affected on the channel.
                     if not nicklist:
@@ -152,7 +211,8 @@ def cb_relay_core(irc, source, command, args):
                     cargs['colored_nicks'] = ', '.join(colored_nicks)
 
                 text = text_template.safe_substitute(cargs)
-                irc.proto.message(irc.pseudoclient.uid, channel, text)
+                # PMs are always sent as notice - this prevents unknown command loops with bots.
+                irc.msg(target, text, loopback=False, notice=private)
 
 utils.add_hook(cb_relay_core, 'CLIENTBOT_MESSAGE')
 utils.add_hook(cb_relay_core, 'CLIENTBOT_KICK')
@@ -162,3 +222,53 @@ utils.add_hook(cb_relay_core, 'CLIENTBOT_QUIT')
 utils.add_hook(cb_relay_core, 'CLIENTBOT_NICK')
 utils.add_hook(cb_relay_core, 'CLIENTBOT_SJOIN')
 utils.add_hook(cb_relay_core, 'CLIENTBOT_SQUIT')
+utils.add_hook(cb_relay_core, 'RELAY_RAW_MODE')
+
+@utils.add_cmd
+def rpm(irc, source, args):
+    """<target nick/UID> <text>
+
+    Sends PMs to users over Relay, if Clientbot PMs are enabled.
+    If the target nick has spaces in it, you may quote the nick as "nick".
+    """
+    args = shlex.split(' '.join(args))  # HACK: use shlex.split so that quotes are preserved
+    try:
+        target = args[0]
+        text = ' '.join(args[1:])
+    except IndexError:
+        irc.error('Not enough arguments. Needs 2: target nick and text.')
+        return
+
+    relay = world.plugins.get('relay')
+    if irc.has_cap('can-spawn-clients'):
+        irc.error('This command is only supported on Clientbot networks. Try /msg %s <text>' % target)
+        return
+    elif relay is None:
+        irc.error('PyLink Relay is not loaded.')
+        return
+    elif not text:
+        irc.error('No text given.')
+        return
+    elif not conf.conf.get('relay').get('allow_clientbot_pms'):
+        irc.error('Private messages with users connected via Clientbot have been '
+                  'administratively disabled.')
+        return
+
+    if target in irc.users:
+        uids = [target]
+    else:
+        uids = irc.nick_to_uid(target, multi=True, filterfunc=lambda u: relay.is_relay_client(irc, u))
+
+    if not uids:
+        irc.error('Unknown user %s.' % target)
+        return
+    elif len(uids) > 1:
+        targets = ['\x02%s\x02: %s @ %s' % (uid, irc.get_hostmask(uid), irc.users[uid].remote[0]) for uid in uids]
+        irc.error('Please select the target you want to PM: %s' % (', '.join(targets)))
+        return
+    else:
+        assert not irc.is_internal_client(source), "rpm is not allowed from PyLink bots"
+
+        # Send the message through relay by faking a hook for its handler.
+        relay.handle_messages(irc, source, 'RELAY_CLIENTBOT_PRIVMSG', {'target': uids[0], 'text': text})
+        irc.reply('Message sent.')

plugins/servermaps.py

@@ -0,0 +1,125 @@
+# servermaps.py: Maps out connected IRC servers.
+
+import collections
+
+from pylinkirc import utils, world
+from pylinkirc.coremods import permissions
+from pylinkirc.log import log
+
+DEFAULT_PERMISSIONS = {"$ircop": ['servermaps.localmap']}
+
+def main(irc=None):
+    """Servermaps plugin main function, called on plugin load."""
+    # Register our permissions.
+    permissions.add_default_permissions(DEFAULT_PERMISSIONS)
+
+def die(irc=None):
+    """Servermaps plugin die function, called on plugin unload."""
+    permissions.remove_default_permissions(DEFAULT_PERMISSIONS)
+
+def _percent(num, total):
+    return '%.1f' % (num/total*100)
+
+def _map(irc, source, args, show_relay=True):
+    """[<network>]
+
+    Shows the network map for the given network, or the current network if not specified."""
+
+    if show_relay:
+        perm = 'servermaps.map'
+    else:
+        perm = 'servermaps.localmap'
+    permissions.check_permissions(irc, source, [perm])
+
+    try:
+        netname = args[0]
+    except IndexError:
+        netname = irc.name
+
+    try:
+        ircobj = world.networkobjects[netname]
+    except KeyError:
+        irc.error('no such network %s' % netname)
+        return
+
+    servers = collections.defaultdict(set)
+    hostsid = ircobj.sid
+    usercount = len(ircobj.users)
+
+    # Iterate over every connected server on every network.
+    for remotenet, remoteirc in world.networkobjects.items():
+        for sid, serverobj in remoteirc.servers.copy().items():
+            if sid == remoteirc.sid:  # Don't re-add our own SID to the index
+                continue
+
+            # Save the server as UNDER its uplink.
+            servers[(remotenet, serverobj.uplink or remoteirc.sid)].add(sid)
+
+    log.debug('(%s) servermaps.map servers fetched for %s: %s', irc.name, netname, servers)
+
+    reply = lambda text: irc.reply(text, private=True)
+
+    def showall(ircobj, sid, hops=0, is_relay_server=False):
+        log.debug('servermaps: got showall() for SID %s on network %s', sid, ircobj.name)
+        serverlist = ircobj.servers.copy()
+        netname = ircobj.name
+
+        if hops == 0:
+            # Show our root server once.
+            rootusers = len(serverlist[sid].users)
+            reply('\x02%s\x02[%s]: %s user(s) (%s%%) {hopcount: %d}' % (serverlist[sid].name, sid,
+                  rootusers, _percent(rootusers, usercount), serverlist[sid].hopcount))
+
+        log.debug('(%s) servermaps: servers under sid %s: %s', irc.name, sid, servers)
+
+        # Every time we descend a server to process its map, raise the hopcount used in formatting.
+        hops += 1
+        leaves = servers[(netname, sid)]
+        for leafcount, leaf in enumerate(leaves):
+            if is_relay_server and hasattr(serverlist[leaf], 'remote'):
+                # Don't show relay subservers more than once.
+                continue
+
+            serverusers = len(serverlist[leaf].users)
+            if is_relay_server:
+                # Skip showing user data for relay servers.
+                reply("%s\x02%s\x02[%s] (via PyLink Relay)" %
+                      ('    '*hops, serverlist[leaf].name, leaf))
+            else:
+                reply("%s\x02%s\x02[%s]: %s user(s) (%s%%) {hopcount: %d}" %
+                      ('    '*hops, serverlist[leaf].name, leaf,
+                       serverusers, _percent(serverusers, usercount), serverlist[leaf].hopcount))
+            showall(ircobj, leaf, hops, is_relay_server=is_relay_server)
+
+            if (not is_relay_server) and hasattr(serverlist[leaf], 'remote') and show_relay:
+                # This is a relay server - display the remote map of the network it represents
+                relay_server = serverlist[leaf].remote
+                remoteirc = world.networkobjects[relay_server]
+                if remoteirc.has_cap('can-track-servers'):
+                    # Only ever show relay subservers once - this prevents infinite loops.
+                    showall(remoteirc, remoteirc.sid, hops=hops, is_relay_server=True)
+                else:
+                    # For Clientbot links, show the server we're actually connected to.
+                    reply("%s\x02%s\x02 (actual server name)" %
+                          ('    '*(hops+1), remoteirc.uplink))
+
+        else:
+            # Afterwards, decrement the hopcount.
+            hops -= 1
+
+    # Start the map at our PyLink server
+    firstserver = hostsid
+    showall(ircobj, firstserver)
+    serverlist = irc.servers
+    reply('Total %s users on %s local servers - average of %1.f per server' % (usercount, len(serverlist),
+          usercount/len(serverlist)))
+
+utils.add_cmd(_map, 'map')
+
+@utils.add_cmd
+def localmap(irc, source, args):
+    """[<network>]
+
+    Shows the network map for the given network, or the current network if not specified.
+    This command does not expand Relay subservers."""
+    _map(irc, source, args, show_relay=False)

plugins/servprotect.py

@@ -1,24 +1,42 @@
 # servprotect.py: Protects against KILL and nick collision floods
-from expiringdict import ExpiringDict
 
-from pylinkirc import utils
+import threading
+
+from pylinkirc import conf, utils
 from pylinkirc.log import log
 
-# TODO: make length and time configurable
+try:
-savecache = ExpiringDict(max_len=5, max_age_seconds=10)
+    from cachetools import TTLCache
-killcache = ExpiringDict(max_len=5, max_age_seconds=10)
+except ImportError:
+    log.warning('servprotect: expiringdict support is deprecated as of PyLink 3.0; consider installing cachetools instead')
+    from expiringdict import ExpiringDict as TTLCache
+
+# check for definitions
+servprotect_conf = conf.conf.get('servprotect', {})
+length = servprotect_conf.get('length', 10)
+age = servprotect_conf.get('age', 10)
+
+def _new_cache_dict():
+    return TTLCache(length, age)
+
+savecache = _new_cache_dict()
+killcache = _new_cache_dict()
+lock = threading.Lock()
 
 def handle_kill(irc, numeric, command, args):
     """
     Tracks kills against PyLink clients. If too many are received,
     automatically disconnects from the network.
     """
-    if killcache.setdefault(irc.name, 1) >= 5:
-        log.error('(%s) servprotect: Too many kills received, aborting!', irc.name)
-        irc.disconnect()
 
-    log.debug('(%s) servprotect: Incrementing killcache by 1', irc.name)
+    if (args['userdata'] and irc.is_internal_server(args['userdata'].server)) or irc.is_internal_client(args['target']):
-    killcache[irc.name] += 1
+        with lock:
+            if killcache.setdefault(irc.name, 1) >= length:
+                log.error('(%s) servprotect: Too many kills received, aborting!', irc.name)
+                irc.disconnect()
+
+            log.debug('(%s) servprotect: Incrementing killcache by 1', irc.name)
+            killcache[irc.name] += 1
 
 utils.add_hook(handle_kill, 'KILL')
 
@@ -27,11 +45,13 @@ def handle_save(irc, numeric, command, args):
     Tracks SAVEs (nick collision) against PyLink clients. If too many are received,
     automatically disconnects from the network.
     """
-    if savecache.setdefault(irc.name, 0) >= 5:
+    if irc.is_internal_client(args['target']):
-        log.error('(%s) servprotect: Too many nick collisions, aborting!', irc.name)
+        with lock:
-        irc.disconnect()
+            if savecache.setdefault(irc.name, 0) >= length:
+                log.error('(%s) servprotect: Too many nick collisions, aborting!', irc.name)
+                irc.disconnect()
 
-    log.debug('(%s) servprotect: Incrementing savecache by 1', irc.name)
+            log.debug('(%s) servprotect: Incrementing savecache by 1', irc.name)
-    savecache[irc.name] += 1
+            savecache[irc.name] += 1
 
 utils.add_hook(handle_save, 'SAVE')

plugins/stats.py

@@ -0,0 +1,129 @@
+"""
+stats.py: Simple statistics for PyLink IRC Services.
+"""
+import datetime
+import time
+
+from pylinkirc import conf, utils, world
+from pylinkirc.coremods import permissions
+from pylinkirc.log import log
+
+
+def timediff(before, now):
+    """
+    Returns the time difference between "before" and "now" as a formatted string.
+    """
+    td = datetime.timedelta(seconds=now-before)
+    days = td.days
+
+    hours, leftover = divmod(td.seconds, 3600)
+    minutes, seconds = divmod(leftover, 60)
+
+    # XXX: I would make this more configurable but it's a lot of work for little gain,
+    # since there's no strftime for time differences.
+    return '%d day%s, %02d:%02d:%02d' % (td.days, 's' if td.days != 1 else '',
+                                         hours, minutes, seconds)
+
+# From RFC 2822: https://tools.ietf.org/html/rfc2822.html#section-3.3
+DEFAULT_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S +0000"
+
+@utils.add_cmd
+def uptime(irc, source, args):
+    """[<network> / --all]
+
+    Returns the uptime for PyLink and the given network's connection (or the current network if not specified).
+    The --all argument can also be given to show the uptime for all networks."""
+    permissions.check_permissions(irc, source, ['stats.uptime'])
+
+    try:
+        network = args[0]
+    except IndexError:
+        network = irc.name
+
+    if network == '--all':  # XXX: we really need smart argument parsing some time
+        # Filter by all connected networks.
+        ircobjs = {k:v for k,v in world.networkobjects.items() if v.connected.is_set()}
+    else:
+        try:
+            ircobjs = {network: world.networkobjects[network]}
+        except KeyError:
+            irc.error("No such network %r." % network)
+            return
+        if not world.networkobjects[network].connected.is_set():
+            irc.error("Network %s is not connected." % network)
+            return
+
+    current_time = int(time.time())
+    time_format = conf.conf.get('stats', {}).get('time_format', DEFAULT_TIME_FORMAT)
+
+    irc.reply("PyLink uptime: \x02%s\x02 (started on %s)" %
+              (timediff(world.start_ts, current_time),
+               time.strftime(time_format, time.gmtime(world.start_ts))
+              )
+             )
+
+    for network, ircobj in sorted(ircobjs.items()):
+        irc.reply("Connected to %s: \x02%s\x02 (connected on %s)" %
+                  (network,
+                   timediff(ircobj.start_ts, current_time),
+                   time.strftime(time_format, time.gmtime(ircobj.start_ts))
+                  )
+                 )
+
+def handle_stats(irc, source, command, args):
+    """/STATS handler. Currently supports the following:
+
+    c - link blocks
+    o - oper blocks (accounts)
+    u - shows uptime
+    """
+
+    stats_type = args['stats_type'][0].lower()  # stats_type shouldn't be more than 1 char anyways
+
+    perms = ['stats.%s' % stats_type]
+
+    if stats_type == 'u':
+        perms.append('stats.uptime')  # Consistency
+
+    try:
+        permissions.check_permissions(irc, source, perms)
+    except utils.NotAuthorizedError as e:
+        # Note, no irc.error() because this is not a command, but a handler
+        irc.msg(source, 'Error: %s' % e, notice=True)
+        return
+
+    log.info('(%s) /STATS %s requested by %s', irc.name, stats_type, irc.get_hostmask(source))
+
+    def _num(num, text):
+        irc.numeric(args['target'], num, source, text)
+
+    if stats_type == 'c':
+        # 213/RPL_STATSCLINE: "C <host> * <name> <port> <class>"
+        for netname, serverdata in sorted(conf.conf['servers'].items()):
+            # We're cramming as much as we can into the class field...
+            _num(213, "C %s * %s %s [%s:%s:%s]" %
+                 (serverdata.get('ip', '0.0.0.0'),
+                  netname,
+                  serverdata.get('port', 0),
+                  serverdata['protocol'],
+                  'ssl' if serverdata.get('ssl') else 'no-ssl',
+                  serverdata.get('encoding', 'utf-8'))
+                 )
+    elif stats_type == 'o':
+        # 243/RPL_STATSOLINE: "O <hostmask> * <nick> [:<info>]"
+        # New style accounts only!
+        for accountname, accountdata in conf.conf['login'].get('accounts', {}).items():
+            networks = accountdata.get('networks', [])
+            if irc.name in networks or not networks:
+                hosts = ' '.join(accountdata.get('hosts', ['*@*']))
+                needoper = 'needoper' if accountdata.get('require_oper') else ''
+                _num(243, "O %s * %s :%s" % (hosts, accountname, needoper))
+
+    elif stats_type == 'u':
+        # 242/RPL_STATSUPTIME: ":Server Up <days> days <hours>:<minutes>:<seconds>"
+        _num(242, ':Server Up %s' % timediff(world.start_ts, int(time.time())))
+
+    else:
+        log.info('(%s) Unknown /STATS type %r requested by %s', irc.name, stats_type, irc.get_hostmask(source))
+    _num(219, "%s :End of /STATS report" % stats_type)
+utils.add_hook(handle_stats, 'STATS')

protocols/__init__.py

@@ -1 +1,2 @@
-# Stub so that pylinkirc.protocols is a module
+# Abstract modules containing shared protocol code; modules higher in the hierarchy go first
+common_modules = ['ircs2s_common', 'ts6_common']

protocols/hybrid.py

@@ -1,45 +1,50 @@
+"""
+hybrid.py: IRCD-Hybrid protocol module for PyLink.
+"""
+
 import time
-import sys
-import os
-import re
 
-from pylinkirc import utils
+from pylinkirc import conf
-from pylinkirc.log import log
 from pylinkirc.classes import *
-from pylinkirc.protocols.ts6 import *
+from pylinkirc.log import log
+from pylinkirc.protocols.ts6 import TS6Protocol
 
+__all__ = ['HybridProtocol']
+
+
+# This protocol module inherits from the TS6 protocol.
 class HybridProtocol(TS6Protocol):
-    def __init__(self, irc):
+    def __init__(self, *args, **kwargs):
-        # This protocol module inherits from the TS6 protocol.
+        super().__init__(*args, **kwargs)
-        super().__init__(irc)
 
         self.casemapping = 'ascii'
-        self.caps = {}
         self.hook_map = {'EOB': 'ENDBURST', 'TBURST': 'TOPIC', 'SJOIN': 'JOIN'}
-        self.has_eob = False
+        self.protocol_caps -= {'slash-in-hosts'}
 
-    def connect(self):
+    def post_connect(self):
         """Initializes a connection to a server."""
-        ts = self.irc.start_ts
+        ts = self.start_ts
-        self.has_eob = False
+        f = self.send
-        f = self.irc.send
 
         # https://github.com/grawity/irc-docs/blob/master/server/ts6.txt#L80
+        # Note: according to hybrid source code, +p is paranoia, noknock,
+        # AND rfc1459-style private, though the last isn't documented.
         cmodes = {
             # TS6 generic modes:
             'op': 'o', 'halfop': 'h', 'voice': 'v', 'ban': 'b', 'key': 'k',
             'limit': 'l', 'moderated': 'm', 'noextmsg': 'n',
-            'secret': 's', 'topiclock': 't',
+            'secret': 's', 'topiclock': 't', 'private': 'p',
             # hybrid-specific modes:
             'blockcolor': 'c', 'inviteonly': 'i', 'noctcp': 'C',
             'regmoderated': 'M', 'operonly': 'O', 'regonly': 'R',
             'sslonly': 'S', 'banexception': 'e', 'noknock': 'p',
-            'registered': 'r', 'invex': 'I',
+            'registered': 'r', 'invex': 'I', 'paranoia': 'p',
+            'banexception': 'e',
             # Now, map all the ABCD type modes:
             '*A': 'beI', '*B': 'k', '*C': 'l', '*D': 'cimnprstCMORS'
         }
 
-        self.irc.cmodes = cmodes
+        self.cmodes = cmodes
 
         umodes = {
             'oper': 'o', 'invisible': 'i', 'wallops': 'w', 'locops': 'l',
@@ -47,20 +52,21 @@ class HybridProtocol(TS6Protocol):
             'callerid': 'g', 'admin': 'a', 'deaf_commonchan': 'G', 'hideoper': 'H',
             'webirc': 'W', 'sno_clientconnections': 'c', 'sno_badclientconnections': 'u',
             'sno_rejectedclients': 'j', 'sno_skill': 'k', 'sno_fullauthblock': 'f',
-            'sno_remoteclientconnections': 'F', 'sno_admin_requests': 'y', 'sno_debug': 'd',
+            'sno_remoteclientconnections': 'F', 'sno_stats': 'y', 'sno_debug': 'd',
             'sno_nickchange': 'n', 'hideidle': 'q', 'registered': 'r',
-            'snomask': 's', 'ssl': 'S', 'sno_server_connects': 'e', 'sno_botfloods': 'b',
+            'snomask': 's', 'ssl': 'S', 'sno_serverconnects': 'e', 'sno_botfloods': 'b',
             # Now, map all the ABCD type modes:
             '*A': '', '*B': '', '*C': '', '*D': 'DFGHRSWabcdefgijklnopqrsuwxy'
         }
 
-        self.irc.umodes = umodes
+        self.umodes = umodes
+        self.extbans_matching.clear()
 
         # halfops is mandatory on Hybrid
-        self.irc.prefixmodes = {'o': '@', 'h': '%', 'v': '+'}
+        self.prefixmodes = {'o': '@', 'h': '%', 'v': '+'}
 
         # https://github.com/grawity/irc-docs/blob/master/server/ts6.txt#L55
-        f('PASS %s TS 6 %s' % (self.irc.serverdata["sendpass"], self.irc.sid))
+        f('PASS %s TS 6 %s' % (self.serverdata["sendpass"], self.sid))
 
         # We request the following capabilities (for hybrid):
 
@@ -70,7 +76,7 @@ class HybridProtocol(TS6Protocol):
         # CHW: Allow sending messages to @#channel and the like.
         # KNOCK: Support for /knock
         # SVS: Deal with extended NICK/UID messages that contain service IDs/stamps
-        # TBURST: Topic Burst command; we send this in topicBurst
+        # TBURST: Topic Burst command; we send this in topic_burst
         # DLN: DLINE command
         # UNDLN: UNDLINE command
         # KLN: KLINE command
@@ -81,13 +87,13 @@ class HybridProtocol(TS6Protocol):
         # EOB: Supports EOB (end of burst) command
         f('CAPAB :TBURST DLN KNOCK UNDLN UNKLN KLN ENCAP IE EX HOPS CHW SVS CLUSTER EOB QS')
 
-        f('SERVER %s 0 :%s' % (self.irc.serverdata["hostname"],
+        f('SERVER %s 0 :%s' % (self.serverdata["hostname"],
-                               self.irc.serverdata.get('serverdesc') or self.irc.botdata['serverdesc']))
+                               self.serverdata.get('serverdesc') or conf.conf['pylink']['serverdesc']))
 
         # send endburst now
-        self.irc.send(':%s EOB' % (self.irc.sid,))
+        self.send(':%s EOB' % (self.sid,))
 
-    def spawnClient(self, nick, ident='null', host='null', realhost=None, modes=set(),
+    def spawn_client(self, nick, ident='null', host='null', realhost=None, modes=set(),
             server=None, ip='0.0.0.0', realname=None, ts=None, opertype=None,
             manipulatable=False):
         """
@@ -97,58 +103,82 @@ class HybridProtocol(TS6Protocol):
         up to plugins to make sure they don't introduce anything invalid.
         """
 
-        server = server or self.irc.sid
+        server = server or self.sid
-        if not self.irc.isInternalServer(server):
+        if not self.is_internal_server(server):
             raise ValueError('Server %r is not a PyLink server!' % server)
 
         uid = self.uidgen[server].next_uid()
 
         ts = ts or int(time.time())
-        realname = realname or self.irc.botdata['realname']
+        realname = realname or conf.conf['pylink']['realname']
         realhost = realhost or host
-        raw_modes = self.irc.joinModes(modes)
+        raw_modes = self.join_modes(modes)
-        u = self.irc.users[uid] = IrcUser(nick, ts, uid, ident=ident, host=host, realname=realname,
+
+        u = self.users[uid] = User(self, nick, ts, uid, server, ident=ident, host=host, realname=realname,
             realhost=realhost, ip=ip, manipulatable=manipulatable)
-        self.irc.applyModes(uid, modes)
+
-        self.irc.servers[server].users.add(uid)
+        self.apply_modes(uid, modes)
-        self._send(server, "UID {nick} 1 {ts} {modes} {ident} {host} {ip} {uid} "
+        self.servers[server].users.add(uid)
+
+        self._send_with_prefix(server, "UID {nick} {hopcount} {ts} {modes} {ident} {host} {ip} {uid} "
                 "* :{realname}".format(ts=ts, host=host,
                 nick=nick, ident=ident, uid=uid,
-                modes=raw_modes, ip=ip, realname=realname))
+                modes=raw_modes, ip=ip, realname=realname,
+                hopcount=self.servers[server].hopcount))
         return u
 
-    def updateClient(self, target, field, text):
+    def update_client(self, target, field, text):
         """Updates the ident, host, or realname of a PyLink client."""
         # https://github.com/ircd-hybrid/ircd-hybrid/blob/58323b8/modules/m_svsmode.c#L40-L103
         # parv[0] = command
-        # parv[1] = nickname <-- UID works too -GLolol
+        # parv[1] = nickname <-- UID works too -jlu5
-        # parv[2] = TS <-- Of the user, not the current time. -GLolol
+        # parv[2] = TS <-- Of the user, not the current time. -jlu5
         # parv[3] = mode
         # parv[4] = optional argument (services account, vhost)
         field = field.upper()
 
-        ts = self.irc.users[target].ts
+        ts = self.users[target].ts
 
         if field == 'HOST':
-            self.irc.users[target].host = text
+            self.users[target].host = text
             # On Hybrid, it appears that host changing is actually just forcing umode
-            # "+x <hostname>" on the target. -GLolol
+            # "+x <hostname>" on the target. -jlu5
-            self._send(self.irc.sid, 'SVSMODE %s %s +x %s' % (target, ts, text))
+            self._send_with_prefix(self.sid, 'SVSMODE %s %s +x %s' % (target, ts, text))
         else:
             raise NotImplementedError("Changing field %r of a client is unsupported by this protocol." % field)
 
-    def topicBurst(self, numeric, target, text):
+    def oper_notice(self, source, text):
+        """
+        Send a message to all opers.
+        """
+        self._send_with_prefix(source, 'GLOBOPS :%s' % text)
+
+    def set_server_ban(self, source, duration, user='*', host='*', reason='User banned'):
+        """
+        Sets a server ban.
+        """
+        # source: user
+        # parameters: target server mask, duration, user mask, host mask, reason
+        assert not (user == host == '*'), "Refusing to set ridiculous ban on *@*"
+
+        if not source in self.users:
+            log.debug('(%s) Forcing KLINE sender to %s as TS6 does not allow KLINEs from servers', self.name, self.pseudoclient.uid)
+            source = self.pseudoclient.uid
+
+        self._send_with_prefix(source, 'KLINE * %s %s %s :%s' % (duration, user, host, reason))
+
+    def topic_burst(self, numeric, target, text):
         """Sends a topic change from a PyLink server. This is usually used on burst."""
         # <- :0UY TBURST 1459308205 #testchan 1459309379 dan!~d@localhost :sdf
-        if not self.irc.isInternalServer(numeric):
+        if not self.is_internal_server(numeric):
             raise LookupError('No such PyLink server exists.')
 
-        ts = self.irc.channels[target].ts
+        ts = self._channels[target].ts
-        servername = self.irc.servers[numeric].name
+        servername = self.servers[numeric].name
 
-        self._send(numeric, 'TBURST %s %s %s %s :%s' % (ts, target, int(time.time()), servername, text))
+        self._send_with_prefix(numeric, 'TBURST %s %s %s %s :%s' % (ts, target, int(time.time()), servername, text))
-        self.irc.channels[target].topic = text
+        self._channels[target].topic = text
-        self.irc.channels[target].topicset = True
+        self._channels[target].topicset = True
 
     # command handlers
 
@@ -156,14 +186,11 @@ class HybridProtocol(TS6Protocol):
         # We only get a list of keywords here. Hybrid obviously assumes that
         # we know what modes it supports (indeed, this is a standard list).
         # <- CAPAB :UNDLN UNKLN KLN TBURST KNOCK ENCAP DLN IE EX HOPS CHW SVS CLUSTER EOB QS
-        self.irc.caps = caps = args[0].split()
+        self._caps = caps = args[0].split()
-        for required_cap in ('EX', 'IE', 'SVS', 'EOB', 'HOPS', 'QS', 'TBURST', 'SVS'):
+        for required_cap in ('SVS', 'EOB', 'HOPS', 'QS', 'TBURST'):
              if required_cap not in caps:
                  raise ProtocolError('%s not found in TS6 capabilities list; this is required! (got %r)' % (required_cap, caps))
 
-        log.debug('(%s) self.irc.connected set!', self.irc.name)
-        self.irc.connected.set()
-
     def handle_uid(self, numeric, command, args):
         """
         Handles Hybrid-style UID commands (user introduction). This is INCOMPATIBLE
@@ -171,47 +198,55 @@ class HybridProtocol(TS6Protocol):
         """
         # <- :0UY UID dan 1 1451041551 +Facdeiklosuw ~ident localhost 127.0.0.1 0UYAAAAAB * :realname
         nick = args[0]
+        self._check_nick_collision(nick)
         ts, modes, ident, host, ip, uid, account, realname = args[2:10]
+        ts = int(ts)
         if account == '*':
             account = None
         log.debug('(%s) handle_uid: got args nick=%s ts=%s uid=%s ident=%s '
-                  'host=%s realname=%s ip=%s', self.irc.name, nick, ts, uid,
+                  'host=%s realname=%s ip=%s', self.name, nick, ts, uid,
                   ident, host, realname, ip)
 
-        self.irc.users[uid] = IrcUser(nick, ts, uid, ident, host, realname, host, ip)
+        self.users[uid] = User(self, nick, ts, uid, numeric, ident, host, realname, host, ip)
 
-        parsedmodes = self.irc.parseModes(uid, [modes])
+        parsedmodes = self.parse_modes(uid, [modes])
-        log.debug('(%s) handle_uid: Applying modes %s for %s', self.irc.name, parsedmodes, uid)
+        log.debug('(%s) handle_uid: Applying modes %s for %s', self.name, parsedmodes, uid)
-        self.irc.applyModes(uid, parsedmodes)
+        self.apply_modes(uid, parsedmodes)
-        self.irc.servers[numeric].users.add(uid)
+        self.servers[numeric].users.add(uid)
 
         # Call the OPERED UP hook if +o is being added to the mode list.
-        if ('+o', None) in parsedmodes:
+        self._check_oper_status_change(uid, parsedmodes)
-            self.irc.callHooks([uid, 'CLIENT_OPERED', {'text': 'IRC_Operator'}])
+
+        # Track SSL/TLS status
+        has_ssl = self.users[uid].ssl = ('+S', None) in parsedmodes
 
         # Set the account name if present
         if account:
-            self.irc.callHooks([uid, 'CLIENT_SERVICES_LOGIN', {'text': account}])
+            self.call_hooks([uid, 'CLIENT_SERVICES_LOGIN', {'text': account}])
 
-        return {'uid': uid, 'ts': ts, 'nick': nick, 'realname': realname, 'host': host, 'ident': ident, 'ip': ip}
+        return {'uid': uid, 'ts': ts, 'nick': nick, 'realname': realname, 'host': host, 'ident': ident, 'ip': ip, 'secure': has_ssl}
 
     def handle_tburst(self, numeric, command, args):
         """Handles incoming topic burst (TBURST) commands."""
         # <- :0UY TBURST 1459308205 #testchan 1459309379 dan!~d@localhost :sdf
-        channel = self.irc.toLower(args[1])
+        channel = args[1]
         ts = args[2]
         setter = args[3]
         topic = args[-1]
-        self.irc.channels[channel].topic = topic
+        self._channels[channel].topic = topic
-        self.irc.channels[channel].topicset = True
+        self._channels[channel].topicset = True
         return {'channel': channel, 'setter': setter, 'ts': ts, 'text': topic}
 
     def handle_eob(self, numeric, command, args):
-        log.debug('(%s) end of burst received', self.irc.name)
+        """EOB (end-of-burst) handler."""
-        if not self.has_eob:  # Only call ENDBURST hooks if we haven't already.
+        log.debug('(%s) end of burst received from %s', self.name, numeric)
-            return {}
+        if not self.servers[numeric].has_eob:
+            # Don't fight with TS6's generic PING-as-EOB
+            self.servers[numeric].has_eob = True
 
-        self.has_eob = True
+            if numeric == self.uplink:
+                self.connected.set()
+            return {}
 
     def handle_svsmode(self, numeric, command, args):
         """
@@ -222,14 +257,14 @@ class HybridProtocol(TS6Protocol):
         target = args[0]
         ts = args[1]
         modes = args[2:]
-        parsedmodes = self.irc.parseModes(target, modes)
+        parsedmodes = self.parse_modes(target, modes)
 
         for modepair in parsedmodes:
             if modepair[0] == '+d':
                 # Login sequence (tested with Anope 2.0.4-git):
                 # A mode change +d accountname is used to propagate logins,
                 # before setting umode +r on the target.
-                # <- :5ANAAAAAG SVSMODE 5HYAAAAAA 1460175209 +d GL
+                # <- :5ANAAAAAG SVSMODE 5HYAAAAAA 1460175209 +d jlu5
                 # <- :5ANAAAAAG SVSMODE 5HYAAAAAA 1460175209 +r
 
                 # Logout sequence:
@@ -242,7 +277,7 @@ class HybridProtocol(TS6Protocol):
 
                 # Send the login hook, and remove this mode from the mode
                 # list, as it shouldn't be parsed literally.
-                self.irc.callHooks([target, 'CLIENT_SERVICES_LOGIN', {'text': account}])
+                self.call_hooks([target, 'CLIENT_SERVICES_LOGIN', {'text': account}])
                 parsedmodes.remove(modepair)
 
             elif modepair[0] == '+x':
@@ -251,16 +286,16 @@ class HybridProtocol(TS6Protocol):
                 # to some.host, for example.
                 host = args[-1]
 
-                self.irc.users[target].host = host
+                self.users[target].host = host
 
                 # Propagate the hostmask change as a hook.
-                self.irc.callHooks([numeric, 'CHGHOST',
+                self.call_hooks([numeric, 'CHGHOST',
-                                   {'target': target, 'newhost': host}])
+                                 {'target': target, 'newhost': host}])
 
                 parsedmodes.remove(modepair)
 
         if parsedmodes:
-            self.irc.applyModes(target, parsedmodes)
+            self.apply_modes(target, parsedmodes)
 
         return {'target': target, 'modes': parsedmodes}
 

protocols/ircs2s_common.py

@@ -2,63 +2,710 @@
 ircs2s_common.py: Common base protocol class with functions shared by TS6 and P10-based protocols.
 """
 
-from pylinkirc.classes import Protocol
+import re
+import time
+
+from pylinkirc import conf
+from pylinkirc.classes import IRCNetwork, ProtocolError
 from pylinkirc.log import log
 
-class IRCS2SProtocol(Protocol):
+__all__ = ['UIDGenerator', 'IRCCommonProtocol', 'IRCS2SProtocol']
 
-    def handle_kill(self, source, command, args):
+class UIDGenerator():
-        """Handles incoming KILLs."""
+    """
-        killed = args[0]
+    Generate UIDs for IRC S2S.
-        # Depending on whether the IRCd sends explicit QUIT messages for
+    """
-        # killed clients, the user may or may not have automatically been
-        # removed from our user list.
-        # If not, we have to assume that KILL = QUIT and remove them
-        # ourselves.
-        data = self.irc.users.get(killed)
-        if data:
-            self.removeClient(killed)
 
-        # TS6-style kills look something like this:
+    def __init__(self, uidchars, length, sid):
-        # <- :GL KILL 38QAAAAAA :hidden-1C620195!GL (test)
+        self.uidchars = uidchars  # corpus of characters to choose from
-        # What we actually want is to format a pretty kill message, in the form
+        self.length = length  # desired length of uid part, padded with uidchars[0]
-        # "Killed (killername (reason))".
+        self.sid = str(sid)  #  server id (prefixed to every result)
+        self.counter = 0
 
-        try:
+    def next_uid(self):
-            # Get the nick or server name of the caller.
+        """
-            killer = self.irc.getFriendlyName(source)
+        Returns the next unused UID for the server.
-        except KeyError:
+        """
-            # Killer was... neither? We must have aliens or something. Fallback
+        uid = ''
-            # to the given "UID".
+        num = self.counter
-            killer = source
+        if num >= (len(self.uidchars) ** self.length):
+            raise RuntimeError("UID overflowed")
+        while num > 0:
+            num, index = divmod(num, len(self.uidchars))
+            uid = self.uidchars[index] + uid
 
-        # Get the reason, which is enclosed in brackets.
+        self.counter += 1
-        reason = ' '.join(args[1].split(" ")[1:])
+        uid = uid.rjust(self.length, self.uidchars[0])
+        return self.sid + uid
 
-        killmsg = "Killed (%s %s)" % (killer, reason)
+class IRCCommonProtocol(IRCNetwork):
 
-        return {'target': killed, 'text': killmsg, 'userdata': data}
+    COMMON_PREFIXMODES = [('h', 'halfop'), ('a', 'admin'), ('q', 'owner'), ('y', 'owner')]
 
-    def handle_squit(self, numeric, command, args):
+    def __init__(self, *args, **kwargs):
-        """Handles incoming SQUITs."""
+        super().__init__(*args, **kwargs)
-        return self._squit(numeric, command, args)
 
-    def handle_away(self, numeric, command, args):
+        self._caps = {}
+        self._use_builtin_005_handling = False  # Disabled by default for greater security
+        self.protocol_caps |= {'has-irc-modes', 'can-manage-bot-channels'}
+
+    def post_connect(self):
+        self._caps.clear()
+
+    def validate_server_conf(self):
+        """Validates that the server block given contains the required keys."""
+        for k in self.conf_keys:
+            log.debug('(%s) Checking presence of conf key %r', self.name, k)
+            conf.validate(k in self.serverdata,
+                     "Missing option %r in server block for network %s."
+                     % (k, self.name))
+
+        port = self.serverdata['port']
+        conf.validate(isinstance(port, int) and 0 < port < 65535,
+                 "Invalid port %r for network %s"
+                 % (port, self.name))
+
+    @staticmethod
+    def parse_args(args):
+        """
+        Parses a string or list of of RFC1459-style arguments, where ":" may
+        be used for multi-word arguments that last until the end of a line.
+        """
+        if isinstance(args, str):
+            args = args.split(' ')
+
+        real_args = []
+        for idx, arg in enumerate(args):
+            if arg.startswith(':') and idx != 0:
+                # ":" is used to begin multi-word arguments that last until the end of the message.
+                # Use list splicing here to join them into one argument, and then add it to our list of args.
+                joined_arg = ' '.join(args[idx:])[1:]  # Cut off the leading : as well
+                real_args.append(joined_arg)
+                break
+            elif arg.strip(' '):  # Skip empty args that aren't part of the multi-word arg
+                real_args.append(arg)
+
+        return real_args
+
+    @classmethod
+    def parse_prefixed_args(cls, args):
+        """Similar to parse_args(), but stripping leading colons from the first argument
+        of a line (usually the sender field)."""
+        args = cls.parse_args(args)
+        args[0] = args[0].split(':', 1)[1]
+        return args
+
+    @staticmethod
+    def parse_isupport(args, fallback=''):
+        """
+        Parses a string of capabilities in the 005 / RPL_ISUPPORT format.
+        """
+
+        if isinstance(args, str):
+            args = args.split(' ')
+
+        caps = {}
+        for cap in args:
+            try:
+                # Try to split it as a KEY=VALUE pair.
+                key, value = cap.split('=', 1)
+            except ValueError:
+                key = cap
+                value = fallback
+            caps[key] = value
+
+        return caps
+
+    @staticmethod
+    def parse_isupport_prefixes(args):
+        """
+        Separates prefixes field like "(qaohv)~&@%+" into a dict mapping mode characters to mode
+        prefixes.
+        """
+        prefixsearch = re.search(r'\(([A-Za-z]+)\)(.*)', args)
+        return dict(zip(prefixsearch.group(1), prefixsearch.group(2)))
+
+    @classmethod
+    def parse_message_tags(cls, data):
+        """
+        Parses IRCv3.2 message tags from a message, as described at http://ircv3.net/specs/core/message-tags-3.2.html
+
+        data is a list of command arguments, split by spaces.
+        """
+        # Example query:
+        # @aaa=bbb;ccc;example.com/ddd=eee :nick!ident@host.com PRIVMSG me :Hello
+        if data[0].startswith('@'):
+            tagdata = data[0].lstrip('@').split(';')
+            for idx, tag in enumerate(tagdata):
+                tag = tag.replace('\\s', ' ')
+                tag = tag.replace('\\r', '\r')
+                tag = tag.replace('\\n', '\n')
+                tag = tag.replace('\\:', ';')
+
+                # We want to drop lone \'s but keep \\ as \ ...
+                tag = tag.replace('\\\\', '\x00')
+                tag = tag.replace('\\', '')
+                tag = tag.replace('\x00', '\\')
+                tagdata[idx] = tag
+
+            results = cls.parse_isupport(tagdata, fallback='')
+            return results
+        return {}
+
+    def handle_away(self, source, command, args):
         """Handles incoming AWAY messages."""
         # TS6:
         # <- :6ELAAAAAB AWAY :Auto-away
+        # <- :6ELAAAAAB AWAY
         # P10:
         # <- ABAAA A :blah
         # <- ABAAA A
+        if source not in self.users:
+            return
+
         try:
-            self.irc.users[numeric].away = text = args[0]
+            self.users[source].away = text = args[0]
         except IndexError:  # User is unsetting away status
-            self.irc.users[numeric].away = text = ''
+            self.users[source].away = text = ''
         return {'text': text}
 
-    def handle_version(self, numeric, command, args):
+    def handle_error(self, numeric, command, args):
-        """Handles requests for the PyLink server version."""
+        """Handles ERROR messages - these mean that our uplink has disconnected us!"""
-        return {}  # See coremods/handlers.py for how this hook is used
+        raise ProtocolError('Received an ERROR, disconnecting!')
+
+    def handle_pong(self, source, command, args):
+        """Handles incoming PONG commands."""
+        if source == self.uplink:
+            self.lastping = time.time()
+
+    def handle_005(self, source, command, args):
+        """
+        Handles 005 / RPL_ISUPPORT. This is used by at least Clientbot and ngIRCd (for server negotiation).
+        """
+        # ngIRCd:
+        # <- :ngircd.midnight.local 005 pylink-devel.int NETWORK=ngircd-test :is my network name
+        # <- :ngircd.midnight.local 005 pylink-devel.int RFC2812 IRCD=ngIRCd CHARSET=UTF-8 CASEMAPPING=ascii PREFIX=(qaohv)~&@%+ CHANTYPES=#&+ CHANMODES=beI,k,l,imMnOPQRstVz CHANLIMIT=#&+:10 :are supported on this server
+        # <- :ngircd.midnight.local 005 pylink-devel.int CHANNELLEN=50 NICKLEN=21 TOPICLEN=490 AWAYLEN=127 KICKLEN=400 MODES=5 MAXLIST=beI:50 EXCEPTS=e INVEX=I PENALTY :are supported on this server
+
+        # Regular clientbot, connecting to InspIRCd:
+        # <- :millennium.overdrivenetworks.com 005 ice AWAYLEN=200 CALLERID=g CASEMAPPING=rfc1459 CHANMODES=IXbegw,k,FJLfjl,ACKMNOPQRSTUcimnprstz CHANNELLEN=64 CHANTYPES=# CHARSET=ascii ELIST=MU ESILENCE EXCEPTS=e EXTBAN=,ACNOQRSTUcmprsuz FNC INVEX=I :are supported by this server
+        # <- :millennium.overdrivenetworks.com 005 ice KICKLEN=255 MAP MAXBANS=60 MAXCHANNELS=30 MAXPARA=32 MAXTARGETS=20 MODES=20 NAMESX NETWORK=OVERdrive-IRC NICKLEN=21 OVERRIDE PREFIX=(Yqaohv)*~&@%+ SILENCE=32 :are supported by this server
+        # <- :millennium.overdrivenetworks.com 005 ice SSL=[::]:6697 STARTTLS STATUSMSG=*~&@%+ TOPICLEN=307 UHNAMES USERIP VBANLIST WALLCHOPS WALLVOICES WATCH=32 :are supported by this server
+
+        if not self._use_builtin_005_handling:
+            log.warning("(%s) Got spurious 005 message from %s: %r", self.name, source, args)
+            return
+
+        newcaps = self.parse_isupport(args[1:-1])
+        self._caps.update(newcaps)
+        log.debug('(%s) handle_005: self._caps is %s', self.name, self._caps)
+
+        if 'CHANMODES' in newcaps:
+            self.cmodes['*A'], self.cmodes['*B'], self.cmodes['*C'], self.cmodes['*D'] = \
+                newcaps['CHANMODES'].split(',')
+        log.debug('(%s) handle_005: cmodes: %s', self.name, self.cmodes)
+
+        if 'USERMODES' in newcaps:
+            self.umodes['*A'], self.umodes['*B'], self.umodes['*C'], self.umodes['*D'] = \
+                newcaps['USERMODES'].split(',')
+        log.debug('(%s) handle_005: umodes: %s', self.name, self.umodes)
+
+        if 'CASEMAPPING' in newcaps:
+            self.casemapping = newcaps.get('CASEMAPPING', self.casemapping)
+            log.debug('(%s) handle_005: casemapping set to %s', self.name, self.casemapping)
+
+        if 'PREFIX' in newcaps:
+            self.prefixmodes = prefixmodes = self.parse_isupport_prefixes(newcaps['PREFIX'])
+            log.debug('(%s) handle_005: prefix modes set to %s', self.name, self.prefixmodes)
+
+            # Autodetect common prefix mode names.
+            for char, modename in self.COMMON_PREFIXMODES:
+                # Don't overwrite existing named mode definitions.
+                if char in self.prefixmodes and modename not in self.cmodes:
+                    self.cmodes[modename] = char
+                    log.debug('(%s) handle_005: autodetecting mode %s (%s) as %s', self.name,
+                              char, self.prefixmodes[char], modename)
+
+        # https://defs.ircdocs.horse/defs/isupport.html
+        if 'EXCEPTS' in newcaps:
+            # Handle EXCEPTS=e or EXCEPTS fields
+            self.cmodes['banexception'] = newcaps.get('EXCEPTS') or 'e'
+            log.debug('(%s) handle_005: got cmode banexception=%r', self.name, self.cmodes['banexception'])
+
+        if 'INVEX' in newcaps:
+            # Handle INVEX=I, INVEX fields
+            self.cmodes['invex'] = newcaps.get('INVEX') or 'I'
+            log.debug('(%s) handle_005: got cmode invex=%r', self.name, self.cmodes['invex'])
+
+        if 'NICKLEN' in newcaps:
+            # Handle NICKLEN=number
+            assert newcaps['NICKLEN'], "Got NICKLEN tag with no content?"
+            self.maxnicklen = int(newcaps['NICKLEN'])
+            log.debug('(%s) handle_005: got %r for maxnicklen', self.name, self.maxnicklen)
+
+        if 'DEAF' in newcaps:
+            # Handle DEAF=D, DEAF fields
+            self.umodes['deaf'] = newcaps.get('DEAF') or 'D'
+            log.debug('(%s) handle_005: got umode deaf=%r', self.name, self.umodes['deaf'])
+
+        if 'CALLERID' in newcaps:
+            # Handle CALLERID=g, CALLERID fields
+            self.umodes['callerid'] = newcaps.get('CALLERID') or 'g'
+            log.debug('(%s) handle_005: got umode callerid=%r', self.name, self.umodes['callerid'])
+
+        if 'STATUSMSG' in newcaps:
+            # Note: This assumes that all available prefixes can be used in STATUSMSG too.
+            # Even though this isn't always true, I don't see the point in making things
+            # any more complicated.
+            self.protocol_caps |= {'has-statusmsg'}
+
+    def _send_with_prefix(self, source, msg, **kwargs):
+        """Sends a RFC1459-style raw command from the given sender."""
+        self.send(':%s %s' % (self._expandPUID(source), msg), **kwargs)
+
+class IRCS2SProtocol(IRCCommonProtocol):
+    COMMAND_TOKENS = {}
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.protocol_caps |= {'can-spawn-clients', 'has-ts', 'can-host-relay',
+                               'can-track-servers'}
+
+        # Alias
+        self.handle_squit = self._squit
+
+    def handle_events(self, data):
+        """Event handler for RFC1459-like protocols.
+
+        This passes most commands to the various handle_ABCD() functions
+        elsewhere defined protocol modules, coersing various sender prefixes
+        from nicks and server names to UIDs and SIDs respectively,
+        whenever possible.
+
+        Commands sent without an explicit sender prefix will have them set to
+        the SID of the uplink server.
+        """
+        data = data.split(" ")
+
+        tags = self.parse_message_tags(data)
+        if tags:
+            # If we have message tags, split off the first argument.
+            data = data[1:]
+
+        args = self.parse_args(data)
+
+        sender = args[0]
+        if sender.startswith(':'):
+            sender = sender[1:]
+
+            # If the sender isn't in numeric format, try to convert it automatically.
+            sender_sid = self._get_SID(sender)
+            sender_uid = self._get_UID(sender)
+
+            if sender_sid in self.servers:
+                sender = sender_sid
+            elif sender_uid in self.users:
+                sender = sender_uid
+        else:
+            # No sender prefix; treat as coming from uplink IRCd.
+            sender = self.uplink
+            args.insert(0, sender)
+
+        raw_command = args[1].upper()
+        args = args[2:]
+
+        log.debug('(%s) Found message sender as %s, raw_command=%r, args=%r', self.name, sender, raw_command, args)
+
+        # For P10, convert the command token into a regular command, if present.
+        command = self.COMMAND_TOKENS.get(raw_command, raw_command)
+        if command != raw_command:
+            log.debug('(%s) Translating token %s to command %s', self.name, raw_command, command)
+
+        if self.is_internal_client(sender) or self.is_internal_server(sender):
+            log.warning("(%s) Received command %s being routed the wrong way!", self.name, command)
+            return
+
+        if command == 'ENCAP':
+            # Special case for TS6 encapsulated commands (ENCAP), in forms like this:
+            # <- :00A ENCAP * SU 42XAAAAAC :jlu5
+            command = args[1]
+            args = args[2:]
+            log.debug("(%s) Rewriting incoming ENCAP to command %s (args: %s)", self.name, command, args)
+
+        try:
+            func = getattr(self, 'handle_'+command.lower())
+        except AttributeError:  # Unhandled command
+            pass
+        else:
+            parsed_args = func(sender, command, args)
+            if parsed_args is not None:
+                if tags:
+                    parsed_args['tags'] = tags  # Add message tags to this hook payload.
+                return [sender, command, parsed_args]
+
+    def invite(self, source, target, channel):
+        """Sends an INVITE from a PyLink client."""
+        if not self.is_internal_client(source):
+            raise LookupError('No such PyLink client exists.')
+
+        self._send_with_prefix(source, 'INVITE %s %s' % (self._expandPUID(target), channel))
+
+    def kick(self, numeric, channel, target, reason=None):
+        """Sends kicks from a PyLink client/server."""
+
+        if (not self.is_internal_client(numeric)) and \
+                (not self.is_internal_server(numeric)):
+            raise LookupError('No such PyLink client/server exists.')
+
+        if not reason:
+            reason = 'No reason given'
+
+        # Mangle kick targets for IRCds that require it.
+        real_target = self._expandPUID(target)
+
+        self._send_with_prefix(numeric, 'KICK %s %s :%s' % (channel, real_target, reason))
+
+        # We can pretend the target left by its own will; all we really care about
+        # is that the target gets removed from the channel userlist, and calling
+        # handle_part() does that just fine.
+        self.handle_part(target, 'KICK', [channel])
+
+    def oper_notice(self, source, text):
+        """
+        Send a message to all opers.
+        """
+        self._send_with_prefix(source, 'WALLOPS :%s' % text)
+
+    def numeric(self, source, numeric, target, text):
+        """Sends raw numerics from a server to a remote client. This is used for WHOIS replies."""
+        # Mangle the target for IRCds that require it.
+        target = self._expandPUID(target)
+
+        self._send_with_prefix(source, '%s %s %s' % (numeric, target, text))
+
+    def part(self, client, channel, reason=None):
+        """Sends a part from a PyLink client."""
+        if not self.is_internal_client(client):
+            log.error('(%s) Error trying to part %r from %r (no such client exists)', self.name, client, channel)
+            raise LookupError('No such PyLink client exists.')
+        msg = "PART %s" % channel
+        if reason:
+            msg += " :%s" % reason
+        self._send_with_prefix(client, msg)
+        self.handle_part(client, 'PART', [channel])
+
+    def _ping_uplink(self):
+        """Sends a PING to the uplink.
+
+        This is mostly used by PyLink internals to check whether the remote link is up."""
+        if self.sid and self.connected.is_set():
+            self._send_with_prefix(self.sid, 'PING %s' % self._expandPUID(self.uplink))
+
+    def quit(self, numeric, reason):
+        """Quits a PyLink client."""
+        if self.is_internal_client(numeric):
+            self._send_with_prefix(numeric, "QUIT :%s" % reason)
+            self._remove_client(numeric)
+        else:
+            raise LookupError("No such PyLink client exists.")
+
+    def message(self, numeric, target, text):
+        """Sends a PRIVMSG from a PyLink client."""
+        if not self.is_internal_client(numeric):
+            raise LookupError('No such PyLink client exists.')
+
+        # Mangle message targets for IRCds that require it.
+        target = self._expandPUID(target)
+
+        self._send_with_prefix(numeric, 'PRIVMSG %s :%s' % (target, text))
+
+    def notice(self, numeric, target, text):
+        """Sends a NOTICE from a PyLink client or server."""
+        if (not self.is_internal_client(numeric)) and \
+                (not self.is_internal_server(numeric)):
+            raise LookupError('No such PyLink client/server exists.')
+
+        # Mangle message targets for IRCds that require it.
+        target = self._expandPUID(target)
+
+        self._send_with_prefix(numeric, 'NOTICE %s :%s' % (target, text))
+
+    def squit(self, source, target, text='No reason given'):
+        """SQUITs a PyLink server."""
+        # -> SQUIT 9PZ :blah, blah
+        log.debug('(%s) squit: source=%s, target=%s', self.name, source, target)
+        self._send_with_prefix(source, 'SQUIT %s :%s' % (self._expandPUID(target), text))
+        self.handle_squit(source, 'SQUIT', [target, text])
+
+    def topic(self, source, target, text):
+        """Sends a TOPIC change from a PyLink client or server."""
+        if (not self.is_internal_client(source)) and (not self.is_internal_server(source)):
+            raise LookupError('No such PyLink client/server exists.')
+
+        self._send_with_prefix(source, 'TOPIC %s :%s' % (target, text))
+        self._channels[target].topic = text
+        self._channels[target].topicset = True
+    topic_burst = topic
+
+    def handle_invite(self, numeric, command, args):
+        """Handles incoming INVITEs."""
+        # TS6:
+        #  <- :70MAAAAAC INVITE 0ALAAAAAA #blah 12345
+        # P10:
+        #  <- ABAAA I PyLink-devel #services 1460948992
+        #  Note that the target is a nickname, not a numeric.
+
+        target = self._get_UID(args[0])
+        channel = args[1]
+
+        curtime = int(time.time())
+        try:
+            ts = int(args[2])
+        except IndexError:
+            ts = curtime
+
+        ts = ts or curtime  # Treat 0 timestamps (e.g. inspircd) as the current time.
+
+        return {'target': target, 'channel': channel, 'ts': ts}
+
+    def handle_kick(self, source, command, args):
+        """Handles incoming KICKs."""
+        # :70MAAAAAA KICK #test 70MAAAAAA :some reason
+        channel = args[0]
+        kicked = self._get_UID(args[1])
+
+        try:
+            reason = args[2]
+        except IndexError:
+            reason = ''
+
+        log.debug('(%s) Removing kick target %s from %s', self.name, kicked, channel)
+        self.handle_part(kicked, 'KICK', [channel, reason])
+        return {'channel': channel, 'target': kicked, 'text': reason}
+
+    def handle_kill(self, source, command, args):
+        """Handles incoming KILLs."""
+        killed = self._get_UID(args[0])
+        # Some IRCds send explicit QUIT messages for their killed clients in addition to KILL,
+        # meaning that our target client may have been removed already. If this is the case,
+        # don't bother forwarding this message on.
+        # Generally, we only need to distinguish between KILL and QUIT if the target is
+        # one of our clients, in which case the above statement isn't really applicable.
+        if killed in self.users:
+            userdata = self._remove_client(killed)
+        else:
+            return
+
+        # TS6-style kills look something like this:
+        # <- :jlu5 KILL 38QAAAAAA :hidden-1C620195!jlu5 (test)
+        # What we actually want is to format a pretty kill message, in the form
+        # "Killed (killername (reason))".
+
+        if '!' in args[1].split(" ", 1)[0]:
+            try:
+                # Get the nick or server name of the caller.
+                killer = self.get_friendly_name(source)
+            except KeyError:
+                # Killer was... neither? We must have aliens or something. Fallback
+                # to the given "UID".
+                killer = source
+
+            # Get the reason, which is enclosed in brackets.
+            killmsg = ' '.join(args[1].split(" ")[1:])[1:-1]
+            if not killmsg:
+                log.warning('(%s) Failed to extract kill reason: %r', self.name, args)
+                killmsg = args[1]
+        else:
+            # We already have a preformatted kill, so just pass it on as is.
+            # XXX: this does create a convoluted kill string if we want to forward kills
+            # over relay.
+            # InspIRCd:
+            # <- :1MLAAAAA1 KILL 0ALAAAAAC :Killed (jlu5 (test))
+            # ngIRCd:
+            # <- :jlu5 KILL PyLink-devel :KILLed by jlu5: ?
+            killmsg = args[1]
+
+        return {'target': killed, 'text': killmsg, 'userdata': userdata}
+
+    def _check_cloak_change(self, uid):  # Stub by default
+        return
+
+    def _check_umode_away_change(self, uid):
+        # Handle away status changes based on umode +a
+        awaymode = self.umodes.get('away')
+        if uid in self.users and awaymode:
+            u = self.users[uid]
+            old_away_status = u.away
+
+            # Check whether the user is marked away, and send a hook update only if the status has changed.
+            away_status = (awaymode, None) in u.modes
+            if away_status != bool(old_away_status):
+                # This sets a dummy away reason of "Away" because no actual text is provided.
+                self.call_hooks([uid, 'AWAY', {'text': 'Away' if away_status else ''}])
+
+    def _check_oper_status_change(self, uid, modes):
+        if uid in self.users:
+            u = self.users[uid]
+            if 'servprotect' in self.umodes and (self.umodes['servprotect'], None) in u.modes:
+                opertype = 'Network Service'
+            elif 'netadmin' in self.umodes and (self.umodes['netadmin'], None) in u.modes:
+                opertype = 'Network Administrator'
+            elif 'admin' in self.umodes and (self.umodes['admin'], None) in u.modes:
+                opertype = 'Server Administrator'
+            else:
+                opertype = 'IRC Operator'
+
+            if ('+o', None) in modes:
+                self.call_hooks([uid, 'CLIENT_OPERED', {'text': opertype}])
+
+    def handle_mode(self, source, command, args):
+        """Handles mode changes."""
+        # InspIRCd:
+        # <- :70MAAAAAA MODE 70MAAAAAA -i+xc
+
+        # P10:
+        # <- ABAAA M jlu5 -w
+        # <- ABAAA M #test +v ABAAB 1460747615
+        # <- ABAAA OM #test +h ABAAA
+        target = self._get_UID(args[0])
+        if self.is_channel(target):
+            channeldata = self._channels[target].deepcopy()
+        else:
+            channeldata = None
+
+        modestrings = args[1:]
+        changedmodes = self.parse_modes(target, modestrings)
+        self.apply_modes(target, changedmodes)
+
+        if target in self.users:
+            # Target was a user. Check for any cloak and away status changes.
+            self._check_cloak_change(target)
+            self._check_umode_away_change(target)
+            self._check_oper_status_change(target, changedmodes)
+
+        return {'target': target, 'modes': changedmodes, 'channeldata': channeldata}
+
+    def handle_part(self, source, command, args):
+        """Handles incoming PART commands."""
+        channels = args[0].split(',')
+
+        for channel in channels.copy():
+            if channel not in self._channels or source not in self._channels[channel].users:
+                # Ignore channels the user isn't on, and remove them from any hook payloads.
+                channels.remove(channel)
+
+            self._channels[channel].remove_user(source)
+            try:
+                self.users[source].channels.discard(channel)
+            except KeyError:
+                log.debug("(%s) handle_part: KeyError trying to remove %r from %r's channel list?", self.name, channel, source)
+
+            try:
+                reason = args[1]
+            except IndexError:
+                reason = ''
+
+        if channels:
+            return {'channels': channels, 'text': reason}
+
+    def handle_privmsg(self, source, command, args):
+        """Handles incoming PRIVMSG/NOTICE."""
+        # TS6:
+        # <- :70MAAAAAA PRIVMSG #dev :afasfsa
+        # <- :70MAAAAAA NOTICE 0ALAAAAAA :afasfsa
+        # P10:
+        # <- ABAAA P AyAAA :privmsg text
+        # <- ABAAA O AyAAA :notice text
+        raw_target = args[0]
+        server_check = None
+        if '@' in raw_target and not self.is_channel(raw_target.lstrip(''.join(self.prefixmodes.values()))):
+            log.debug('(%s) Processing user@server message with target %s',
+                      self.name, raw_target)
+            raw_target, server_check = raw_target.split('@', 1)
+
+            if not self.is_server_name(server_check):
+                log.warning('(%s) Got user@server message with invalid server '
+                            'name %r (full target: %r)', self.name, server_check,
+                            args[0])
+                return
+
+        target = self._get_UID(raw_target)
+
+        if server_check is not None:
+            not_found = False
+            if target not in self.users:
+                # Most IRCds don't check locally if the target nick actually exists.
+                # If it doesn't, send an error back.
+                not_found = True
+            else:
+                # I guess we can technically leave this up to the IRCd to do the right
+                # checks here, but maybe that ruins the point of this 'security feature'
+                # in the first place.
+                log.debug('(%s) Checking if target %s/%s exists on server %s',
+                          self.name, target, raw_target, server_check)
+                sid = self._get_SID(server_check)
+
+                if not sid:
+                    log.debug('(%s) Failed user@server server check: %s does not exist.',
+                              self.name, server_check)
+                    not_found = True
+                elif sid != self.get_server(target):
+                    log.debug("(%s) Got user@server message for %s/%s, but they "
+                              "aren't on the server %s/%s. (full target: %r)",
+                              self.name, target, raw_target, server_check, sid,
+                              args[0])
+                    not_found = True
+
+            if not_found:
+                self.numeric(self.sid, 401, source, '%s :No such nick' %
+                             args[0])
+                return
+
+        # Coerse =#channel from Charybdis op moderated +z to @#channel.
+        if target.startswith('='):
+            target = '@' + target[1:]
+
+        return {'target': target, 'text': args[1]}
+
+    handle_notice = handle_privmsg
+
+    def handle_quit(self, numeric, command, args):
+        """Handles incoming QUIT commands."""
+        # TS6:
+        # <- :1SRAAGB4T QUIT :Quit: quit message goes here
+        # P10:
+        # <- ABAAB Q :Killed (jlu5_ (bangbang))
+        userdata = self._remove_client(numeric)
+        if userdata:
+            try:
+                reason = args[0]
+            except IndexError:
+                reason = ''
+            return {'text': reason, 'userdata': userdata}
+
+    def handle_stats(self, numeric, command, args):
+        """Handles the IRC STATS command."""
+        # IRCds are mostly consistent with this syntax, with the caller being the source,
+        # the stats type as arg 0, and the target server (SID or hostname) as arg 1
+        # <- :42XAAAAAB STATS c :7PY
+        return {'stats_type': args[0], 'target': self._get_SID(args[1])}
+
+    def handle_topic(self, numeric, command, args):
+        """Handles incoming TOPIC changes from clients."""
+        # <- :70MAAAAAA TOPIC #test :test
+        channel = args[0]
+        topic = args[1]
+
+        oldtopic = self._channels[channel].topic
+        self._channels[channel].topic = topic
+        self._channels[channel].topicset = True
+
+        return {'channel': channel, 'setter': numeric, 'text': topic,
+                'oldtopic': oldtopic}
+
+    def handle_time(self, numeric, command, args):
+        """Handles incoming /TIME requests."""
+        return {'target': args[0]}
 
     def handle_whois(self, numeric, command, args):
         """Handles incoming WHOIS commands.."""
@@ -72,17 +719,8 @@ class IRCS2SProtocol(Protocol):
         # WHOIS commands received are for us, since we don't host any real servers
         # to route it to.
 
-        return {'target': self._getUid(args[-1])}
+        return {'target': self._get_UID(args[-1])}
 
-    def handle_quit(self, numeric, command, args):
+    def handle_version(self, numeric, command, args):
-        """Handles incoming QUIT commands."""
+        """Handles requests for the PyLink server version."""
-        # TS6:
+        return {}  # See coremods/handlers.py for how this hook is used
-        # <- :1SRAAGB4T QUIT :Quit: quit message goes here
-        # P10:
-        # <- ABAAB Q :Killed (GL_ (bangbang))
-        self.removeClient(numeric)
-        return {'text': args[0]}
-
-    def handle_time(self, numeric, command, args):
-        """Handles incoming /TIME requests."""
-        return {'target': args[0]}

protocols/ngircd.py

@@ -0,0 +1,569 @@
+"""
+ngircd.py: PyLink protocol module for ngIRCd.
+"""
+##
+# Server protocol docs for ngIRCd can be found at:
+#     https://github.com/ngircd/ngircd/blob/master/doc/Protocol.txt
+# and https://tools.ietf.org/html/rfc2813
+##
+
+import re
+import time
+
+from pylinkirc import __version__, conf, utils
+from pylinkirc.classes import *
+from pylinkirc.log import log
+from pylinkirc.protocols.ircs2s_common import *
+
+__all__ = ['NgIRCdProtocol']
+
+
+class NgIRCdProtocol(IRCS2SProtocol):
+    def __init__(self, irc):
+        super().__init__(irc)
+
+        self.conf_keys -= {'sid', 'sidrange'}
+        self.casemapping = 'ascii'  # This is the default; it's actually set on server negotiation
+        self.hook_map = {'NJOIN': 'JOIN'}
+
+        # Track whether we've received end-of-burst from the uplink.
+        self.has_eob = False
+
+        self._caps = {}
+        self._use_builtin_005_handling = True
+
+        # ngIRCd has no TS tracking.
+        self.protocol_caps.discard('has-ts')
+
+        # Slash in nicks is problematic; while it works for basic things like JOIN and messages,
+        # attempts to set user modes fail.
+        self.protocol_caps |= {'slash-in-hosts', 'underscore-in-hosts'}
+
+    ### Commands
+
+    def post_connect(self):
+        self.send('PASS %s 0210-IRC+ PyLink|%s:CHLMoX' % (self.serverdata['sendpass'], __version__))
+
+        # Note: RFC 2813 mandates another server token value after the hopcount (1), but ngIRCd
+        # doesn't follow that behaviour per https://github.com/ngircd/ngircd/issues/224
+        self.send("SERVER %s 1 :%s" % (self.serverdata['hostname'],
+                                       self.serverdata.get('serverdesc') or conf.conf['pylink']['serverdesc']))
+
+        self._uidgen = PUIDGenerator('PUID')
+
+        # The first "SID" this generator should return is 2, because server token 1 is implied to be
+        # the main PyLink server. RFC2813 has no official definition of SIDs, but rather uses
+        # integer tokens in the SERVER and NICK (user introduction) commands to keep track of which
+        # user exists on which server. Why did they do it this way? Who knows!
+        self._sidgen = PUIDGenerator('PSID', start=1)
+        self.sid = self._sidgen.next_sid(prefix=self.serverdata['hostname'])
+
+        self._caps.clear()
+
+        self.cmodes.update({
+            'banexception': 'e',
+            'invex': 'I',
+            'noinvite': 'V',
+            'nokick': 'Q',
+            'nonick': 'N',
+            'operonly': 'O',
+            'permanent': 'P',
+            'registered': 'r',
+            'regmoderated': 'M',
+            'regonly': 'R',
+            'sslonly': 'z'
+        })
+
+        self.umodes.update({
+            'away': 'a',
+            'bot': 'B',
+            'cloak': 'x',
+            'deaf_commonchan': 'C',
+            'floodexempt': 'F',
+            'hidechans': 'I',
+            'privdeaf': 'b',
+            'registered': 'R',
+            'restricted': 'r',
+            'servprotect': 'q',
+            'sno_clientconnections': 'c'
+        })
+
+    def spawn_client(self, nick, ident='null', host='null', realhost=None, modes=set(),
+            server=None, ip='0.0.0.0', realname=None, ts=None, opertype='IRC Operator',
+            manipulatable=False):
+        """
+        Spawns a new client with the given options.
+
+        Note: No nick collision / valid nickname checks are done here; it is
+        up to plugins to make sure they don't introduce anything invalid.
+
+        Note 2: IP and realhost are ignored because ngIRCd does not send them.
+        """
+        server = server or self.sid
+        assert '@' in server, "Need PSID for spawn_client, not pure server name!"
+        if not self.is_internal_server(server):
+            raise ValueError('Server %r is not a PyLink server!' % server)
+
+        realname = realname or conf.conf['pylink']['realname']
+
+        uid = self._uidgen.next_uid(prefix=nick)
+        userobj = self.users[uid] = User(self, nick, ts or int(time.time()), uid, server,
+                                         ident=ident, host=host, realname=realname,
+                                         manipulatable=manipulatable, opertype=opertype,
+                                         realhost=host)
+
+        self.apply_modes(uid, modes)
+        self.servers[server].users.add(uid)
+
+        # Grab our server token; this is used instead of server name to denote where the client is.
+        server_token = server.rsplit('@')[-1]
+        # <- :ngircd.midnight.local NICK jlu5 1 ~jlu5 localhost 1 +io :realname
+        self._send_with_prefix(server, 'NICK %s %s %s %s %s %s :%s' % (nick, self.servers[server].hopcount,
+                               ident, host, server_token, self.join_modes(modes), realname))
+        return userobj
+
+    def spawn_server(self, name, sid=None, uplink=None, desc=None):
+        """
+        Spawns a server off a PyLink server.
+
+        * desc (server description) defaults to the one in the config.
+        * uplink defaults to the main PyLink server.
+        * SID is set equal to the server name for ngIRCd, as server UIDs are not used.
+        """
+        uplink = uplink or self.sid
+        assert uplink in self.servers, "Unknown uplink %r?" % uplink
+        name = name.lower()
+        sid = self._sidgen.next_sid(prefix=name)
+
+        desc = desc or self.serverdata.get('serverdesc') or conf.conf['pylink']['serverdesc']
+
+        if sid in self.servers:
+            raise ValueError('A server named %r already exists!' % sid)
+
+        if not self.is_internal_server(uplink):
+            raise ValueError('Server %r is not a PyLink server!' % uplink)
+
+        if not self.is_server_name(name):
+            raise ValueError('Invalid server name %r' % name)
+
+        # https://tools.ietf.org/html/rfc2813#section-4.1.2
+        # We need to store a server token to introduce clients on the right server. Since this is just
+        # a number, we can simply use the counter in our PSID generator for this.
+        server_token = sid.rsplit('@')[-1]
+        self.servers[sid] = Server(self, uplink, name, internal=True, desc=desc)
+        self._send_with_prefix(uplink, 'SERVER %s %s %s :%s' % (name, self.servers[sid].hopcount, server_token, desc))
+        return sid
+
+    def away(self, source, text):
+        """Sends an AWAY message from a PyLink client. If the text is empty, away status is unset."""
+        if not self.is_internal_client(source):
+            raise LookupError('No such PyLink client exists.')
+
+        # Away status is denoted on ngIRCd with umode +a.
+        modes = self.users[source].modes
+        if text and (('a', None) not in modes):
+            # Set umode +a if it isn't already set
+            self.mode(source, source, [('+a', None)])
+        elif ('a', None) in modes:
+            # Ditto, only unset the mode if it *was* set.
+            self.mode(source, source, [('-a', None)])
+        self.users[source].away = text
+
+    def join(self, client, channel):
+
+        if not self.is_internal_client(client):
+            raise LookupError('No such PyLink client exists.')
+
+        self._send_with_prefix(client, "JOIN %s" % channel)
+        self._channels[channel].users.add(client)
+        self.users[client].channels.add(channel)
+
+    def kill(self, source, target, reason):
+        """Sends a kill from a PyLink client/server."""
+        if (not self.is_internal_client(source)) and \
+                (not self.is_internal_server(source)):
+            raise LookupError('No such PyLink client/server exists.')
+
+        # Follow ngIRCd's formatting of the kill messages for the most part
+        self._send_with_prefix(source, 'KILL %s :KILLed by %s: %s' % (self._expandPUID(target),
+                               self.get_friendly_name(source), reason))
+
+        # Implicitly remove our own client if one was the target.
+        if self.is_internal_client(target):
+            self._remove_client(target)
+
+    def knock(self, numeric, target, text):
+        raise NotImplementedError('KNOCK is not supported on ngIRCd.')
+
+    def mode(self, source, target, modes, ts=None):
+        """Sends mode changes from a PyLink client/server. The TS option is not used on ngIRCd."""
+
+        if (not self.is_internal_client(source)) and \
+                (not self.is_internal_server(source)):
+            raise LookupError('No such PyLink client/server %r exists' % source)
+
+        self.apply_modes(target, modes)
+        modes = list(modes)  # Work around TypeError in the expand PUID section
+
+        if self.is_channel(target):
+            msgprefix = ':%s MODE %s ' % (self._expandPUID(source), target)
+            bufsize = self.S2S_BUFSIZE - len(msgprefix)
+
+            # Expand PUIDs when sending outgoing prefix modes.
+            for idx, mode in enumerate(modes):
+                if mode[0][-1] in self.prefixmodes:
+                    log.debug('(%s) mode: expanding PUID of mode %s', self.name, str(mode))
+                    modes[idx] = (mode[0], self._expandPUID(mode[1]))
+
+            for modestr in self.wrap_modes(modes, bufsize, max_modes_per_msg=12):
+                self.send(msgprefix + modestr)
+        else:
+            joinedmodes = self.join_modes(modes)
+            self._send_with_prefix(source, 'MODE %s %s' % (self._expandPUID(target), joinedmodes))
+
+    def nick(self, source, newnick):
+        """Changes the nick of a PyLink client."""
+        if not self.is_internal_client(source):
+            raise LookupError('No such PyLink client exists.')
+
+        self._send_with_prefix(source, 'NICK %s' % newnick)
+
+        self.users[source].nick = newnick
+
+        # Update the nick TS for consistency with other protocols (it isn't actually used in S2S)
+        self.users[source].ts = int(time.time())
+
+    def sjoin(self, server, channel, users, ts=None, modes=set()):
+        """Sends an SJOIN for a group of users to a channel.
+
+        The sender should always be a Server ID (SID). TS is optional, and defaults
+        to the one we've stored in the channel state if not given.
+        <users> is a list of (prefix mode, UID) pairs:
+
+        Example uses:
+            sjoin('100', '#test', [('', 'user0@0'), ('o', user1@1'), ('v', 'someone@2')])
+            sjoin(self.sid, '#test', [('o', self.pseudoclient.uid)])
+        """
+
+        server = server or self.sid
+        if not server:
+            raise LookupError('No such PyLink client exists.')
+        log.debug('(%s) sjoin: got %r for users', self.name, users)
+
+        njoin_prefix = ':%s NJOIN %s :' % (self._expandPUID(server), channel)
+        # Format the user list into strings such as @user1, +user2, user3, etc.
+        nicks_to_send = []
+        for userpair in users:
+            prefixes, uid = userpair
+
+            if uid not in self.users:
+                log.warning('(%s) Trying to NJOIN missing user %s?', self.name, uid)
+                continue
+            elif uid in self._channels[channel].users:
+                # Don't rejoin users already in the channel, this causes errors with ngIRCd.
+                continue
+
+            self._channels[channel].users.add(uid)
+            self.users[uid].channels.add(channel)
+
+            self.apply_modes(channel, (('+%s' % prefix, uid) for prefix in userpair[0]))
+
+            nicks_to_send.append(''.join(self.prefixmodes[modechar] for modechar in userpair[0]) + \
+                                 self._expandPUID(userpair[1]))
+
+        if nicks_to_send:
+            # Use 13 args max per line: this is equal to the max of 15 minus the command name and target channel.
+            for message in utils.wrap_arguments(njoin_prefix, nicks_to_send, self.S2S_BUFSIZE, separator=',', max_args_per_line=13):
+                self.send(message)
+
+        if modes:
+            # Burst modes separately if there are any.
+            log.debug("(%s) sjoin: bursting modes %r for channel %r now", self.name, modes, channel)
+            self.mode(server, channel, modes)
+
+    def set_server_ban(self, source, duration, user='*', host='*', reason='User banned'):
+        """
+        Sets a server ban.
+        """
+        # <- :jlu5 GLINE *!*@bad.user 3d :test
+        assert not (user == host == '*'), "Refusing to set ridiculous ban on *@*"
+        self._send_with_prefix(source, 'GLINE *!%s@%s %s :%s' % (user, host, duration, reason))
+
+    def update_client(self, target, field, text):
+        """Updates the ident, host, or realname of any connected client."""
+        field = field.upper()
+
+        if field not in ('IDENT', 'HOST', 'REALNAME', 'GECOS'):
+            raise NotImplementedError("Changing field %r of a client is "
+                                      "unsupported by this protocol." % field)
+
+        real_target = self._expandPUID(target)
+        if field == 'IDENT':
+            self.users[target].ident = text
+            self._send_with_prefix(self.sid, 'METADATA %s user :%s' % (real_target, text))
+
+            if not self.is_internal_client(target):
+                # If the target wasn't one of our clients, send a hook payload for other plugins to listen to.
+                self.call_hooks([self.sid, 'CHGIDENT', {'target': target, 'newident': text}])
+
+        elif field == 'HOST':
+            self.users[target].host = text
+
+            if self.is_internal_client(target):
+                # For our own clients, replace the real host.
+                self._send_with_prefix(self.sid, 'METADATA %s host :%s' % (real_target, text))
+            else:
+                # For others, update the cloaked host and force a umode +x.
+                self._send_with_prefix(self.sid, 'METADATA %s cloakhost :%s' % (real_target, text))
+
+                if ('x', None) not in self.users[target].modes:
+                    log.debug('(%s) Forcing umode +x on %r as part of cloak setting', self.name, target)
+                    self.mode(self.sid, target, [('+x', None)])
+
+                self.call_hooks([self.sid, 'CHGHOST', {'target': target, 'newhost': text}])
+
+        elif field in ('REALNAME', 'GECOS'):
+            self.users[target].realname = text
+            self._send_with_prefix(self.sid, 'METADATA %s info :%s' % (real_target, text))
+
+            if not self.is_internal_client(target):
+                self.call_hooks([self.sid, 'CHGNAME', {'target': target, 'newgecos': text}])
+
+    ### Handlers
+
+    def handle_376(self, source, command, args):
+        # 376 is used to denote end of server negotiation - we send our info back at this point.
+        # <- :ngircd.midnight.local 005 pylink-devel.int NETWORK=ngircd-test :is my network name
+        # <- :ngircd.midnight.local 005 pylink-devel.int RFC2812 IRCD=ngIRCd CHARSET=UTF-8 CASEMAPPING=ascii PREFIX=(qaohv)~&@%+ CHANTYPES=#&+ CHANMODES=beI,k,l,imMnOPQRstVz CHANLIMIT=#&+:10 :are supported on this server
+        # <- :ngircd.midnight.local 005 pylink-devel.int CHANNELLEN=50 NICKLEN=21 TOPICLEN=490 AWAYLEN=127 KICKLEN=400 MODES=5 MAXLIST=beI:50 EXCEPTS=e INVEX=I PENALTY :are supported on this server
+        def f(numeric, msg):
+            self._send_with_prefix(self.sid, '%s %s %s' % (numeric, self.uplink, msg))
+        f('005', 'NETWORK=%s :is my network name' % self.get_full_network_name())
+        f('005', 'RFC2812 IRCD=PyLink CHARSET=UTF-8 CASEMAPPING=%s PREFIX=%s CHANTYPES=# '
+          'CHANMODES=%s,%s,%s,%s :are supported on this server' % (self.casemapping, self._caps['PREFIX'],
+          self.cmodes['*A'], self.cmodes['*B'], self.cmodes['*C'], self.cmodes['*D']))
+        f('005', 'CHANNELLEN NICKLEN=%s EXCEPTS=E INVEX=I :are supported on this server' % self.maxnicklen)
+
+        # 376 (end of MOTD) marks the end of extended server negotiation per
+        # https://github.com/ngircd/ngircd/blob/master/doc/Protocol.txt#L103-L112
+        f('376', ":End of server negotiation, happy PyLink'ing!")
+
+    def handle_chaninfo(self, source, command, args):
+        # https://github.com/ngircd/ngircd/blob/722afc1b810cef74dbd2738d71866176fd974ec2/doc/Protocol.txt#L146-L159
+        # CHANINFO has 3 styles depending on the amount of information applicable to a channel:
+        #    CHANINFO <channel> +<modes>
+        #    CHANINFO <channel> +<modes> <topic>
+        #    CHANINFO <channel> +<modes> <key> <limit> <topic>
+        # If there is no key, the key is "*". If there is no limit, the limit is "0".
+
+        channel = args[0]
+        # Get rid of +l and +k in the initial parsing; we handle that later by looking at the CHANINFO arguments
+        modes = self.parse_modes(channel, args[1].replace('l', '').replace('k', ''))
+
+        if len(args) >= 3:
+            topic = args[-1]
+            if topic:
+                log.debug('(%s) handle_chaninfo: setting topic for %s to %r', self.name, channel, topic)
+                self._channels[channel].topic = topic
+                self._channels[channel].topicset = True
+
+        if len(args) >= 5:
+            key = args[2]
+            limit = args[3]
+            if key != '*':
+                modes.append(('+k', key))
+            if limit != '0':
+                modes.append(('+l', limit))
+
+        self.apply_modes(channel, modes)
+
+    def handle_join(self, source, command, args):
+        # RFC 2813 is odd to say the least... https://tools.ietf.org/html/rfc2813#section-4.2.1
+        # Basically, we expect messages of the forms:
+        # <- :jlu5 JOIN #test\x07o
+        # <- :jlu5 JOIN #moretest
+        for chanpair in args[0].split(','):
+            # Normalize channel case.
+            try:
+                channel, status = chanpair.split('\x07', 1)
+                if status in 'ov':
+                    self.apply_modes(channel, [('+' + status, source)])
+            except ValueError:
+                channel = chanpair
+
+            c = self._channels[channel]
+
+            self.users[source].channels.add(channel)
+            self._channels[channel].users.add(source)
+
+            # Call hooks manually, because one JOIN command have multiple channels.
+            self.call_hooks([source, command, {'channel': channel, 'users': [source], 'modes': c.modes}])
+
+    def handle_kill(self, source, command, args):
+        """Handles incoming KILLs."""
+
+        # ngIRCd sends QUIT after KILL for its own clients, so we shouldn't process this by itself
+        # unless we're the target.
+        killed = self._get_UID(args[0])
+        if self.is_internal_client(killed):
+            return super().handle_kill(source, command, args)
+        else:
+            log.debug("(%s) Ignoring KILL to %r as it isn't meant for us; we should see a QUIT soon",
+                      self.name, killed)
+
+    def _check_cloak_change(self, target):
+        u = self.users[target]
+        old_host = u.host
+
+        if ('x', None) in u.modes and u.cloaked_host:
+            u.host = u.cloaked_host
+        elif u.realhost:
+            u.host = u.realhost
+
+        # Something changed, so send a CHGHOST hook
+        if old_host != u.host:
+            self.call_hooks([target, 'CHGHOST', {'target': target, 'newhost': u.host}])
+
+    def handle_metadata(self, source, command, args):
+        """Handles various user metadata for ngIRCd (cloaked host, account name, etc.)"""
+        # <- :ngircd.midnight.local METADATA jlu5 cloakhost :hidden-3a2a739e.ngircd.midnight.local
+        target = self._get_UID(args[0])
+
+        if target not in self.users:
+            log.warning("(%s) Ignoring METADATA to missing user %r?", self.name, target)
+            return
+
+        datatype = args[1]
+        u = self.users[target]
+
+        if datatype == 'cloakhost':  # Set cloaked host
+            u.cloaked_host = args[-1]
+            self._check_cloak_change(target)
+
+        elif datatype == 'host':  # Host changing. This actually sets the "real host" that ngIRCd stores
+            u.realhost = args[-1]
+            self._check_cloak_change(target)
+
+        elif datatype == 'user':  # Ident changing
+            u.ident = args[-1]
+            self.call_hooks([target, 'CHGIDENT', {'target': target, 'newident': args[-1]}])
+
+        elif datatype == 'info':  # Realname changing
+            u.realname = args[-1]
+            self.call_hooks([target, 'CHGNAME', {'target': target, 'newgecos': args[-1]}])
+
+        elif datatype == 'accountname':  # Services account
+            self.call_hooks([target, 'CLIENT_SERVICES_LOGIN', {'text': args[-1]}])
+
+    def handle_nick(self, source, command, args):
+        """
+        Handles the NICK command, used for server introductions and nick changes.
+        """
+        if len(args) >= 2:
+            # User introduction:
+            # <- :ngircd.midnight.local NICK jlu5 1 ~jlu5 localhost 1 +io :realname
+            nick = args[0]
+            assert source in self.servers, "Server %r tried to introduce nick %r but isn't in the servers index?" % (source, nick)
+            self._check_nick_collision(nick)
+
+            ident = args[2]
+            host = args[3]
+            uid = self._uidgen.next_uid(prefix=nick)
+            realname = args[-1]
+
+            ts = int(time.time())
+            self.users[uid] = User(self, nick, ts, uid, source, ident=ident, host=host,
+                                   realname=realname, realhost=host)
+            parsedmodes = self.parse_modes(uid, [args[5]])
+            self.apply_modes(uid, parsedmodes)
+
+            # Add the nick to the list of users on its server; this is used for SQUIT tracking
+            self.servers[source].users.add(uid)
+
+            # Check away status and cloaked host changes
+            self._check_umode_away_change(uid)
+            self._check_cloak_change(uid)
+
+            return {'uid': uid, 'ts': ts, 'nick': nick, 'realhost': host, 'host': host, 'ident': ident,
+                    'parse_as': 'UID', 'ip': '0.0.0.0'}
+        else:
+            # Nick changes:
+            # <- :jlu5 NICK :jlu5_
+            oldnick = self.users[source].nick
+            newnick = self.users[source].nick = args[0]
+            return {'newnick': newnick, 'oldnick': oldnick}
+
+    def handle_njoin(self, source, command, args):
+        # <- :ngircd.midnight.local NJOIN #test :tester,@%jlu5
+
+        channel = args[0]
+        chandata = self._channels[channel].deepcopy()
+        namelist = []
+
+        # Reverse the modechar->modeprefix mapping for quicker lookup
+        prefixchars = {v: k for k, v in self.prefixmodes.items()}
+        for userpair in args[1].split(','):
+            # Some regex magic to split the prefix from the nick.
+            r = re.search(r'([%s]*)(.*)' % ''.join(self.prefixmodes.values()), userpair)
+            user = self._get_UID(r.group(2))
+            modeprefix = r.group(1)
+
+            if modeprefix:
+                modes = {('+%s' % prefixchars[mode], user) for mode in modeprefix}
+                self.apply_modes(channel, modes)
+            namelist.append(user)
+
+            # Final bits of state tracking. (I hate having to do this everywhere...)
+            self.users[user].channels.add(channel)
+            self._channels[channel].users.add(user)
+
+        return {'channel': channel, 'users': namelist, 'modes': [], 'channeldata': chandata}
+
+    def handle_pass(self, source, command, args):
+        """
+        Handles phase one of the ngIRCd login process (password auth and version info).
+        """
+        # PASS is step one of server introduction, and is used to send the server info and password.
+        # <- :ngircd.midnight.local PASS xyzpassword 0210-IRC+ ngIRCd|24~3-gbc728f92:CHLMSXZ PZ
+        recvpass = args[0]
+        if recvpass != self.serverdata['recvpass']:
+            raise ProtocolError("RECVPASS from uplink does not match configuration!")
+
+        if 'IRC+' not in args[1]:
+            raise ProtocolError("Linking to non-ngIRCd server using this protocol module is not supported")
+
+    def handle_ping(self, source, command, args):
+        """
+        Handles incoming PINGs (and implicit end of burst).
+        """
+        self._send_with_prefix(self.sid, 'PONG %s :%s' % (self._expandPUID(self.sid), args[-1]), queue=False)
+
+        if not self.servers[source].has_eob:
+            # Treat the first PING we receive as end of burst.
+            self.servers[source].has_eob = True
+
+            if source == self.uplink:
+                self.connected.set()
+
+            # Return the endburst hook.
+            return {'parse_as': 'ENDBURST'}
+
+    def handle_server(self, source, command, args):
+        """
+        Handles the SERVER command.
+        """
+        # <- :ngircd.midnight.local SERVER ngircd.midnight.local 1 :ngIRCd dev server
+        servername = args[0].lower()
+        serverdesc = args[-1]
+
+        # The uplink should be set to None for the uplink; otherwise, set it equal to the sender server.
+        self.servers[servername] = Server(self, source if source != servername else None, servername, desc=serverdesc)
+
+        if self.uplink is None:
+            self.uplink = servername
+            log.debug('(%s) Got %s as uplink', self.name, servername)
+        else:
+            # Only send the SERVER hook if this isn't the initial connection.
+            return {'name': servername, 'sid': None, 'text': serverdesc}
+
+Class = NgIRCdProtocol

protocols/ts6.py

@@ -2,29 +2,47 @@
 ts6.py: PyLink protocol module for TS6-based IRCds (charybdis, elemental-ircd).
 """
 
-import time
-import sys
-import os
 import re
+import time
 
-from pylinkirc import utils
+from pylinkirc import conf, utils
 from pylinkirc.classes import *
 from pylinkirc.log import log
-from pylinkirc.protocols.ts6_common import *
+from pylinkirc.protocols.ts6_common import TS6BaseProtocol
+
+__all__ = ['TS6Protocol']
+
 
 class TS6Protocol(TS6BaseProtocol):
-    def __init__(self, irc):
+
-        super().__init__(irc)
+    SUPPORTED_IRCDS = ('charybdis', 'elemental', 'chatircd', 'ratbox')
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._ircd = self.serverdata.get('ircd', 'elemental' if self.serverdata.get('use_elemental_modes')
+                                                             else 'charybdis')
+        self._ircd = self._ircd.lower()
+        if self._ircd not in self.SUPPORTED_IRCDS:
+            log.warning("(%s) Unsupported IRCd %r; falling back to 'charybdis' instead", self.name, self._ircd)
+            self._ircd = 'charybdis'
+
+        self._can_chghost = False
+        if self._ircd in ('charybdis', 'elemental', 'chatircd'):
+            # Charybdis and derivatives allow slashes in hosts. Ratbox does not.
+            self.protocol_caps |= {'slash-in-hosts'}
+            self._can_chghost = True
+
         self.casemapping = 'rfc1459'
         self.hook_map = {'SJOIN': 'JOIN', 'TB': 'TOPIC', 'TMODE': 'MODE', 'BMASK': 'MODE',
-                         'EUID': 'UID', 'RSFNC': 'SVSNICK', 'ETB': 'TOPIC'}
+                         'EUID': 'UID', 'RSFNC': 'SVSNICK', 'ETB': 'TOPIC',
+                         # ENCAP LOGIN is used on burst for EUID-less servers
+                         'LOGIN': 'CLIENT_SERVICES_LOGIN'}
 
-        # Track whether we've received end-of-burst from the uplink.
+        self.required_caps = {'TB', 'ENCAP', 'QS', 'CHW'}
-        self.has_eob = False
 
     ### OUTGOING COMMANDS
 
-    def spawnClient(self, nick, ident='null', host='null', realhost=None, modes=set(),
+    def spawn_client(self, nick, ident='null', host='null', realhost=None, modes=set(),
             server=None, ip='0.0.0.0', realname=None, ts=None, opertype='IRC Operator',
             manipulatable=False):
         """
@@ -33,9 +51,9 @@ class TS6Protocol(TS6BaseProtocol):
         Note: No nick collision / valid nickname checks are done here; it is
         up to plugins to make sure they don't introduce anything invalid.
         """
-        server = server or self.irc.sid
+        server = server or self.sid
 
-        if not self.irc.isInternalServer(server):
+        if not self.is_internal_server(server):
             raise ValueError('Server %r is not a PyLink server!' % server)
 
         uid = self.uidgen[server].next_uid()
@@ -44,34 +62,69 @@ class TS6Protocol(TS6BaseProtocol):
         # parameters: nickname, hopcount, nickTS, umodes, username,
         # visible hostname, IP address, UID, real hostname, account name, gecos
         ts = ts or int(time.time())
-        realname = realname or self.irc.botdata['realname']
+        realname = realname or conf.conf['pylink']['realname']
-        realhost = realhost or host
+        raw_modes = self.join_modes(modes)
-        raw_modes = self.irc.joinModes(modes)
+        u = self.users[uid] = User(self, nick, ts, uid, server, ident=ident, host=host,
-        u = self.irc.users[uid] = IrcUser(nick, ts, uid, ident=ident, host=host, realname=realname,
+                                   realname=realname, realhost=realhost or host, ip=ip,
-            realhost=realhost, ip=ip, manipulatable=manipulatable, opertype=opertype)
+                                   manipulatable=manipulatable, opertype=opertype)
 
-        self.irc.applyModes(uid, modes)
+        self.apply_modes(uid, modes)
-        self.irc.servers[server].users.add(uid)
+        self.servers[server].users.add(uid)
 
-        self._send(server, "EUID {nick} 1 {ts} {modes} {ident} {host} {ip} {uid} "
+        if 'EUID' in self._caps:
-                "{realhost} * :{realname}".format(ts=ts, host=host,
+            # charybdis-style EUID
-                nick=nick, ident=ident, uid=uid,
+            self._send_with_prefix(server, "EUID {nick} {hopcount} {ts} {modes} {ident} {host} {ip} {uid} "
-                modes=raw_modes, ip=ip, realname=realname,
+                                           "{realhost} * :{realname}".format(ts=ts, host=host,
-                realhost=realhost))
+                                           nick=nick, ident=ident, uid=uid,
+                                           modes=raw_modes, ip=ip, realname=realname,
+                                           realhost=realhost or host,
+                                           hopcount=self.servers[server].hopcount))
+        else:
+            # Basic ratbox UID
+            self._send_with_prefix(server, "UID {nick} {hopcount} {ts} {modes} {ident} {host} {ip} {uid} "
+                                           ":{realname}".format(ts=ts, host=host,
+                                           nick=nick, ident=ident, uid=uid,
+                                           modes=raw_modes, ip=ip, realname=realname,
+                                           hopcount=self.servers[server].hopcount))
+
+            if realhost:
+                # If real host is specified, send it using ENCAP REALHOST
+                self._send_with_prefix(uid, "ENCAP * REALHOST %s" % realhost)
 
         return u
 
+    def spawn_server(self, name, sid=None, uplink=None, desc=None):
+        """
+        Spawns a server off a PyLink server. desc (server description)
+        defaults to the one in the config. uplink defaults to the main PyLink
+        server, and sid (the server ID) is automatically generated if not
+        given.
+        """
+        desc = desc or self.serverdata.get('serverdesc') or conf.conf['pylink']['serverdesc']
+        if self.serverdata.get('hidden', False):
+            desc = '(H) ' + desc
+
+        return super().spawn_server(name, sid=sid, uplink=uplink, desc=desc)
+
     def join(self, client, channel):
         """Joins a PyLink client to a channel."""
-        channel = self.irc.toLower(channel)
         # JOIN:
         # parameters: channelTS, channel, '+' (a plus sign)
-        if not self.irc.isInternalClient(client):
+        if not self.is_internal_client(client):
-            log.error('(%s) Error trying to join %r to %r (no such client exists)', self.irc.name, client, channel)
+            log.error('(%s) Error trying to join %r to %r (no such client exists)', self.name, client, channel)
             raise LookupError('No such PyLink client exists.')
-        self._send(client, "JOIN {ts} {channel} +".format(ts=self.irc.channels[channel].ts, channel=channel))
+        self._send_with_prefix(client, "JOIN {ts} {channel} +".format(ts=self._channels[channel].ts, channel=channel))
-        self.irc.channels[channel].users.add(client)
+        self._channels[channel].users.add(client)
-        self.irc.users[client].channels.add(channel)
+        self.users[client].channels.add(channel)
+
+    def oper_notice(self, source, text):
+        """
+        Send a message to all opers.
+        """
+        if self.is_internal_server(source):
+            # Charybdis TS6 only allows OPERWALL from users
+            source = self.pseudoclient.uid
+        self._send_with_prefix(source, 'OPERWALL :%s' % text)
 
     def sjoin(self, server, channel, users, ts=None, modes=set()):
         """Sends an SJOIN for a group of users to a channel.
@@ -82,7 +135,7 @@ class TS6Protocol(TS6BaseProtocol):
 
         Example uses:
             sjoin('100', '#test', [('', '100AAABBC'), ('o', 100AAABBB'), ('v', '100AAADDD')])
-            sjoin(self.irc.sid, '#test', [('o', self.irc.pseudoclient.uid)])
+            sjoin(self.sid, '#test', [('o', self.pseudoclient.uid)])
         """
         # https://github.com/grawity/irc-docs/blob/master/server/ts6.txt#L821
         # parameters: channelTS, channel, simple modes, opt. mode parameters..., nicklist
@@ -93,30 +146,33 @@ class TS6Protocol(TS6BaseProtocol):
         # their status ('@+', '@', '+' or ''), for example:
         # '@+1JJAAAAAB +2JJAAAA4C 1JJAAAADS'. All users must be behind the source server
         # so it is not possible to use this message to force users to join a channel.
-        channel = self.irc.toLower(channel)
+        server = server or self.sid
-        server = server or self.irc.sid
         assert users, "sjoin: No users sent?"
-        log.debug('(%s) sjoin: got %r for users', self.irc.name, users)
+        log.debug('(%s) sjoin: got %r for users', self.name, users)
         if not server:
             raise LookupError('No such PyLink client exists.')
 
-        modes = set(modes or self.irc.channels[channel].modes)
+        modes = set(modes or self._channels[channel].modes)
-        orig_ts = self.irc.channels[channel].ts
+        orig_ts = self._channels[channel].ts
         ts = ts or orig_ts
 
         # Get all the ban modes in a separate list. These are bursted using a separate BMASK
         # command.
-        banmodes = {k: set() for k in self.irc.cmodes['*A']}
+        banmodes = {k: [] for k in self.cmodes['*A']}
         regularmodes = []
-        log.debug('(%s) Unfiltered SJOIN modes: %s', self.irc.name, modes)
+        log.debug('(%s) Unfiltered SJOIN modes: %s', self.name, modes)
         for mode in modes:
             modechar = mode[0][-1]
-            if modechar in self.irc.cmodes['*A']:
+            if modechar in self.cmodes['*A']:
                 # Mode character is one of 'beIq'
-                banmodes[modechar].add(mode[1])
+                if (modechar, mode[1]) in self._channels[channel].modes:
+                    # Don't reset modes that are already set.
+                    continue
+
+                banmodes[modechar].append(mode[1])
             else:
                 regularmodes.append(mode)
-        log.debug('(%s) Filtered SJOIN modes to be regular modes: %s, banmodes: %s', self.irc.name, regularmodes, banmodes)
+        log.debug('(%s) Filtered SJOIN modes to be regular modes: %s, banmodes: %s', self.name, regularmodes, banmodes)
 
         changedmodes = modes
         while users[:12]:
@@ -128,22 +184,22 @@ class TS6Protocol(TS6BaseProtocol):
                 prefixes, user = userpair
                 prefixchars = ''
                 for prefix in prefixes:
-                    pr = self.irc.prefixmodes.get(prefix)
+                    pr = self.prefixmodes.get(prefix)
                     if pr:
                         prefixchars += pr
                         changedmodes.add(('+%s' % prefix, user))
                 namelist.append(prefixchars+user)
                 uids.append(user)
                 try:
-                    self.irc.users[user].channels.add(channel)
+                    self.users[user].channels.add(channel)
                 except KeyError:  # Not initialized yet?
-                    log.debug("(%s) sjoin: KeyError trying to add %r to %r's channel list?", self.irc.name, channel, user)
+                    log.debug("(%s) sjoin: KeyError trying to add %r to %r's channel list?", self.name, channel, user)
             users = users[12:]
             namelist = ' '.join(namelist)
-            self._send(server, "SJOIN {ts} {channel} {modes} :{users}".format(
+            self._send_with_prefix(server, "SJOIN {ts} {channel} {modes} :{users}".format(
                     ts=ts, users=namelist, channel=channel,
-                    modes=self.irc.joinModes(regularmodes)))
+                    modes=self.join_modes(regularmodes)))
-            self.irc.channels[channel].users.update(uids)
+            self._channels[channel].users.update(uids)
 
         # Now, burst bans.
         # <- :42X BMASK 1424222769 #dev b :*!test@*.isp.net *!badident@*
@@ -151,12 +207,12 @@ class TS6Protocol(TS6BaseProtocol):
             # Max 15-3 = 12 bans per line to prevent cut off. (TS6 allows a max of 15 parameters per
             # line)
             if bans:
-                log.debug('(%s) sjoin: bursting mode %s with bans %s, ts:%s', self.irc.name, bmode, bans, ts)
+                log.debug('(%s) sjoin: bursting mode %s with bans %s, ts:%s', self.name, bmode, bans, ts)
-                bans = list(bans)  # Convert into list for splicing
+                msgprefix = ':{sid} BMASK {ts} {channel} {bmode} :'.format(sid=server, ts=ts,
-                while bans[:12]:
+                                                                          channel=channel, bmode=bmode)
-                    self._send(server, "BMASK {ts} {channel} {bmode} :{bans}".format(ts=ts,
+                # Actually, we cut off at 17 arguments/line, since the prefix and command name don't count.
-                               channel=channel, bmode=bmode, bans=' '.join(bans[:12])))
+                for msg in utils.wrap_arguments(msgprefix, bans, self.S2S_BUFSIZE, max_args_per_line=17):
-                    bans = bans[12:]
+                    self.send(msg)
 
         self.updateTS(server, channel, ts, changedmodes)
 
@@ -165,155 +221,188 @@ class TS6Protocol(TS6BaseProtocol):
         # c <- :0UYAAAAAA TMODE 0 #a +o 0T4AAAAAC
         # u <- :0UYAAAAAA MODE 0UYAAAAAA :-Facdefklnou
 
-        if (not self.irc.isInternalClient(numeric)) and \
+        if (not self.is_internal_client(numeric)) and \
-                (not self.irc.isInternalServer(numeric)):
+                (not self.is_internal_server(numeric)):
             raise LookupError('No such PyLink client/server exists.')
 
-        self.irc.applyModes(target, modes)
+        self.apply_modes(target, modes)
         modes = list(modes)
 
-        if utils.isChannel(target):
+        if self.is_channel(target):
-            ts = ts or self.irc.channels[self.irc.toLower(target)].ts
+            ts = ts or self._channels[target].ts
             # TMODE:
             # parameters: channelTS, channel, cmode changes, opt. cmode parameters...
 
             # On output, at most ten cmode parameters should be sent; if there are more,
             # multiple TMODE messages should be sent.
-            while modes[:10]:
+            msgprefix = ':%s TMODE %s %s ' % (numeric, ts, target)
-                # Seriously, though. If you send more than 10 mode parameters in
+            bufsize = self.S2S_BUFSIZE - len(msgprefix)
-                # a line, charybdis will silently REJECT the entire command!
-                joinedmodes = self.irc.joinModes([m for m in modes[:10] if m[0] not in self.irc.cmodes['*A']])
-                modes = modes[10:]
-                self._send(numeric, 'TMODE %s %s %s' % (ts, target, joinedmodes))
-        else:
-            joinedmodes = self.irc.joinModes(modes)
-            self._send(numeric, 'MODE %s %s' % (target, joinedmodes))
 
-    def topicBurst(self, numeric, target, text):
+            for modestr in self.wrap_modes(modes, bufsize, max_modes_per_msg=10):
+                self.send(msgprefix + modestr)
+        else:
+            joinedmodes = self.join_modes(modes)
+            self._send_with_prefix(numeric, 'MODE %s %s' % (target, joinedmodes))
+
+    def topic_burst(self, numeric, target, text):
         """Sends a topic change from a PyLink server. This is usually used on burst."""
-        if not self.irc.isInternalServer(numeric):
+        if not self.is_internal_server(numeric):
             raise LookupError('No such PyLink server exists.')
         # TB
         # capab: TB
         # source: server
         # propagation: broadcast
         # parameters: channel, topicTS, opt. topic setter, topic
-        ts = self.irc.channels[target].ts
+        ts = self._channels[target].ts
-        servername = self.irc.servers[numeric].name
+        servername = self.servers[numeric].name
-        self._send(numeric, 'TB %s %s %s :%s' % (target, ts, servername, text))
+        self._send_with_prefix(numeric, 'TB %s %s %s :%s' % (target, ts, servername, text))
-        self.irc.channels[target].topic = text
+        self._channels[target].topic = text
-        self.irc.channels[target].topicset = True
+        self._channels[target].topicset = True
 
     def invite(self, numeric, target, channel):
         """Sends an INVITE from a PyLink client.."""
-        if not self.irc.isInternalClient(numeric):
+        if not self.is_internal_client(numeric):
             raise LookupError('No such PyLink client exists.')
-        self._send(numeric, 'INVITE %s %s %s' % (target, channel, self.irc.channels[channel].ts))
+        self._send_with_prefix(numeric, 'INVITE %s %s %s' % (target, channel, self._channels[channel].ts))
 
     def knock(self, numeric, target, text):
         """Sends a KNOCK from a PyLink client."""
-        if 'KNOCK' not in self.irc.caps:
+        if 'KNOCK' not in self._caps:
             log.debug('(%s) knock: Dropping KNOCK to %r since the IRCd '
-                      'doesn\'t support it.', self.irc.name, target)
+                      'doesn\'t support it.', self.name, target)
             return
-        if not self.irc.isInternalClient(numeric):
+        if not self.is_internal_client(numeric):
             raise LookupError('No such PyLink client exists.')
         # No text value is supported here; drop it.
-        self._send(numeric, 'KNOCK %s' % target)
+        self._send_with_prefix(numeric, 'KNOCK %s' % target)
 
-    def updateClient(self, target, field, text):
+    def set_server_ban(self, source, duration, user='*', host='*', reason='User banned'):
+        """
+        Sets a server ban.
+        """
+        # source: user
+        # parameters: target server mask, duration, user mask, host mask, reason
+        assert not (user == host == '*'), "Refusing to set ridiculous ban on *@*"
+
+        if not source in self.users:
+            log.debug('(%s) Forcing KLINE sender to %s as TS6 does not allow KLINEs from servers', self.name, self.pseudoclient.uid)
+            source = self.pseudoclient.uid
+
+        self._send_with_prefix(source, 'ENCAP * KLINE %s %s %s :%s' % (duration, user, host, reason))
+
+    def update_client(self, target, field, text):
         """Updates the hostname of any connected client."""
         field = field.upper()
-        if field == 'HOST':
+        if field == 'HOST' and self._can_chghost:
-            self.irc.users[target].host = text
+            self.users[target].host = text
-            self._send(self.irc.sid, 'CHGHOST %s :%s' % (target, text))
+            self._send_with_prefix(self.sid, 'CHGHOST %s :%s' % (target, text))
-            if not self.irc.isInternalClient(target):
+            if not self.is_internal_client(target):
                 # If the target isn't one of our clients, send hook payload
                 # for other plugins to listen to.
-                self.irc.callHooks([self.irc.sid, 'CHGHOST',
+                self.call_hooks([self.sid, 'CHGHOST',
                                    {'target': target, 'newhost': text}])
         else:
             raise NotImplementedError("Changing field %r of a client is "
-                                      "unsupported by this protocol." % field)
+                                      "unsupported by this IRCd." % field)
-
-    def ping(self, source=None, target=None):
-        """Sends a PING to a target server. Periodic PINGs are sent to our uplink
-        automatically by the Irc() internals; plugins shouldn't have to use this."""
-        source = source or self.irc.sid
-        if source is None:
-            return
-        if target is not None:
-            self._send(source, 'PING %s %s' % (source, target))
-        else:
-            self._send(source, 'PING %s' % source)
 
     ### Core / handlers
 
-    def connect(self):
+    def post_connect(self):
         """Initializes a connection to a server."""
-        ts = self.irc.start_ts
+        ts = self.start_ts
-        self.has_eob = False
 
-        f = self.irc.send
+        f = self.send
+
+        # Base TS6 mode set from ratbox.
+        self.cmodes.update({'sslonly': 'S', 'noknock': 'p',
+                            '*A': 'beI',
+                            '*B': 'k',
+                            '*C': 'l',
+                            '*D': 'imnpstrS'})
 
         # https://github.com/grawity/irc-docs/blob/master/server/ts6.txt#L80
-        chary_cmodes = { # TS6 generic modes (note that +p is noknock instead of private):
+        if self._ircd in ('charybdis', 'elemental', 'chatircd'):
-                        'op': 'o', 'voice': 'v', 'ban': 'b', 'key': 'k', 'limit':
+            self.cmodes.update({
-                        'l', 'moderated': 'm', 'noextmsg': 'n', 'noknock': 'p',
+                'quiet': 'q', 'redirect': 'f', 'freetarget': 'F',
-                        'secret': 's', 'topiclock': 't', 'inviteonly': 'i',
+                'joinflood': 'j', 'largebanlist': 'L', 'permanent': 'P',
-                         # charybdis-specific modes:
+                'noforwards': 'Q', 'stripcolor': 'c', 'allowinvite':
-                        'quiet': 'q', 'redirect': 'f', 'freetarget': 'F',
+                'g', 'opmoderated': 'z', 'noctcp': 'C',
-                        'joinflood': 'j', 'largebanlist': 'L', 'permanent': 'P',
+                # charybdis modes provided by extensions
-                        'noforwards': 'Q', 'stripcolor': 'c', 'allowinvite':
+                'operonly': 'O', 'adminonly': 'A', 'sslonly': 'S',
-                        'g', 'opmoderated': 'z', 'noctcp': 'C',
+                'nonotice': 'T',
-                         # charybdis-specific modes provided by EXTENSIONS
+                '*A': 'beIq', '*B': 'k', '*C': 'lfj', '*D': 'mnprstFLPQcgzCOAST'
-                        'operonly': 'O', 'adminonly': 'A', 'sslonly': 'S',
+            })
-                        'nonotice': 'T',
+            self.umodes.update({
-                         # Now, map all the ABCD type modes:
+                'deaf': 'D', 'servprotect': 'S', 'admin': 'a',
-                        '*A': 'beIq', '*B': 'k', '*C': 'lfj', '*D': 'mnprstFLPQcgzCOAST'}
+                'invisible': 'i', 'oper': 'o', 'wallops': 'w',
+                'snomask': 's', 'noforward': 'Q', 'regdeaf': 'R',
+                'callerid': 'g', 'operwall': 'z', 'locops': 'l',
+                'cloak': 'x', 'override': 'p', 'ssl': 'Z',
+                '*A': '', '*B': '', '*C': '', '*D': 'DSaiowsQRgzlxpZ'
+            })
 
-        if self.irc.serverdata.get('use_owner'):
+            # Charybdis extbans
-            chary_cmodes['owner'] = 'y'
+            self.extbans_matching = {'ban_all_registered': '$a', 'ban_inchannel': '$c:', 'ban_account': '$a:',
-            self.irc.prefixmodes['y'] = '~'
+                                     'ban_all_opers': '$o', 'ban_realname': '$r:', 'ban_server': '$s:',
-        if self.irc.serverdata.get('use_admin'):
+                                     'ban_banshare': '$j:', 'ban_extgecos': '$x:', 'ban_all_ssl': '$z'}
-            chary_cmodes['admin'] = 'a'
+        elif self._ircd == 'ratbox':
-            self.irc.prefixmodes['a'] = '!'
+            self.umodes.update({
-        if self.irc.serverdata.get('use_halfop'):
+                'callerid': 'g', 'admin': 'a', 'sno_botfloods': 'b',
-            chary_cmodes['halfop'] = 'h'
+                'sno_clientconnections': 'c', 'sno_extclientconnections': 'C', 'sno_debug': 'd',
-            self.irc.prefixmodes['h'] = '%'
+                'sno_fullauthblock': 'f', 'sno_skill': 'k', 'locops': 'l', 'sno_rejectedclients': 'r',
+                'snomask': 's', 'sno_badclientconnections': 'u', 'sno_serverconnects': 'x',
+                'sno_stats': 'y', 'operwall': 'z', 'sno_operspy': 'Z', 'deaf': 'D', 'servprotect': 'S',
+                '*A': '', '*B': '', '*C': '', '*D': 'igoabcCdfklrsuwxyzZD'
+            })
 
-        self.irc.cmodes = chary_cmodes
+        # TODO: make these more flexible...
-
+        if self.serverdata.get('use_owner'):
-        # Define supported user modes
+            self.cmodes['owner'] = 'y'
-        chary_umodes = {'deaf': 'D', 'servprotect': 'S', 'admin': 'a',
+            self.prefixmodes['y'] = '~'
-                        'invisible': 'i', 'oper': 'o', 'wallops': 'w',
+        if self.serverdata.get('use_admin'):
-                        'snomask': 's', 'noforward': 'Q', 'regdeaf': 'R',
+            self.cmodes['admin'] = 'a'
-                        'callerid': 'g', 'operwall': 'z', 'locops': 'l',
+            self.prefixmodes['a'] = '!' if self._ircd != 'chatircd' else '&'
-                        'cloak': 'x', 'override': 'p',
+        if self.serverdata.get('use_halfop'):
-                        # Now, map all the ABCD type modes:
+            self.cmodes['halfop'] = 'h'
-                        '*A': '', '*B': '', '*C': '', '*D': 'DSaiowsQRgzlxp'}
+            self.prefixmodes['h'] = '%'
-        self.irc.umodes = chary_umodes
 
         # Toggles support of shadowircd/elemental-ircd specific channel modes:
         # +T (no notice), +u (hidden ban list), +E (no kicks), +J (blocks kickrejoin),
         # +K (no repeat messages), +d (no nick changes), and user modes:
-        # +B (bot), +C (blocks CTCP), +D (deaf), +V (no invites), +I (hides channel list)
+        # +B (bot), +C (blocks CTCP), +V (no invites), +I (hides channel list)
-        if self.irc.serverdata.get('use_elemental_modes'):
+        if self._ircd == 'elemental':
             elemental_cmodes = {'hiddenbans': 'u', 'nokick': 'E',
                                 'kicknorejoin': 'J', 'repeat': 'K', 'nonick': 'd',
                                 'blockcaps': 'G'}
-            self.irc.cmodes.update(elemental_cmodes)
+            self.cmodes.update(elemental_cmodes)
-            self.irc.cmodes['*D'] += ''.join(elemental_cmodes.values())
+            self.cmodes['*D'] += ''.join(elemental_cmodes.values())
 
-            elemental_umodes = {'noctcp': 'C', 'deaf': 'D', 'bot': 'B', 'noinvite': 'V',
+            elemental_umodes = {'noctcp': 'C', 'bot': 'B', 'noinvite': 'V', 'hidechans': 'I'}
-                                'hidechans': 'I'}
+            self.umodes.update(elemental_umodes)
-            self.irc.umodes.update(elemental_umodes)
+            self.umodes['*D'] += ''.join(elemental_umodes.values())
-            self.irc.umodes['*D'] += ''.join(elemental_umodes.values())
+
+        elif self._ircd == 'chatircd':
+            chatircd_cmodes = {'netadminonly': 'N'}
+            self.cmodes.update(chatircd_cmodes)
+            self.cmodes['*D'] += ''.join(chatircd_cmodes.values())
+
+            chatircd_umodes = {'netadmin': 'n', 'bot': 'B', 'sslonlymsg': 't'}
+            self.umodes.update(chatircd_umodes)
+            self.umodes['*D'] += ''.join(chatircd_umodes.values())
+
+        # Add definitions for all the inverted versions of the extbans.
+        if self.extbans_matching:
+            for k, v in self.extbans_matching.copy().items():
+                if k == 'ban_all_registered':
+                    newk = 'ban_unregistered'
+                else:
+                    newk = k.replace('_all_', '_').replace('ban_', 'ban_not_')
+                self.extbans_matching[newk] = '$~' + v[1:]
 
         # https://github.com/grawity/irc-docs/blob/master/server/ts6.txt#L55
-        f('PASS %s TS 6 %s' % (self.irc.serverdata["sendpass"], self.irc.sid))
+        f('PASS %s TS 6 %s' % (self.serverdata["sendpass"], self.sid))
 
-        # We request the following capabilities (for charybdis):
+        # We request the following capabilities:
 
         # QS: SQUIT doesn't send recursive quits for each users; required
         # by charybdis (Source: https://github.com/grawity/irc-docs/blob/master/server/ts-capab.txt)
@@ -324,20 +413,23 @@ class TS6Protocol(TS6BaseProtocol):
         # KNOCK: support for /knock
         # SAVE: support for SAVE (forces user to UID in nick collision)
         # SERVICES: adds mode +r (only registered users can join a channel)
-        # TB: topic burst command; we send this in topicBurst
+        # TB: topic burst command; we send this in topic_burst
         # EUID: extended UID command, which includes real hostname + account data info,
         #       and allows sending CHGHOST without ENCAP.
         # RSFNC: states that we support RSFNC (forced nick changed attempts). XXX: With atheme services,
         #        does this actually do anything?
         # EOPMOD: supports ETB (extended TOPIC burst) and =#channel messages for opmoderated +z
-        f('CAPAB :QS ENCAP EX CHW IE KNOCK SAVE SERVICES TB EUID RSFNC EOPMOD')
+        # KLN: supports remote KLINEs
+        f('CAPAB :QS ENCAP EX CHW IE KNOCK SAVE SERVICES TB EUID RSFNC EOPMOD SAVETS_100 KLN')
 
-        f('SERVER %s 0 :%s' % (self.irc.serverdata["hostname"],
+        sdesc = self.serverdata.get('serverdesc') or conf.conf['pylink']['serverdesc']
-                               self.irc.serverdata.get('serverdesc') or self.irc.botdata['serverdesc']))
+        if self.serverdata.get('hidden', False):
+            sdesc = '(H) ' + sdesc
+        f('SERVER %s 0 :%s' % (self.serverdata["hostname"], sdesc))
 
         # Finally, end all the initialization with a PING - that's Charybdis'
         # way of saying end-of-burst :)
-        self.ping()
+        self._ping_uplink()
 
     def handle_pass(self, numeric, command, args):
         """
@@ -346,20 +438,20 @@ class TS6Protocol(TS6BaseProtocol):
         """
         # <- PASS $somepassword TS 6 :42X
 
-        if args[0] != self.irc.serverdata['recvpass']:
+        if args[0] != self.serverdata['recvpass']:
             # Check if recvpass is correct
-            raise ProtocolError('Recvpass from uplink server %s does not match configuration!' % servername)
+            raise ProtocolError('Recvpass from uplink server %r does not match configuration!' % numeric)
 
         if args[1] != 'TS' and args[2] != '6':
             raise ProtocolError("Remote protocol version is too old! Is this even TS6?")
 
         numeric = args[-1]
-        log.debug('(%s) Found uplink SID as %r', self.irc.name, numeric)
+        log.debug('(%s) Found uplink SID as %r', self.name, numeric)
 
         # Server name and SID are sent in different messages, so we fill this
         # with dummy information until we get the actual sid.
-        self.irc.servers[numeric] = IrcServer(None, '')
+        self.servers[numeric] = Server(self, None, '')
-        self.irc.uplink = numeric
+        self.uplink = numeric
 
     def handle_capab(self, numeric, command, args):
         """
@@ -368,21 +460,18 @@ class TS6Protocol(TS6BaseProtocol):
         # We only get a list of keywords here. Charybdis obviously assumes that
         # we know what modes it supports (indeed, this is a standard list).
         # <- CAPAB :BAN CHW CLUSTER ENCAP EOPMOD EUID EX IE KLN KNOCK MLOCK QS RSFNC SAVE SERVICES TB UNKLN
-        self.irc.caps = caps = args[0].split()
+        self._caps = caps = args[0].split()
 
-        for required_cap in ('EUID', 'SAVE', 'TB', 'ENCAP', 'QS', 'CHW'):
+        for required_cap in self.required_caps:
             if required_cap not in caps:
                 raise ProtocolError('%s not found in TS6 capabilities list; this is required! (got %r)' % (required_cap, caps))
 
         if 'EX' in caps:
-            self.irc.cmodes['banexception'] = 'e'
+            self.cmodes['banexception'] = 'e'
         if 'IE' in caps:
-            self.irc.cmodes['invex'] = 'I'
+            self.cmodes['invex'] = 'I'
         if 'SERVICES' in caps:
-            self.irc.cmodes['regonly'] = 'r'
+            self.cmodes['regonly'] = 'r'
-
-        log.debug('(%s) self.irc.connected set!', self.irc.name)
-        self.irc.connected.set()
 
     def handle_ping(self, source, command, args):
         """Handles incoming PING commands."""
@@ -399,39 +488,38 @@ class TS6Protocol(TS6BaseProtocol):
         try:
             destination = args[1]
         except IndexError:
-            destination = self.irc.sid
+            destination = self.sid
-        if self.irc.isInternalServer(destination):
+        if self.is_internal_server(destination):
-            self._send(destination, 'PONG %s %s' % (destination, source))
+            self._send_with_prefix(destination, 'PONG %s %s' % (destination, source), queue=False)
 
-            if destination == self.irc.sid and not self.has_eob:
+            if not self.servers[source].has_eob:
-                # Charybdis' idea of endburst is just sending a PING. No, really!
+                # TS6 endburst is just sending a PING to the other server.
                 # https://github.com/charybdis-ircd/charybdis/blob/dc336d1/modules/core/m_server.c#L484-L485
-                self.has_eob = True
+                self.servers[source].has_eob = True
+
+                if source == self.uplink:
+                    log.debug('(%s) self.connected set!', self.name)
+                    self.connected.set()
 
                 # Return the endburst hook.
                 return {'parse_as': 'ENDBURST'}
 
-
-    def handle_pong(self, source, command, args):
-        """Handles incoming PONG commands."""
-        if source == self.irc.uplink:
-            self.irc.lastping = time.time()
-
     def handle_sjoin(self, servernumeric, command, args):
         """Handles incoming SJOIN commands."""
         # parameters: channelTS, channel, simple modes, opt. mode parameters..., nicklist
         # <- :0UY SJOIN 1451041566 #channel +nt :@0UYAAAAAB
-        channel = self.irc.toLower(args[1])
+        channel = args[1]
+        chandata = self._channels[channel].deepcopy()
         userlist = args[-1].split()
 
         modestring = args[2:-1] or args[2]
-        parsedmodes = self.irc.parseModes(channel, modestring)
+        parsedmodes = self.parse_modes(channel, modestring)
         namelist = []
 
         # Keep track of other modes that are added due to prefix modes being joined too.
         changedmodes = set(parsedmodes)
 
-        log.debug('(%s) handle_sjoin: got userlist %r for %r', self.irc.name, userlist, channel)
+        log.debug('(%s) handle_sjoin: got userlist %r for %r', self.name, userlist, channel)
         for userpair in userlist:
             # charybdis sends this in the form "@+UID1, +UID2, UID3, @UID4"
             r = re.search(r'([^\d]*)(.*)', userpair)
@@ -439,33 +527,34 @@ class TS6Protocol(TS6BaseProtocol):
             modeprefix = r.group(1) or ''
             finalprefix = ''
             assert user, 'Failed to get the UID from %r; our regex needs updating?' % userpair
-            log.debug('(%s) handle_sjoin: got modeprefix %r for user %r', self.irc.name, modeprefix, user)
+            log.debug('(%s) handle_sjoin: got modeprefix %r for user %r', self.name, modeprefix, user)
 
             # Don't crash when we get an invalid UID.
-            if user not in self.irc.users:
+            if user not in self.users:
                 log.debug('(%s) handle_sjoin: tried to introduce user %s not in our user list, ignoring...',
-                          self.irc.name, user)
+                          self.name, user)
                 continue
 
             for m in modeprefix:
                 # Iterate over the mapping of prefix chars to prefixes, and
                 # find the characters that match.
-                for char, prefix in self.irc.prefixmodes.items():
+                for char, prefix in self.prefixmodes.items():
                     if m == prefix:
                         finalprefix += char
             namelist.append(user)
-            self.irc.users[user].channels.add(channel)
+            self.users[user].channels.add(channel)
 
             # Only save mode changes if the remote has lower TS than us.
             changedmodes |= {('+%s' % mode, user) for mode in finalprefix}
-            self.irc.channels[channel].users.add(user)
+            self._channels[channel].users.add(user)
 
         # Statekeeping with timestamps
         their_ts = int(args[0])
-        our_ts = self.irc.channels[channel].ts
+        our_ts = self._channels[channel].ts
         self.updateTS(servernumeric, channel, their_ts, changedmodes)
 
-        return {'channel': channel, 'users': namelist, 'modes': parsedmodes, 'ts': their_ts}
+        return {'channel': channel, 'users': namelist, 'modes': parsedmodes, 'ts': their_ts,
+                'channeldata': chandata}
 
     def handle_join(self, numeric, command, args):
         """Handles incoming channel JOINs."""
@@ -474,175 +563,156 @@ class TS6Protocol(TS6BaseProtocol):
         ts = int(args[0])
         if args[0] == '0':
             # /join 0; part the user from all channels
-            oldchans = self.irc.users[numeric].channels.copy()
+            oldchans = self.users[numeric].channels.copy()
             log.debug('(%s) Got /join 0 from %r, channel list is %r',
-                      self.irc.name, numeric, oldchans)
+                      self.name, numeric, oldchans)
             for channel in oldchans:
-                self.irc.channels[channel].users.discard(numeric)
+                self._channels[channel].users.discard(numeric)
-                self.irc.users[numeric].channels.discard(channel)
+                self.users[numeric].channels.discard(channel)
             return {'channels': oldchans, 'text': 'Left all channels.', 'parse_as': 'PART'}
         else:
-            channel = self.irc.toLower(args[1])
+            channel = args[1]
             self.updateTS(numeric, channel, ts)
 
-            self.irc.users[numeric].channels.add(channel)
+            self.users[numeric].channels.add(channel)
-            self.irc.channels[channel].users.add(numeric)
+            self._channels[channel].users.add(numeric)
 
         # We send users and modes here because SJOIN and JOIN both use one hook,
         # for simplicity's sake (with plugins).
         return {'channel': channel, 'users': [numeric], 'modes':
-                self.irc.channels[channel].modes, 'ts': ts}
+                self._channels[channel].modes, 'ts': ts}
 
     def handle_euid(self, numeric, command, args):
         """Handles incoming EUID commands (user introduction)."""
-        # <- :42X EUID GL 1 1437505322 +ailoswz ~gl 127.0.0.1 127.0.0.1 42XAAAAAB * * :realname
+        # <- :42X EUID jlu5 1 1437505322 +ailoswz ~jlu5 127.0.0.1 127.0.0.1 42XAAAAAB * * :realname
         nick = args[0]
+        self._check_nick_collision(nick)
         ts, modes, ident, host, ip, uid, realhost, accountname, realname = args[2:11]
+        ts = int(ts)
         if realhost == '*':
-            realhost = None
+            realhost = host
 
         log.debug('(%s) handle_euid got args: nick=%s ts=%s uid=%s ident=%s '
-                  'host=%s realname=%s realhost=%s ip=%s', self.irc.name, nick, ts, uid,
+                  'host=%s realname=%s realhost=%s ip=%s', self.name, nick, ts, uid,
                   ident, host, realname, realhost, ip)
         assert ts != 0, "Bad TS 0 for user %s" % uid
 
         if ip == '0':  # IP was invalid; something used for services.
             ip = '0.0.0.0'
 
-        self.irc.users[uid] = IrcUser(nick, ts, uid, ident, host, realname, realhost, ip)
+        self.users[uid] = User(self, nick, ts, uid, numeric, ident, host, realname, realhost, ip)
 
-        parsedmodes = self.irc.parseModes(uid, [modes])
+        parsedmodes = self.parse_modes(uid, [modes])
         log.debug('Applying modes %s for %s', parsedmodes, uid)
-        self.irc.applyModes(uid, parsedmodes)
+        self.apply_modes(uid, parsedmodes)
-        self.irc.servers[numeric].users.add(uid)
+        self.servers[numeric].users.add(uid)
 
         # Call the OPERED UP hook if +o is being added to the mode list.
-        if ('+o', None) in parsedmodes:
+        self._check_oper_status_change(uid, parsedmodes)
-            otype = 'Server Administrator' if ('+a', None) in parsedmodes else 'IRC Operator'
-            self.irc.callHooks([uid, 'CLIENT_OPERED', {'text': otype}])
 
         # Set the accountname if present
         if accountname != "*":
-            self.irc.callHooks([uid, 'CLIENT_SERVICES_LOGIN', {'text': accountname}])
+            self.call_hooks([uid, 'CLIENT_SERVICES_LOGIN', {'text': accountname}])
 
-        return {'uid': uid, 'ts': ts, 'nick': nick, 'realhost': realhost, 'host': host, 'ident': ident, 'ip': ip}
+        # charybdis and derivatives have a usermode (+Z) to mark SSL connections
+        # ratbox doesn't appear to have this
+        has_ssl = self.users[uid].ssl = ('+%s' % self.umodes.get('ssl'), None) in parsedmodes
+
+        return {'uid': uid, 'ts': ts, 'nick': nick, 'realhost': realhost, 'host': host, 'ident': ident, 'ip': ip, 'secure': has_ssl}
 
     def handle_uid(self, numeric, command, args):
-        raise ProtocolError("Servers should use EUID instead of UID to send users! "
+        """Handles legacy user introductions (UID)."""
-                            "This IS a required capability after all...")
+        # tl;dr We want to convert the following UID parameters:
+        #    nickname, hopcount, nickTS, umodes, username, visible hostname, IP address, UID, gecos
+        # to EUID parameters when parsing:
+        #    nickname, hopcount, nickTS, umodes, username, visible hostname, IP address, UID,
+        #    real hostname, account name, gecos
 
-    def handle_sid(self, numeric, command, args):
+        euid_args = args[:]
-        """Handles incoming server introductions."""
+
-        # parameters: server name, hopcount, sid, server description
+        # Insert a * to denote that the user is not logged in.
-        servername = args[0].lower()
+        euid_args.insert(8, '*')
-        sid = args[2]
+
-        sdesc = args[-1]
+        # Copy the visible hostname to the real hostname, as this data isn't sent yet.
-        self.irc.servers[sid] = IrcServer(numeric, servername, desc=sdesc)
+        euid_args.insert(8, args[5])
-        return {'name': servername, 'sid': sid, 'text': sdesc}
+
+        return self.handle_euid(numeric, command, euid_args)
 
     def handle_server(self, numeric, command, args):
         """
         Handles 1) incoming legacy (no SID) server introductions,
         2) Sending server data in initial connection.
         """
-        if numeric == self.irc.uplink and not self.irc.servers[numeric].name:
+        if numeric == self.uplink and not self.servers[numeric].name:
             # <- SERVER charybdis.midnight.vpn 1 :charybdis test server
             sname = args[0].lower()
 
-            log.debug('(%s) Found uplink server name as %r', self.irc.name, sname)
+            log.debug('(%s) Found uplink server name as %r', self.name, sname)
-            self.irc.servers[numeric].name = sname
+            self.servers[numeric].name = sname
-            self.irc.servers[numeric].desc = args[-1]
+            self.servers[numeric].desc = args[-1]
 
             # According to the TS6 protocol documentation, we should send SVINFO
             # when we get our uplink's SERVER command.
-            self.irc.send('SVINFO 6 6 0 :%s' % int(time.time()))
+            self.send('SVINFO 6 6 0 :%s' % int(time.time()))
 
             return
 
-        # <- :services.int SERVER a.bc 2 :(H) [GL] a
+        # <- :services.int SERVER a.bc 2 :(H) [jlu5] a
-        servername = args[0].lower()
+        return super().handle_server(numeric, command, args)
-        sdesc = args[-1]
-        self.irc.servers[servername] = IrcServer(numeric, servername, desc=sdesc)
-        return {'name': servername, 'sid': None, 'text': sdesc}
 
     def handle_tmode(self, numeric, command, args):
         """Handles incoming TMODE commands (channel mode change)."""
         # <- :42XAAAAAB TMODE 1437450768 #test -c+lkC 3 agte4
         # <- :0UYAAAAAD TMODE 0 #a +h 0UYAAAAAD
-        channel = self.irc.toLower(args[1])
+        channel = args[1]
-        oldobj = self.irc.channels[channel].deepcopy()
+        oldobj = self._channels[channel].deepcopy()
         modes = args[2:]
-        changedmodes = self.irc.parseModes(channel, modes)
+        changedmodes = self.parse_modes(channel, modes)
-        self.irc.applyModes(channel, changedmodes)
+        self.apply_modes(channel, changedmodes)
         ts = int(args[0])
         return {'target': channel, 'modes': changedmodes, 'ts': ts,
-                'oldchan': oldobj}
+                'channeldata': oldobj}
-
-    def handle_mode(self, numeric, command, args):
-        """Handles incoming user mode changes."""
-        # <- :70MAAAAAA MODE 70MAAAAAA -i+xc
-        target = args[0]
-        modestrings = args[1:]
-        changedmodes = self.irc.parseModes(target, modestrings)
-        self.irc.applyModes(target, changedmodes)
-        # Call the OPERED UP hook if +o is being set.
-        if ('+o', None) in changedmodes:
-            otype = 'Server Administrator' if ('a', None) in self.irc.users[target].modes else 'IRC Operator'
-            self.irc.callHooks([target, 'CLIENT_OPERED', {'text': otype}])
-        return {'target': target, 'modes': changedmodes}
 
     def handle_tb(self, numeric, command, args):
         """Handles incoming topic burst (TB) commands."""
-        # <- :42X TB #chat 1467427448 GL!~gl@127.0.0.1 :test
+        # <- :42X TB #chat 1467427448 jlu5!~jlu5@127.0.0.1 :test
-        channel = self.irc.toLower(args[0])
+        channel = args[0]
         ts = args[1]
         setter = args[2]
         topic = args[-1]
-        self.irc.channels[channel].topic = topic
+        self._channels[channel].topic = topic
-        self.irc.channels[channel].topicset = True
+        self._channels[channel].topicset = True
         return {'channel': channel, 'setter': setter, 'ts': ts, 'text': topic}
 
     def handle_etb(self, numeric, command, args):
         """Handles extended topic burst (ETB)."""
-        # <- :00AAAAAAC ETB 0 #test 1470021157 GL :test | abcd
+        # <- :00AAAAAAC ETB 0 #test 1470021157 jlu5 :test | abcd
         # Same as TB, with extra TS and extensions arguments.
-        channel = self.irc.toLower(args[1])
+        channel = args[1]
         ts = args[2]
         setter = args[3]
         topic = args[-1]
-        self.irc.channels[channel].topic = topic
+        self._channels[channel].topic = topic
-        self.irc.channels[channel].topicset = True
+        self._channels[channel].topicset = True
         return {'channel': channel, 'setter': setter, 'ts': ts, 'text': topic}
 
-    def handle_invite(self, numeric, command, args):
-        """Handles incoming INVITEs."""
-        # <- :70MAAAAAC INVITE 0ALAAAAAA #blah 12345
-        target = args[0]
-        channel = self.irc.toLower(args[1])
-        try:
-            ts = args[3]
-        except IndexError:
-            ts = int(time.time())
-        # We don't actually need to process this; it's just something plugins/hooks can use
-        return {'target': target, 'channel': channel, 'ts': ts}
-
     def handle_chghost(self, numeric, command, args):
         """Handles incoming CHGHOST commands."""
-        target = args[0]
+        target = self._get_UID(args[0])
-        self.irc.users[target].host = newhost = args[1]
+        self.users[target].host = newhost = args[1]
         return {'target': target, 'newhost': newhost}
 
     def handle_bmask(self, numeric, command, args):
         """Handles incoming BMASK commands (ban propagation on burst)."""
         # <- :42X BMASK 1424222769 #dev b :*!test@*.isp.net *!badident@*
         # This is used for propagating bans, not TMODE!
-        channel = self.irc.toLower(args[1])
+        channel = args[1]
         mode = args[2]
         ts = int(args[0])
         modes = []
         for ban in args[-1].split():
             modes.append(('+%s' % mode, ban))
-        self.irc.applyModes(channel, modes)
+        self.apply_modes(channel, modes)
         return {'target': channel, 'modes': modes, 'ts': ts}
 
     def handle_472(self, numeric, command, args):
@@ -653,7 +723,7 @@ class TS6Protocol(TS6BaseProtocol):
         the administrator that certain extensions should be loaded for the best
         compatibility.
         """
-        # <- :charybdis.midnight.vpn 472 GL|devel O :is an unknown mode char to me
+        # <- :charybdis.midnight.vpn 472 jlu5|devel O :is an unknown mode char to me
         badmode = args[1]
         reason = args[-1]
         setter = args[0]
@@ -663,14 +733,14 @@ class TS6Protocol(TS6BaseProtocol):
             log.warning('(%s) User %r attempted to set channel mode %r, but the '
                         'extension providing it isn\'t loaded! To prevent possible'
                         ' desyncs, try adding the line "loadmodule "extensions/%s.so";" to '
-                        'your IRCd configuration.', self.irc.name, setter, badmode,
+                        'your IRCd configuration.', self.name, setter, badmode,
                         charlist[badmode])
 
     def handle_su(self, numeric, command, args):
         """
         Handles SU, which is used for setting login information.
         """
-        # <- :00A ENCAP * SU 42XAAAAAC :GLolol
+        # <- :00A ENCAP * SU 42XAAAAAC :jlu5
         # <- :00A ENCAP * SU 42XAAAAAC
         try:
             account = args[1]  # Account name is being set
@@ -678,7 +748,7 @@ class TS6Protocol(TS6BaseProtocol):
             account = ''  # No account name means a logout
 
         uid = args[0]
-        self.irc.callHooks([uid, 'CLIENT_SERVICES_LOGIN', {'text': account}])
+        self.call_hooks([uid, 'CLIENT_SERVICES_LOGIN', {'text': account}])
 
     def handle_rsfnc(self, numeric, command, args):
         """
@@ -687,4 +757,14 @@ class TS6Protocol(TS6BaseProtocol):
         # <- :00A ENCAP somenet.relay RSFNC 801AAAAAB Guest75038 1468299643 :1468299675
         return {'target': args[0], 'newnick': args[1]}
 
+    def handle_realhost(self, uid, command, args):
+        """Handles real host propagation."""
+        log.debug('(%s) Got REALHOST %s for %s', self.name, args[0], uid)
+        self.users[uid].realhost = args[0]
+
+    def handle_login(self, uid, command, args):
+        """Handles login propagation on burst."""
+        self.users[uid].services_account = args[0]
+        return {'text': args[0]}
+
 Class = TS6Protocol

protocols/ts6_common.py

@@ -5,11 +5,14 @@ ts6_common.py: Common base protocol class with functions shared by the UnrealIRC
 import string
 import time
 
-from pylinkirc import utils, structures
+from pylinkirc import conf, structures
 from pylinkirc.classes import *
 from pylinkirc.log import log
 from pylinkirc.protocols.ircs2s_common import *
 
+__all__ = ['TS6BaseProtocol']
+
+
 class TS6SIDGenerator():
     """
     TS6 SID Generator. <query> is a 3 character string with any combination of
@@ -56,7 +59,6 @@ class TS6SIDGenerator():
                 self.output[idx] = self.allowedchars[idx][0]
                 next(self.iters[idx])
 
-
     def increment(self, pos=2):
         """
         Increments the SID generator to the next available SID.
@@ -85,79 +87,39 @@ class TS6SIDGenerator():
         sid = ''.join(self.output)
         return sid
 
-class TS6UIDGenerator(utils.IncrementalUIDGenerator):
+class TS6UIDGenerator(UIDGenerator):
      """Implements an incremental TS6 UID Generator."""
 
      def __init__(self, sid):
-         # Define the options for IncrementalUIDGenerator, and then
-         # initialize its functions.
          # TS6 UIDs are 6 characters in length (9 including the SID).
          # They go from ABCDEFGHIJKLMNOPQRSTUVWXYZ -> 0123456789 -> wrap around:
          # e.g. AAAAAA, AAAAAB ..., AAAAA8, AAAAA9, AAAABA, etc.
-         self.allowedchars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456879'
+         uidchars = string.ascii_uppercase + string.digits
-         self.length = 6
+         length = 6
-         super().__init__(sid)
+         super().__init__(uidchars, length, sid)
 
 class TS6BaseProtocol(IRCS2SProtocol):
-
+    def __init__(self, *args, **kwargs):
-    def __init__(self, irc):
+        super().__init__(*args, **kwargs)
-        super().__init__(irc)
 
         # Dictionary of UID generators (one for each server).
         self.uidgen = structures.KeyedDefaultdict(TS6UIDGenerator)
 
         # SID generator for TS6.
-        self.sidgen = TS6SIDGenerator(irc)
+        self.sidgen = TS6SIDGenerator(self)
 
-    def _send(self, source, msg):
+        # Most TS6 variations (unreal, inspircd, charybdis) support this. For
-        """Sends a TS6-style raw command from a source numeric to the self.irc connection given."""
+        # pure TS6, we also require the CHW capability which explicitly declares
-        self.irc.send(':%s %s' % (source, msg))
+        # support.
-
+        self.protocol_caps |= {'has-statusmsg'}
-    def _expandPUID(self, uid):
-        """
-        Returns the outgoing nick for the given UID. In the base ts6_common implementation,
-        this does nothing, but other modules subclassing this can override it.
-        For example, this can be used to turn PUIDs (used to store legacy, UID-less users)
-        to actual nicks in outgoing messages, so that a remote IRCd can understand it.
-        """
-        return uid
 
     ### OUTGOING COMMANDS
 
-    def numeric(self, source, numeric, target, text):
-        """Sends raw numerics from a server to a remote client, used for WHOIS
-        replies."""
-        # Mangle the target for IRCds that require it.
-        target = self._expandPUID(target)
-
-        self._send(source, '%s %s %s' % (numeric, target, text))
-
-    def kick(self, numeric, channel, target, reason=None):
-        """Sends kicks from a PyLink client/server."""
-
-        if (not self.irc.isInternalClient(numeric)) and \
-                (not self.irc.isInternalServer(numeric)):
-            raise LookupError('No such PyLink client/server exists.')
-
-        channel = self.irc.toLower(channel)
-        if not reason:
-            reason = 'No reason given'
-
-        # Mangle kick targets for IRCds that require it.
-        target = self._expandPUID(target)
-
-        self._send(numeric, 'KICK %s %s :%s' % (channel, target, reason))
-
-        # We can pretend the target left by its own will; all we really care about
-        # is that the target gets removed from the channel userlist, and calling
-        # handle_part() does that just fine.
-        self.handle_part(target, 'KICK', [channel])
-
     def kill(self, numeric, target, reason):
         """Sends a kill from a PyLink client/server."""
 
-        if (not self.irc.isInternalClient(numeric)) and \
+        if (not self.is_internal_client(numeric)) and \
-                (not self.irc.isInternalServer(numeric)):
+                (not self.is_internal_server(numeric)):
             raise LookupError('No such PyLink client/server exists.')
 
         # From TS6 docs:
@@ -168,220 +130,98 @@ class TS6BaseProtocol(IRCS2SProtocol):
         # the kill followed by a space and a parenthesized reason. To avoid overflow,
         # it is recommended not to add anything to the path.
 
-        assert target in self.irc.users, "Unknown target %r for kill()!" % target
+        assert target in self.users, "Unknown target %r for kill()!" % target
 
-        if numeric in self.irc.users:
+        if numeric in self.users:
             # Killer was an user. Follow examples of setting the path to be "killer.host!killer.nick".
-            userobj = self.irc.users[numeric]
+            userobj = self.users[numeric]
             killpath = '%s!%s' % (userobj.host, userobj.nick)
-        elif numeric in self.irc.servers:
+        elif numeric in self.servers:
             # Sender was a server; killpath is just its name.
-            killpath = self.irc.servers[numeric].name
+            killpath = self.servers[numeric].name
         else:
             # Invalid sender?! This shouldn't happen, but make the killpath our server name anyways.
             log.warning('(%s) Invalid sender %s for kill(); using our server name instead.',
-                        self.irc.name, numeric)
+                        self.name, numeric)
-            killpath = self.irc.servers[self.irc.sid].name
+            killpath = self.servers[self.sid].name
 
-        self._send(numeric, 'KILL %s :%s (%s)' % (target, killpath, reason))
+        self._send_with_prefix(numeric, 'KILL %s :%s (%s)' % (target, killpath, reason))
-        self.removeClient(target)
+        self._remove_client(target)
 
     def nick(self, numeric, newnick):
         """Changes the nick of a PyLink client."""
-        if not self.irc.isInternalClient(numeric):
+        if not self.is_internal_client(numeric):
             raise LookupError('No such PyLink client exists.')
 
-        self._send(numeric, 'NICK %s %s' % (newnick, int(time.time())))
+        self._send_with_prefix(numeric, 'NICK %s %s' % (newnick, int(time.time())))
 
-        self.irc.users[numeric].nick = newnick
+        self.users[numeric].nick = newnick
 
         # Update the NICK TS.
-        self.irc.users[numeric].ts = int(time.time())
+        self.users[numeric].ts = int(time.time())
 
-    def part(self, client, channel, reason=None):
+    def spawn_server(self, name, sid=None, uplink=None, desc=None):
-        """Sends a part from a PyLink client."""
-        channel = self.irc.toLower(channel)
-        if not self.irc.isInternalClient(client):
-            log.error('(%s) Error trying to part %r from %r (no such client exists)', self.irc.name, client, channel)
-            raise LookupError('No such PyLink client exists.')
-        msg = "PART %s" % channel
-        if reason:
-            msg += " :%s" % reason
-        self._send(client, msg)
-        self.handle_part(client, 'PART', [channel])
-
-    def quit(self, numeric, reason):
-        """Quits a PyLink client."""
-        if self.irc.isInternalClient(numeric):
-            self._send(numeric, "QUIT :%s" % reason)
-            self.removeClient(numeric)
-        else:
-            raise LookupError("No such PyLink client exists.")
-
-    def message(self, numeric, target, text):
-        """Sends a PRIVMSG from a PyLink client."""
-        if not self.irc.isInternalClient(numeric):
-            raise LookupError('No such PyLink client exists.')
-
-        # Mangle message targets for IRCds that require it.
-        target = self._expandPUID(target)
-
-        self._send(numeric, 'PRIVMSG %s :%s' % (target, text))
-
-    def notice(self, numeric, target, text):
-        """Sends a NOTICE from a PyLink client."""
-        if not self.irc.isInternalClient(numeric):
-            raise LookupError('No such PyLink client exists.')
-
-        # Mangle message targets for IRCds that require it.
-        target = self._expandPUID(target)
-
-        self._send(numeric, 'NOTICE %s :%s' % (target, text))
-
-    def topic(self, numeric, target, text):
-        """Sends a TOPIC change from a PyLink client."""
-        if not self.irc.isInternalClient(numeric):
-            raise LookupError('No such PyLink client exists.')
-        self._send(numeric, 'TOPIC %s :%s' % (target, text))
-        self.irc.channels[target].topic = text
-        self.irc.channels[target].topicset = True
-
-    def spawnServer(self, name, sid=None, uplink=None, desc=None, endburst_delay=0):
         """
         Spawns a server off a PyLink server. desc (server description)
         defaults to the one in the config. uplink defaults to the main PyLink
         server, and sid (the server ID) is automatically generated if not
         given.
-
-        Note: TS6 doesn't use a specific ENDBURST command, so the endburst_delay
-        option will be ignored if given.
         """
         # -> :0AL SID test.server 1 0XY :some silly pseudoserver
-        uplink = uplink or self.irc.sid
+        uplink = uplink or self.sid
         name = name.lower()
-        desc = desc or self.irc.serverdata.get('serverdesc') or self.irc.botdata['serverdesc']
+        desc = desc or self.serverdata.get('serverdesc') or conf.conf['pylink']['serverdesc']
+
         if sid is None:  # No sid given; generate one!
             sid = self.sidgen.next_sid()
         assert len(sid) == 3, "Incorrect SID length"
-        if sid in self.irc.servers:
+        if sid in self.servers:
             raise ValueError('A server with SID %r already exists!' % sid)
-        for server in self.irc.servers.values():
+        for server in self.servers.values():
             if name == server.name:
                 raise ValueError('A server named %r already exists!' % name)
-        if not self.irc.isInternalServer(uplink):
+        if not self.is_internal_server(uplink):
             raise ValueError('Server %r is not a PyLink server!' % uplink)
-        if not utils.isServerName(name):
+        if not self.is_server_name(name):
             raise ValueError('Invalid server name %r' % name)
-        self._send(uplink, 'SID %s 1 %s :%s' % (name, sid, desc))
-        self.irc.servers[sid] = IrcServer(uplink, name, internal=True, desc=desc)
-        return sid
 
-    def squit(self, source, target, text='No reason given'):
+        self.servers[sid] = Server(self, uplink, name, internal=True, desc=desc)
-        """SQUITs a PyLink server."""
+        self._send_with_prefix(uplink, 'SID %s %s %s :%s' % (name, self.servers[sid].hopcount, sid, desc))
-        # -> SQUIT 9PZ :blah, blah
+        return sid
-        log.debug('source=%s, target=%s', source, target)
-        self._send(source, 'SQUIT %s :%s' % (target, text))
-        self.handle_squit(source, 'SQUIT', [target, text])
 
     def away(self, source, text):
         """Sends an AWAY message from a PyLink client. <text> can be an empty string
         to unset AWAY status."""
         if text:
-            self._send(source, 'AWAY :%s' % text)
+            self._send_with_prefix(source, 'AWAY :%s' % text)
         else:
-            self._send(source, 'AWAY')
+            self._send_with_prefix(source, 'AWAY')
-        self.irc.users[source].away = text
+        self.users[source].away = text
 
     ### HANDLERS
 
-    def handle_events(self, data):
+    def handle_knock(self, numeric, command, args):
-        """Event handler for TS6 protocols.
+        """Handles channel KNOCKs."""
-
+        # InspIRCd:
-        This passes most commands to the various handle_ABCD() functions
+        # <- :70MAAAAAA ENCAP * KNOCK #blah :abcdefg
-        elsewhere defined protocol modules, coersing various sender prefixes
+        # Charybdis:
-        from nicks and server names to UIDs and SIDs respectively,
+        # <- :42XAAAAAC KNOCK #endlessvoid
-        whenever possible.
+        # UnrealIRCd propagates knocks as a channel notice to all ops, so this handler is not used there.
-
+        channel = args[0]
-        Commands sent without an explicit sender prefix will have them set to
-        the SID of the uplink server.
-        """
-        data = data.split(" ")
-        try:  # Message starts with a SID/UID prefix.
-            args = self.parsePrefixedArgs(data)
-            sender = args[0]
-            command = args[1]
-            args = args[2:]
-            # If the sender isn't in UID format, try to convert it automatically.
-            # Unreal's protocol, for example, isn't quite consistent with this yet!
-            sender_server = self._getSid(sender)
-            if sender_server in self.irc.servers:
-                # Sender is a server when converted from name to SID.
-                numeric = sender_server
-            else:
-                # Sender is a user.
-                numeric = self._getUid(sender)
-
-        # parsePrefixedArgs() will raise IndexError if the TS6 sender prefix is missing.
-        except IndexError:
-            # Raw command without an explicit sender; assume it's being sent by our uplink.
-            args = self.parseArgs(data)
-            numeric = self.irc.uplink
-            command = args[0]
-            args = args[1:]
-
-        if command == 'ENCAP':
-            # Special case for encapsulated commands (ENCAP), in forms like this:
-            # <- :00A ENCAP * SU 42XAAAAAC :GLolol
-            command = args[1]
-            args = args[2:]
-            log.debug("(%s) Rewriting incoming ENCAP to command %s (args: %s)", self.irc.name, command, args)
-
         try:
-            func = getattr(self, 'handle_'+command.lower())
+            text = args[1]
-        except AttributeError:  # unhandled command
+        except IndexError:
-            pass
+            text = ''
-        else:
+        return {'channel': channel, 'text': text}
-            parsed_args = func(numeric, command, args)
-            if parsed_args is not None:
-                return [numeric, command, parsed_args]
-
-    def handle_privmsg(self, source, command, args):
-        """Handles incoming PRIVMSG/NOTICE."""
-        # <- :70MAAAAAA PRIVMSG #dev :afasfsa
-        # <- :70MAAAAAA NOTICE 0ALAAAAAA :afasfsa
-        target = args[0]
-
-        # Coerse =#channel from Charybdis op moderated +z to @#channel.
-        if target.startswith('='):
-            target = '@' + target[1:]
-
-        # We use lowercase channels internally, but uppercase UIDs.
-        # Strip the target of leading prefix modes (for targets like @#channel)
-        # before checking whether it's actually a channel.
-        stripped_target = target.lstrip(''.join(self.irc.prefixmodes.values()))
-        if utils.isChannel(stripped_target):
-            target = self.irc.toLower(target)
-
-        return {'target': target, 'text': args[1]}
-
-    handle_notice = handle_privmsg
-
-    def handle_kick(self, source, command, args):
-        """Handles incoming KICKs."""
-        # :70MAAAAAA KICK #test 70MAAAAAA :some reason
-        channel = self.irc.toLower(args[0])
-        kicked = self._getUid(args[1])
-        self.handle_part(kicked, 'KICK', [channel, args[2]])
-        return {'channel': channel, 'target': kicked, 'text': args[2]}
 
     def handle_nick(self, numeric, command, args):
         """Handles incoming NICK changes."""
-        # <- :70MAAAAAA NICK GL-devel 1434744242
+        # <- :70MAAAAAA NICK jlu5-devel 1434744242
-        oldnick = self.irc.users[numeric].nick
+        oldnick = self.users[numeric].nick
-        newnick = self.irc.users[numeric].nick = args[0]
+        newnick = self.users[numeric].nick = args[0]
 
         # Update the nick TS.
-        self.irc.users[numeric].ts = ts = int(args[1])
+        self.users[numeric].ts = ts = int(args[1])
 
         return {'newnick': newnick, 'oldnick': oldnick, 'ts': ts}
 
@@ -396,47 +236,32 @@ class TS6BaseProtocol(IRCS2SProtocol):
         # -> :0AL000001 NICK Derp_ 1433728673
         # <- :70M SAVE 0AL000001 1433728673
         user = args[0]
-        oldnick = self.irc.users[user].nick
+        oldnick = self.users[user].nick
-        self.irc.users[user].nick = user
+        self.users[user].nick = user
 
         # TS6 SAVE sets nick TS to 100. This is hardcoded in InspIRCd and
         # charybdis.
-        self.irc.users[user].ts = 100
+        self.users[user].ts = 100
 
         return {'target': user, 'ts': 100, 'oldnick': oldnick}
 
-    def handle_topic(self, numeric, command, args):
+    def handle_server(self, numeric, command, args):
-        """Handles incoming TOPIC changes from clients. For topic bursts,
+        """Handles the SERVER command, used for introducing older (TS5) servers."""
-        TB (TS6/charybdis) and FTOPIC (InspIRCd) are used instead."""
+        # <- :services.int SERVER a.bc 2 :(H) [jlu5] test jupe
-        # <- :70MAAAAAA TOPIC #test :test
+        servername = args[0].lower()
-        channel = self.irc.toLower(args[0])
+        sdesc = args[-1]
-        topic = args[1]
+        self.servers[servername] = Server(self, numeric, servername, desc=sdesc)
+        return {'name': servername, 'sid': None, 'text': sdesc}
 
-        oldtopic = self.irc.channels[channel].topic
+    def handle_sid(self, numeric, command, args):
-        self.irc.channels[channel].topic = topic
+        """Handles the SID command, used for introducing remote servers by our uplink."""
-        self.irc.channels[channel].topicset = True
+        # <- SID services.int 2 00A :Shaltúre IRC Services
-
+        # parameters: server name, hopcount, sid, server description
-        return {'channel': channel, 'setter': numeric, 'text': topic,
+        sname = args[0].lower()
-                'oldtopic': oldtopic}
+        sid = args[2]
-
+        sdesc = args[-1]
-    def handle_part(self, source, command, args):
+        self.servers[sid] = Server(self, numeric, sname, desc=sdesc)
-        """Handles incoming PART commands."""
+        return {'name': sname, 'sid': sid, 'text': sdesc}
-        channels = self.irc.toLower(args[0]).split(',')
-        for channel in channels:
-            # We should only get PART commands for channels that exist, right??
-            self.irc.channels[channel].removeuser(source)
-            try:
-                self.irc.users[source].channels.discard(channel)
-            except KeyError:
-                log.debug("(%s) handle_part: KeyError trying to remove %r from %r's channel list?", self.irc.name, channel, source)
-            try:
-                reason = args[1]
-            except IndexError:
-                reason = ''
-            # Clear empty non-permanent channels.
-            if not (self.irc.channels[channel].users or ((self.irc.cmodes.get('permanent'), None) in self.irc.channels[channel].modes)):
-                del self.irc.channels[channel]
-        return {'channels': channels, 'text': reason}
 
     def handle_svsnick(self, source, command, args):
         """Handles SVSNICK (forced nickname change attempts)."""
@@ -445,5 +270,5 @@ class TS6BaseProtocol(IRCS2SProtocol):
         # This is rewritten to SVSNICK with args ['902AAAAAB', 'Guest53593', '1468299404']
 
         # UnrealIRCd:
-        # <- :services.midnight.vpn SVSNICK GL Guest87795 1468303726
+        # <- :services.midnight.vpn SVSNICK jlu5 Guest87795 1468303726
-        return {'target': self._getUid(args[0]), 'newnick': args[1]}
+        return {'target': self._get_UID(args[0]), 'newnick': args[1]}

pylink

@@ -1,71 +1,14 @@
 #!/usr/bin/env python3
-"""
-PyLink IRC Services launcher.
-"""
-
-import os
 import sys
+
 try:
-    from pylinkirc import world
+    from pylinkirc import launcher
 except ImportError:
-    sys.stderr.write("ERROR: Failed to import PyLink main module (pylinkirc.world).\n\nIf you are "
+    print("ERROR: Failed to import PyLink launcher module (pylinkirc.launcher).\n\nIf you are "
-                     "running PyLink from source, please install PyLink first using 'python3 "
+          "running PyLink from source, please install PyLink first using 'python3 "
-                     "setup.py install' (global install) or 'python3 setup.py install --user'"
+          "setup.py install' (global install) or 'python3 setup.py install --user'"
-                     " (local install)\n")
+          " (local install)\n")
     sys.exit(1)
-from pylinkirc import conf, __version__
+else:
-
+    if __name__ == '__main__':
-if __name__ == '__main__':
+        launcher.main()
-    import argparse
-
-    parser = argparse.ArgumentParser(description='Starts an instance of PyLink IRC Services.')
-    parser.add_argument('config', help='specifies the path to the config file (defaults to pylink.yml)', nargs='?', default='pylink.yml')
-    parser.add_argument("-v", "--version", help="displays the program version and exits", action='store_true')
-    parser.add_argument("-n", "--no-pid", help="skips generating PID files", action='store_true')
-    args = parser.parse_args()
-
-    if args.version:  # Display version and exit
-        print('PyLink ' + __version__)
-        sys.exit()
-
-    # Load the config
-    conf.loadConf(args.config)
-
-    from pylinkirc.log import log
-    from pylinkirc import classes, utils, coremods
-    log.info('PyLink %s starting...', __version__)
-
-    # Write a PID file unless specifically told not to.
-    if not args.no_pid:
-        with open('%s.pid' % conf.confname, 'w') as f:
-            f.write(str(os.getpid()))
-
-    # Import plugins first globally, because they can listen for events
-    # that happen before the connection phase.
-    to_load = conf.conf['plugins']
-    # Here, we override the module lookup and import the plugins
-    # dynamically depending on which were configured.
-    for plugin in to_load:
-        try:
-            world.plugins[plugin] = pl = utils.loadPlugin(plugin)
-        except (OSError, ImportError) as e:
-            log.exception('Failed to load plugin %r: %s: %s', plugin, type(e).__name__, str(e))
-        else:
-            if hasattr(pl, 'main'):
-                log.debug('Calling main() function of plugin %r', pl)
-                pl.main()
-
-    # Initialize all the networks one by one
-    for network, sdata in conf.conf['servers'].items():
-
-        try:
-            protoname = sdata['protocol']
-        except (KeyError, TypeError):
-            log.error("(%s) Configuration error: No protocol module specified, aborting.", network)
-        else:
-            # Fetch the correct protocol module
-            proto = utils.getProtocolModule(protoname)
-            world.networkobjects[network] = irc = classes.Irc(network, proto, conf.conf)
-
-    world.started.set()
-    log.info("Loaded plugins: %s", ', '.join(sorted(world.plugins.keys())))

pylink-mkpasswd

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+Password hashing utility for PyLink IRC Services.
+"""
+
+import getpass
+
+from pylinkirc.coremods.login import pwd_context
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Hashes a password for use with PyLink IRC Services.')
+    parser.add_argument('password', help='specifies the password to hash', nargs='?', default='')
+    args = parser.parse_args()
+
+    assert pwd_context, 'Cannot hash passwords because passlib is missing! Install it via "pip3 install passlib".'
+
+    password = args.password
+
+    # If no password was given, enter one on the command line
+    if not password:
+        password = getpass.getpass()
+
+    password = password.strip()
+    assert password, "Password cannot be empty!"
+    print(pwd_context.hash(password))

requirements-docker.txt

@@ -0,0 +1,4 @@
+cachetools
+passlib
+pyyaml
+setuptools

selectdriver.py

@@ -0,0 +1,54 @@
+"""
+Socket handling driver using the selectors module. epoll, kqueue, and devpoll
+are used internally when available.
+"""
+
+import selectors
+import threading
+
+from pylinkirc import world
+from pylinkirc.log import log
+
+__all__ = ['register', 'unregister', 'start']
+
+
+SELECT_TIMEOUT = 0.5
+
+selector = selectors.DefaultSelector()
+
+def _process_conns():
+    """Main loop which processes connected sockets."""
+
+    while not world.shutting_down.is_set():
+        for socketkey, mask in selector.select(timeout=SELECT_TIMEOUT):
+            irc = socketkey.data
+            try:
+                if mask & selectors.EVENT_READ and not irc._aborted.is_set():
+                    irc._run_irc()
+            except:
+                log.exception('Error in select driver loop:')
+                continue
+
+def register(irc):
+    """
+    Registers a network to the global selectors instance.
+    """
+    log.debug('selectdriver: registering %s for network %s', irc._socket, irc.name)
+    selector.register(irc._socket, selectors.EVENT_READ, data=irc)
+
+def unregister(irc):
+    """
+    Removes a network from the global selectors instance.
+    """
+    if irc._socket.fileno() != -1:
+        log.debug('selectdriver: de-registering %s for network %s', irc._socket, irc.name)
+        selector.unregister(irc._socket)
+    else:
+        log.debug('selectdriver: skipping de-registering %s for network %s', irc._socket, irc.name)
+
+def start():
+    """
+    Starts a thread to process connections.
+    """
+    t = threading.Thread(target=_process_conns, name="Selector driver loop")
+    t.start()

setup.py

@@ -1,26 +1,27 @@
 """Setup module for PyLink IRC Services."""
 
+import subprocess
 import sys
+from codecs import open
 
-if sys.version_info < (3, 4):
+if sys.version_info < (3, 7):
-    raise RuntimeError("PyLink requires Python 3.4 or higher.")
+    raise RuntimeError("PyLink requires Python 3.7 or higher.")
 
 try:
     from setuptools import setup, find_packages
 except ImportError:
     raise ImportError("Please install Setuptools and try again.")
-from codecs import open
-import subprocess
 
-# Get version from Git tags.
 with open('VERSION', encoding='utf-8') as f:
     version = f.read().strip()
 
+# Try to fetch the current commit hash from Git.
 try:
     real_version = subprocess.check_output(['git', 'describe', '--tags']).decode('utf-8').strip()
 except Exception as e:
-    print('ERROR: Failed to get Git version from "git describe --tags": %s: %s' % (type(e).__name__, e))
+    print('WARNING: Failed to get Git version from "git describe --tags": %s: %s' % (type(e).__name__, e))
-    real_version = version + '-dirty'
+    print("If you're installing from PyPI or a tarball, ignore the above message.")
+    real_version = version + '-nogit'
 
 # Write the version to disk.
 with open('__init__.py', 'w') as f:
@@ -28,12 +29,11 @@ with open('__init__.py', 'w') as f:
     f.write('__version__ = %r\n' % version)
     f.write('real_version = %r\n' % real_version)
 
-# Convert Markdown to RST for PyPI
 try:
-    import pypandoc
+    with open('README.md') as f:
-    long_description = pypandoc.convert('README.md', 'rst')
+        long_description = f.read()
-except ImportError:
+except OSError:
-    print('WARNING: PyPandoc not available; skipping writing long description.')
+    print('WARNING: Failed to read readme, skipping writing long_description')
     long_description = None
 
 setup(
@@ -42,19 +42,20 @@ setup(
 
     description='PyLink IRC Services',
     long_description=long_description,
+    long_description_content_type='text/markdown',
 
-    url='https://github.com/GLolol/PyLink',
+    url='https://github.com/jlu5/PyLink',
 
     # Author details
     author='James Lu',
-    author_email='GLolol@overdrivenetworks.com',
+    author_email='james@overdrivenetworks.com',
 
-    # Choose your license
+    # License
     license='MPL 2.0',
 
     # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
 
         'Intended Audience :: Developers',
         'Intended Audience :: System Administrators',
@@ -69,23 +70,35 @@ setup(
         'Natural Language :: English',
 
         'Programming Language :: Python :: 3 :: Only',
-        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ],
 
     keywords='IRC services relay',
-    install_requires=['pyyaml', 'ircmatch'],
+    install_requires=['pyyaml', 'cachetools'],
+
+    extras_require={
+        'password-hashing': ['passlib>=1.7.0'],
+        'cron-support': ['psutil'],
+        'relay-unicode': ['unidecode'],
+    },
 
     # Folders (packages of code)
     packages=['pylinkirc', 'pylinkirc.protocols', 'pylinkirc.plugins', 'pylinkirc.coremods'],
 
     # Data files
     package_data={
-        '': ['example-conf.yml'],
+        '': ['example-conf.yml', 'VERSION', 'README.md'],
     },
 
     package_dir = {'pylinkirc': '.'},
 
     # Executable scripts
-    scripts=["pylink"],
+    scripts=["pylink-mkpasswd"],
+
+    entry_points = {
+        'console_scripts': ['pylink=pylinkirc.launcher:main'],
+    }
 )

structures.py

@@ -5,7 +5,25 @@ This module contains custom data structures that may be useful in various situat
 """
 
 import collections
+import collections.abc
 import json
+import os
+import pickle
+import string
+import threading
+from copy import copy, deepcopy
+
+from . import conf
+from .log import log
+
+__all__ = ['KeyedDefaultdict', 'CopyWrapper', 'CaseInsensitiveFixedSet',
+          'CaseInsensitiveDict', 'IRCCaseInsensitiveDict',
+          'CaseInsensitiveSet', 'IRCCaseInsensitiveSet',
+          'CamelCaseToSnakeCase', 'DataStore', 'JSONDataStore',
+          'PickleDataStore']
+
+
+_BLACKLISTED_COPY_TYPES = []
 
 class KeyedDefaultdict(collections.defaultdict):
     """
@@ -14,112 +32,283 @@ class KeyedDefaultdict(collections.defaultdict):
     def __missing__(self, key):
         if self.default_factory is None:
             # If there is no default factory, just let defaultdict handle it
-            super().__missing__(self, key)
+            super().__missing__(key)
         else:
             value = self[key] = self.default_factory(key)
             return value
 
-class DataStore:
+class CopyWrapper():
-    # will come into play with subclassing and db version upgrading
+    """
-    initial_version = 1
+    Base container class implementing copy methods.
+    """
 
-    def __init__(self, name, filename, db_format='json', save_frequency={'seconds': 30}):
+    def copy(self):
-        self.name = name
+        """Returns a shallow copy of this object instance."""
+        return copy(self)
 
-        self._filename = os.path.abspath(os.path.expanduser(filename))
+    def __deepcopy__(self, memo):
-        self._tmp_filename = self._filename + '.tmp'
+        """Returns a deep copy of the channel object."""
-        log.debug('(db:{}) database path set to {}'.format(self.name, self._filename))
+        newobj = copy(self)
+        #log.debug('CopyWrapper: _BLACKLISTED_COPY_TYPES = %s', _BLACKLISTED_COPY_TYPES)
+        for attr, val in self.__dict__.items():
+            # We can't pickle IRCNetwork, so just return a reference of it.
+            if not isinstance(val, tuple(_BLACKLISTED_COPY_TYPES)):
+                #log.debug('CopyWrapper: copying attr %r', attr)
+                setattr(newobj, attr, deepcopy(val))
 
-        self._format = db_format
+        memo[id(self)] = newobj
-        log.debug('(db:{}) format set to {}'.format(self.name, self._format))
 
-        self._save_frequency = timedelta(**save_frequency).total_seconds()
+        return newobj
-        log.debug('(db:{}) saving every {} seconds'.format(self.name, self._save_frequency))
 
-    def create_or_load(self):
+    def deepcopy(self):
-        log.debug('(db:{}) creating/loading datastore using {}'.format(self.name, self._format))
+        """Returns a deep copy of this object instance."""
+        return deepcopy(self)
 
-        if self._format == 'json':
+class CaseInsensitiveFixedSet(collections.abc.Set, CopyWrapper):
-            self._store = {}
+    """
-            self._store_lock = threading.Lock()
+    Implements a fixed set storing items case-insensitively.
+    """
 
-            log.debug('(db:{}) loading json data store from {}'.format(self.name, self._filename))
+    def __init__(self, *, data=None):
-            try:
+        if data is not None:
-                self._store = json.loads(open(self._filename, 'r').read())
+            self._data = data
-            except (ValueError, IOError, FileNotFoundError):
-                log.exception('(db:{}) failed to load existing db, creating new one in memory'.format(self.name))
-                self.put('db.version', self.initial_version)
         else:
-            raise Exception('(db:{}) Data store format [{}] not recognised'.format(self.name, self._format))
+            self._data = set()
+
+    @staticmethod
+    def _keymangle(key):
+        """Converts the given key to lowercase."""
+        if isinstance(key, str):
+            return key.lower()
+        return key
+
+    @classmethod
+    def _from_iterable(cls, it):
+        """Returns a new iterable instance given the data in 'it'."""
+        return cls(data=it)
+
+    def __repr__(self):
+        return "%s(%s)" % (self.__class__.__name__, self._data)
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __len__(self):
+        return len(self._data)
+
+    def __contains__(self, key):
+        return self._data.__contains__(self._keymangle(key))
+
+    def __copy__(self):
+        return self.__class__(data=self._data.copy())
+
+class CaseInsensitiveDict(collections.abc.MutableMapping, CaseInsensitiveFixedSet):
+    """
+    A dictionary storing items case insensitively.
+    """
+    def __init__(self, *, data=None):
+        if data is not None:
+            self._data = data
+        else:
+            self._data = {}
+
+    def __getitem__(self, key):
+        key = self._keymangle(key)
+
+        return self._data[key]
+
+    def __setitem__(self, key, value):
+        self._data[self._keymangle(key)] = value
+
+    def __delitem__(self, key):
+        del self._data[self._keymangle(key)]
+
+class IRCCaseInsensitiveDict(CaseInsensitiveDict):
+    """
+    A dictionary storing items case insensitively, using IRC case mappings.
+    """
+    def __init__(self, irc, *, data=None):
+        super().__init__(data=data)
+        self._irc = irc
+
+    def _keymangle(self, key):
+        """Converts the given key to lowercase."""
+        if isinstance(key, str):
+            return self._irc.to_lower(key)
+        return key
+
+    def _from_iterable(self, it):
+        """Returns a new iterable instance given the data in 'it'."""
+        return self.__class__(self._irc, data=it)
+
+    def __copy__(self):
+        return self.__class__(self._irc, data=self._data.copy())
+
+class CaseInsensitiveSet(collections.abc.MutableSet, CaseInsensitiveFixedSet):
+    """
+    A mutable set storing items case insensitively.
+    """
+
+    def add(self, key):
+        self._data.add(self._keymangle(key))
+
+    def discard(self, key):
+        self._data.discard(self._keymangle(key))
+
+class IRCCaseInsensitiveSet(CaseInsensitiveSet):
+    """
+    A set storing items case insensitively, using IRC case mappings.
+    """
+    def __init__(self, irc, *, data=None):
+        super().__init__(data=data)
+        self._irc = irc
+
+    def _keymangle(self, key):
+        """Converts the given key to lowercase."""
+        if isinstance(key, str):
+            return self._irc.to_lower(key)
+        return key
+
+    def _from_iterable(self, it):
+        """Returns a new iterable instance given the data in 'it'."""
+        return self.__class__(self._irc, data=it)
+
+    def __copy__(self):
+        return self.__class__(self._irc, data=self._data.copy())
+
+class CamelCaseToSnakeCase():
+    """
+    Class which automatically converts missing attributes from camel case to snake case.
+    """
+
+    def __getattr__(self, attr):
+        """
+        Attribute fetching fallback function which normalizes camel case attributes to snake case.
+        """
+        assert isinstance(attr, str), "Requested attribute %r is not a string!" % attr
+
+        normalized_attr = ''  # Start off with the first letter, which is ignored when processing
+        for char in attr:
+            if char in string.ascii_uppercase:
+                char = '_' + char.lower()
+            normalized_attr += char
+
+        classname = self.__class__.__name__
+        if normalized_attr == attr:
+            # __getattr__ only fires if normal attribute fetching fails, so we can assume that
+            # the attribute was tried already and failed.
+            raise AttributeError('%s object has no attribute with normalized name %r' % (classname, attr))
+
+        target = getattr(self, normalized_attr)
+        log.warning('%s.%s is deprecated, considering migrating to %s.%s!', classname, attr, classname, normalized_attr)
+        return target
+
+class DataStore:
+    """
+    Generic database class. Plugins should use a subclass of this such as JSONDataStore or
+    PickleDataStore.
+    """
+    def __init__(self, name, filename, save_frequency=None, default_db=None, data_dir=None):
+        if data_dir is None:
+            data_dir = conf.conf['pylink'].get('data_dir', '')
+
+        filename = os.path.join(data_dir, filename)
+
+        self.name = name
+        self.filename = filename
+        self.tmp_filename = filename + '.tmp'
+
+        log.debug('(DataStore:%s) using implementation %s', self.name, self.__class__.__name__)
+        log.debug('(DataStore:%s) database path set to %s', self.name, self.filename)
+
+        self.save_frequency = save_frequency or conf.conf['pylink'].get('save_delay', 300)
+        log.debug('(DataStore:%s) saving every %s seconds', self.name, self.save_frequency)
+
+        if default_db is not None:
+            self.store = default_db
+        else:
+            self.store = {}
+        self.store_lock = threading.Lock()
+        self.exportdb_timer = None
+
+        self.load()
+
+        if self.save_frequency > 0:
+            # If autosaving is enabled, start the save_callback loop.
+            self.save_callback(starting=True)
+
+    def load(self):
+        """
+        DataStore load stub. Database implementations should subclass DataStore
+        and implement this.
+        """
+        raise NotImplementedError
 
     def save_callback(self, starting=False):
         """Start the DB save loop."""
-        if self._format == 'json':
+        # don't actually save the first time
-            # don't actually save the first time
+        if not starting:
-            if not starting:
+            self.save()
-                self.save()
 
-            # schedule saving in a loop.
+        # schedule saving in a loop.
-            self.exportdb_timer = threading.Timer(self._save_frequency, self.save_callback)
+        self.exportdb_timer = threading.Timer(self.save_frequency, self.save_callback)
-            self.exportdb_timer.name = 'PyLink {} save_callback Loop'.format(self.name)
+        self.exportdb_timer.name = 'DataStore {} save_callback loop'.format(self.name)
-            self.exportdb_timer.start()
+        self.exportdb_timer.start()
-        else:
-            raise Exception('(db:{}) Data store format [{}] not recognised'.format(self.name, self._format))
 
     def save(self):
-        log.debug('(db:{}) saving datastore'.format(self.name))
+        """
-        if self._format == 'json':
+        DataStore save stub. Database implementations should subclass DataStore
-            with open(self._tmp_filename, 'w') as store_file:
+        and implement this.
-                store_file.write(json.dumps(self._store))
+        """
-            os.rename(self._tmp_filename, self._filename)
+        raise NotImplementedError
 
-    # single keys
+    def die(self):
-    def __contains__(self, key):
+        """
-        if self._format == 'json':
+        Saves the database and stops any save loops.
-            return key in self._store
+        """
+        if self.exportdb_timer:
+            self.exportdb_timer.cancel()
 
-    def get(self, key, default=None):
+        self.save()
-        if self._format == 'json':
-            return self._store.get(key, default)
 
-    def put(self, key, value):
+class JSONDataStore(DataStore):
-        if self._format == 'json':
+    def load(self):
-            # make sure we can serialize the given data
+        """Loads the database given via JSON."""
-            # so we don't choke later on saving the db out
+        with self.store_lock:
-            json.dumps(value)
-
-            self._store[key] = value
-
-            return True
-
-    def delete(self, key):
-        if self._format == 'json':
             try:
-                with self._store_lock:
+                with open(self.filename, "r") as f:
-                    del self._store[key]
+                    self.store.clear()
-            except KeyError:
+                    self.store.update(json.load(f))
-                # key is already gone, nothing to do
+            except (ValueError, IOError, OSError):
-                ...
+                log.info("(DataStore:%s) failed to load database %s; creating a new one in "
+                         "memory", self.name, self.filename)
 
-            return True
+    def save(self):
+        """Saves the database given via JSON."""
+        with self.store_lock:
+            with open(self.tmp_filename, 'w') as f:
+                # Pretty print the JSON output for better readability.
+                json.dump(self.store, f, indent=4)
 
-    # multiple keys
+                os.rename(self.tmp_filename, self.filename)
-    def list_keys(self, prefix=None):
-        """Return all key names. If prefix given, return only keys that start with it."""
-        if self._format == 'json':
-            keys = []
 
-            with self._store_lock:
+class PickleDataStore(DataStore):
-                for key in self._store:
+    def load(self):
-                    if prefix is None or key.startswith(prefix):
+        """Loads the database given via pickle."""
-                        keys.append(key)
+        with self.store_lock:
+            try:
+                with open(self.filename, "rb") as f:
+                    self.store.clear()
+                    self.store.update(pickle.load(f))
+            except (ValueError, IOError, OSError):
+                log.info("(DataStore:%s) failed to load database %s; creating a new one in "
+                         "memory", self.name, self.filename)
 
-            return keys
+    def save(self):
+        """Saves the database given via pickle."""
+        with self.store_lock:
+            with open(self.tmp_filename, 'wb') as f:
+                # Force protocol version 4 as that is the lowest Python 3.4 supports.
+                pickle.dump(self.store, f, protocol=4)
 
-    def delete_keys(self, prefix):
+                os.rename(self.tmp_filename, self.filename)
-        """Delete all keys with the given prefix."""
-        if self._format == 'json':
-            with self._store_lock:
-                for key in tuple(self._store):
-                    if key.startswith(prefix):
-                        del self._store[key]

test/parser-tests

@@ -0,0 +1 @@
+Subproject commit 26c9dd841467b9746fb325e16292a0b7a93bc802

test/protocol_test_fixture.py

@@ -0,0 +1,977 @@
+"""
+A test fixture for PyLink protocol modules.
+"""
+import time
+import unittest
+import collections
+import itertools
+from unittest.mock import patch
+
+from pylinkirc import conf, world
+from pylinkirc.log import log
+from pylinkirc.classes import User, Server, Channel
+
+class DummySocket():
+    def __init__(self):
+        #self.recv_messages = collections.deque()
+        self.sent_messages = collections.deque()
+
+    @staticmethod
+    def connect(address):
+        return
+    '''
+    def recv(bufsize, *args):
+        if self.recv_messages:
+            data = self.recv_messages.popleft()
+            print('<-', data)
+            return data
+        else:
+            return None
+    '''
+
+    def recv(self, bufsize, *args):
+        raise NotImplementedError
+
+    def send(self, data):
+        print('->', data)
+        self.sent_messages.append(data)
+
+class BaseProtocolTest(unittest.TestCase):
+    proto_class = None
+    netname = 'test'
+    serverdata = conf.conf['servers'][netname]
+
+    def setUp(self):
+        if not self.proto_class:
+            raise RuntimeError("Must set target protocol module in proto_class")
+        self.p = self.proto_class(self.netname)
+
+        # Stub connect() and the socket for now...
+        self.p.connect = lambda self: None
+        self.p.socket = DummySocket()
+
+        if self.serverdata:
+            self.p.serverdata = self.serverdata
+
+    def _make_user(self, nick, uid, ts=None, sid=None, **kwargs):
+        """
+        Creates a user for testing.
+        """
+        if ts is None:
+            ts = int(time.time())
+        userobj = User(self.p, nick, ts, uid, sid, **kwargs)
+        self.p.users[uid] = userobj
+        return userobj
+
+    ### STATEKEEPING FUNCTIONS
+
+    def test_nick_to_uid(self):
+        self.assertEqual(self.p.nick_to_uid('TestUser'), None)
+
+        self._make_user('TestUser', 'testuid1')
+
+        self.assertEqual(self.p.nick_to_uid('TestUser'), 'testuid1')
+        self.assertEqual(self.p.nick_to_uid('TestUser', multi=True), ['testuid1'])
+        self.assertEqual(self.p.nick_to_uid('BestUser'), None)
+        self.assertEqual(self.p.nick_to_uid('RestUser', multi=True), [])
+
+        self._make_user('TestUser', 'testuid2')
+        self.assertEqual(self.p.nick_to_uid('TestUser', multi=True), ['testuid1', 'testuid2'])
+
+    def test_is_internal(self):
+        self.p.servers['internalserver'] = Server(self.p, None, 'internal.server', internal=True)
+        self.p.sid = 'internalserver'
+        self.p.servers['externalserver'] = Server(self.p, None, 'external.server', internal=False)
+
+        iuser = self._make_user('someone', 'uid1', sid='internalserver')
+        euser = self._make_user('sometwo', 'uid2', sid='externalserver')
+
+        self.assertTrue(self.p.is_internal_server('internalserver'))
+        self.assertFalse(self.p.is_internal_server('externalserver'))
+        self.assertTrue(self.p.is_internal_client('uid1'))
+        self.assertFalse(self.p.is_internal_client('uid2'))
+
+    def test_is_manipulatable(self):
+        self.p.servers['serv1'] = Server(self.p, None, 'myserv.local', internal=True)
+        iuser = self._make_user('yes', 'uid1', sid='serv1', manipulatable=True)
+        euser = self._make_user('no', 'uid2', manipulatable=False)
+
+        self.assertTrue(self.p.is_manipulatable_client('uid1'))
+        self.assertFalse(self.p.is_manipulatable_client('uid2'))
+
+    def test_get_service_bot(self):
+        self.assertFalse(self.p.get_service_bot('nonexistent'))
+
+        regular_user = self._make_user('Guest12345', 'Guest12345@1')
+        service_user = self._make_user('MyServ', 'MyServ@2')
+        service_user.service = 'myserv'
+
+        self.assertFalse(self.p.get_service_bot('Guest12345@1'))
+
+        with patch.dict(world.services, {'myserv': 'myserv instance'}, clear=True):
+            self.assertEqual(self.p.get_service_bot('MyServ@2'), 'myserv instance')
+
+    def test_to_lower(self):
+        check = lambda inp, expected: self.assertEqual(self.p.to_lower(inp), expected)
+        check_unchanged = lambda inp: self.assertEqual(self.p.to_lower(inp), inp)
+
+        check('BLAH!', 'blah!')
+        check('BLAH!', 'blah!')  # since we memoize
+        check_unchanged('zabcdefghijklmnopqrstuvwxy')
+        check('123Xyz !@#$%^&*()-=+', '123xyz !@#$%^&*()-=+')
+
+        if self.p.casemapping == 'rfc1459':
+            check('hello [] {} |\\ ~^', 'hello [] [] \\\\ ^^')
+            check('{Test Case}', '[test case]')
+        else:
+            check_unchanged('hello [] {} |\\ ~^')
+            check('{Test Case}', '{test case}')
+
+    def test_is_nick(self):
+        assertT = lambda inp: self.assertTrue(self.p.is_nick(inp))
+        assertF = lambda inp: self.assertFalse(self.p.is_nick(inp))
+
+        assertT('test')
+        assertT('PyLink')
+        assertT('[bracketman]')
+        assertT('{RACKETman}')
+        assertT('bar|tender')
+        assertT('\\bar|bender\\')
+        assertT('jlu5|ovd')
+        assertT('B')
+        assertT('`')
+        assertT('Hello123')
+        assertT('test-')
+        assertT('test-test')
+        assertT('_jkl9')
+        assertT('_-jkl9')
+
+        assertF('')
+        assertF('?')
+        assertF('nick@network')
+        assertF('Space flight')
+        assertF(' ')
+        assertF(' Space blight')
+        assertF('0')
+        assertF('-test')
+        assertF('#lounge')
+        assertF('\\bar/bender\\')
+        assertF('jlu5/ovd') # Technically not valid, but some IRCds don't care ;)
+        assertF('100AAAAAC') # TS6 UID
+
+        self.assertFalse(self.p.is_nick('longnicklongnicklongnicklongnicklongnicklongnick', nicklen=20))
+        self.assertTrue(self.p.is_nick('ninechars', nicklen=9))
+        self.assertTrue(self.p.is_nick('ChanServ', nicklen=20))
+        self.assertTrue(self.p.is_nick('leneight', nicklen=9))
+        self.assertFalse(self.p.is_nick('bitmonster', nicklen=9))
+        self.assertFalse(self.p.is_nick('ninecharsplus', nicklen=12))
+
+    def test_is_channel(self):
+        assertT = lambda inp: self.assertTrue(self.p.is_channel(inp))
+        assertF = lambda inp: self.assertFalse(self.p.is_channel(inp))
+
+        assertT('#test')
+        assertT('#')
+        assertT('#a#b#c')
+
+        assertF('nick!user@host')
+        assertF('&channel')  # we don't support these yet
+        assertF('lorem ipsum')
+
+    def test_is_server_name(self):
+        self.assertTrue(self.p.is_server_name('test.local'))
+        self.assertTrue(self.p.is_server_name('IRC.example.com'))
+        self.assertTrue(self.p.is_server_name('services.'))
+        self.assertFalse(self.p.is_server_name('.org'))
+        self.assertFalse(self.p.is_server_name('bacon'))
+
+    def test_is_hostmask(self):
+        assertT = lambda inp: self.assertTrue(self.p.is_hostmask(inp))
+        assertF = lambda inp: self.assertFalse(self.p.is_hostmask(inp))
+
+        assertT('nick!user@host')
+        assertT('abc123!~ident@ip1-2-3-4.example.net')
+
+        assertF('brick!user')
+        assertF('user@host')
+        assertF('!@')
+        assertF('!')
+        assertF('@abcd')
+        assertF('#channel')
+        assertF('test.host')
+        assertF('nick ! user @ host')
+        assertF('alpha!beta@example.net#otherchan') # Janus workaround
+
+    def test_get_SID(self):
+        self.p.servers['serv1'] = Server(self.p, None, 'myserv.local', internal=True)
+
+        check = lambda inp, expected: self.assertEqual(self.p._get_SID(inp), expected)
+        check('myserv.local', 'serv1')
+        check('MYSERV.local', 'serv1')
+        check('serv1', 'serv1')
+        check('other.server', 'other.server')
+
+    def test_get_UID(self):
+        u = self._make_user('you', uid='100')
+        check = lambda inp, expected: self.assertEqual(self.p._get_UID(inp), expected)
+
+        check('you', '100')    # nick to UID
+        check('YOu', '100')
+        check('100', '100')    # already a UID
+        check('Test', 'Test')  # non-existent
+
+    def test_get_hostmask(self):
+        u = self._make_user('lorem', 'testUID', ident='ipsum', host='sit.amet')
+        self.assertEqual(self.p.get_hostmask(u.uid), 'lorem!ipsum@sit.amet')
+
+    def test_get_friendly_name(self):
+        u = self._make_user('lorem', 'testUID', ident='ipsum', host='sit.amet')
+        s = self.p.servers['mySID'] = Server(self.p, None, 'irc.example.org')
+        c = self.p._channels['#abc'] = Channel('#abc')
+
+        self.assertEqual(self.p.get_friendly_name(u.uid), 'lorem')
+        self.assertEqual(self.p.get_friendly_name('#abc'), '#abc')
+        self.assertEqual(self.p.get_friendly_name('mySID'), 'irc.example.org')
+
+    # TODO: _squit wrapper
+
+    ### MISC UTILS
+    def test_get_service_option(self):
+        f = self.p.get_service_option
+        self.assertEqual(f('myserv', 'myopt'), None)  # No value anywhere
+        self.assertEqual(f('myserv', 'myopt', default=0), 0)
+
+        # Define global option
+        with patch.dict(conf.conf, {'myserv': {'youropt': True, 'myopt': 1234}}):
+            self.assertEqual(f('myserv', 'myopt'), 1234)  # Read global option
+
+            # Both global and local options exist
+            with patch.dict(self.p.serverdata, {'myserv_myopt': 2345}):
+                self.assertEqual(f('myserv', 'myopt'), 2345)  # Read local option
+                self.assertEqual(f('myserv', 'myopt', global_option='youropt'), 2345)
+
+            # Custom global_option setting
+            self.assertEqual(f('myserv', 'abcdef', global_option='youropt'), True)
+
+        # Define local option
+        with patch.dict(self.p.serverdata, {'myserv_myopt': 998877}):
+            self.assertEqual(f('myserv', 'myopt'), 998877)  # Read local option
+            self.assertEqual(f('myserv', 'myopt', default='unused'), 998877)
+
+    def test_get_service_options_list(self):
+        f = self.p.get_service_options
+        self.assertEqual(f('myserv', 'items', list), [])  # No value anywhere
+
+        # Define global option
+        with patch.dict(conf.conf, {'myserv': {'items': [1, 10, 100], 'empty': []}}):
+            self.assertEqual(f('myserv', 'items', list), [1, 10, 100])  # Global value only
+            self.assertEqual(f('myserv', 'empty', list), [])
+
+            # Both global and local options exist
+            with patch.dict(self.p.serverdata, {'myserv_items': [2, 4, 6, 8], 'empty': []}):
+                self.assertEqual(f('myserv', 'items', list), [1, 10, 100, 2, 4, 6, 8])
+                # Custom global_option setting
+                self.assertEqual(f('myserv', 'items', list, global_option='nonexistent'), [2, 4, 6, 8])
+                self.assertEqual(f('myserv', 'empty', list), [])
+
+        # Define local option
+        with patch.dict(self.p.serverdata, {'myserv_items': [1, 0, 0, 3]}):
+            self.assertEqual(f('myserv', 'items', list), [1, 0, 0, 3])  # Read local option
+
+    def test_get_service_options_dict(self):
+        f = self.p.get_service_options
+        self.assertEqual(f('chanman', 'items', dict), {})  # No value anywhere
+
+        # This is just mildly relevant test data, it's not actually used anywhere.
+        globalopt = {'o': '@', 'v': '+', 'a': '!', 'h': '%'}
+        localopt  = {'a': '&', 'q': '~'}
+        # Define global option
+        with patch.dict(conf.conf, {'chanman': {'prefixes': globalopt, 'empty': {}}}):
+            self.assertEqual(f('chanman', 'prefixes', dict), globalopt)  # Global value only
+            self.assertEqual(f('chanman', 'empty', dict), {})
+
+            # Both global and local options exist
+            with patch.dict(self.p.serverdata, {'chanman_prefixes': localopt, 'empty': {}}):
+                self.assertEqual(f('chanman', 'prefixes', dict), {**globalopt, **localopt})
+
+                self.assertEqual(f('chanman', 'items', dict), {})  # No value anywhere
+                self.assertEqual(f('chanman', 'empty', dict), {})
+
+        # Define local option
+        with patch.dict(self.p.serverdata, {'chanman_prefixes': localopt}):
+            self.assertEqual(f('chanman', 'prefixes', dict), localopt)  # Read local option
+
+    ### MODE HANDLING
+    def test_parse_modes_channel_rfc(self):
+        # These are basic tests that only use RFC 1459 defined modes.
+        # IRCds supporting more complex modes can define new test cases if needed.
+        u = self._make_user('testuser', uid='100')
+
+        c = self.p.channels['#testruns'] = Channel(self.p, name='#testruns')
+
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+m']),
+            [('+m', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+l', '3']),
+            [('+l', '3')]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+ntl', '59']),
+            [('+n', None), ('+t', None), ('+l', '59')]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+k-n', 'test']),
+            [('+k', 'test'), ('-n', None)]
+        )
+
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+o', '102']),  # unknown UID
+            []
+        )
+
+        c.users.add(u)
+        u.channels.add(c)
+
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+o', '100']),
+            [('+o', '100')]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+vip', '100']),
+            [('+v', '100'), ('+i', None), ('+p', None)]
+        )
+
+    def test_parse_modes_channel_rfc2(self):
+        # These are basic tests that only use RFC 1459 defined modes.
+        # IRCds supporting more complex modes can define new test cases if needed.
+        c = self.p.channels['#testruns'] = Channel(self.p, name='#testruns')
+
+        # Note: base case is not defined and raises AssertionError
+
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+m']),   # add modes
+            [('+m', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['-tn']),  # remove modes
+            [('-t', None), ('-n', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#TESTRUNS', ['-tn']),  # different case target
+            [('-t', None), ('-n', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+l', '3']),  # modes w/ arguments
+            [('+l', '3')]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+nlt', '59']),  # combination
+            [('+n', None), ('+l', '59'), ('+t', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+k-n', 'test']),  # swapping +/-
+            [('+k', 'test'), ('-n', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['n-s']),  # sloppy syntax
+            [('+n', None), ('-s', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+bmi', '*!test@example.com']),
+            [('+b', '*!test@example.com'), ('+m', None), ('+i', None)]
+        )
+
+    def test_parse_modes_prefixmodes_rfc(self):
+        c = self.p.channels['#testruns'] = Channel(self.p, name='#testruns')
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+ov', '102', '101']),  # unknown UIDs are ignored
+            []
+        )
+        u = self._make_user('test100', uid='100')
+        c.users.add(u)
+        u.channels.add(c)
+
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+o', '100']),
+            [('+o', '100')]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+vip', '100']),
+            [('+v', '100'), ('+i', None), ('+p', None)]
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['-o+bn', '100', '*!test@example.com']),
+            [('-o', '100'), ('+b', '*!test@example.com'), ('+n', None)]
+        )
+        self.assertEqual(
+            # 2nd user missing
+            self.p.parse_modes('#testruns', ['+oovv', '100', '102', '100', '102']),
+            [('+o', '100'), ('+v', '100')]
+        )
+
+        u2 = self._make_user('test102', uid='102')
+        c.users.add(u2)
+        u2.channels.add(c)
+
+        self.assertEqual(
+            # two users interleaved
+            self.p.parse_modes('#testruns', ['+oovv', '100', '102', '100', '102']),
+            [('+o', '100'), ('+o', '102'), ('+v', '100'), ('+v', '102')]
+        )
+
+        self.assertEqual(
+            # Mode cycle
+            self.p.parse_modes('#testruns', ['+o-o', '100', '100']),
+            [('+o', '100'), ('-o', '100')]
+        )
+
+    def test_parse_modes_channel_ban_complex(self):
+        c = self.p.channels['#testruns'] = Channel(self.p, name='#testruns')
+        self.assertEqual(
+            # add first ban, but don't remove second because it doesn't exist
+            self.p.parse_modes('#testruns', ['+b-b', '*!*@test1', '*!*@test2']),
+            [('+b', '*!*@test1')],
+            "First ban should have been added, and second ignored"
+        )
+        self.p.apply_modes('#testruns', [('+b', '*!*@test1')])
+        self.assertEqual(
+            # remove first ban because it matches case
+            self.p.parse_modes('#testruns', ['-b', '*!*@test1']),
+            [('-b', '*!*@test1')],
+            "First ban should have been removed (same case)"
+        )
+        self.assertEqual(
+            # remove first ban despite different case
+            self.p.parse_modes('#testruns', ['-b', '*!*@TEST1']),
+            [('-b', '*!*@test1')],
+            "First ban should have been removed (different case)"
+        )
+        self.p.apply_modes('#testruns', [('+b', '*!*@Test2')])
+        self.assertEqual(
+            # remove second ban despite different case
+            self.p.parse_modes('#testruns', ['-b', '*!*@test2']),
+            [('-b', '*!*@Test2')],
+            "Second ban should have been removed (different case)"
+        )
+
+    def test_parse_modes_channel_ban_cycle(self):
+        c = self.p.channels['#testruns'] = Channel(self.p, name='#testruns')
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+b-b', '*!*@example.com', '*!*@example.com']),
+            [('+b', '*!*@example.com'), ('-b', '*!*@example.com')],
+            "Cycling a ban +b-b should remove it"
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['-b+b', '*!*@example.com', '*!*@example.com']),
+            [('+b', '*!*@example.com')],
+            "Cycling a ban -b+b should add it"
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+b-b', '*!*@example.com', '*!*@Example.com']),
+            [('+b', '*!*@example.com'), ('-b', '*!*@example.com')],
+            "Cycling a ban +b-b should remove it (different case)"
+        )
+        self.assertEqual(
+            self.p.parse_modes('#testruns', ['+b-b', '*!*@Example.com', '*!*@example.com']),
+            [('+b', '*!*@Example.com'), ('-b', '*!*@Example.com')],
+            "Cycling a ban +b-b should remove it (different case)"
+        )
+
+    def test_parse_modes_channel_prefixmode_has_nick(self):
+        c = self.p.channels['#'] = Channel(self.p, name='#')
+        u = self._make_user('mynick', uid='myuid')
+        c.users.add(u)
+        u.channels.add(c)
+
+        self.assertEqual(
+            self.p.parse_modes('#', ['+o', 'myuid']),
+            [('+o', 'myuid')],
+            "+o on UID should be registered"
+        )
+        self.assertEqual(
+            self.p.parse_modes('#', ['+o', 'mynick']),
+            [('+o', 'myuid')],
+            "+o on nick should be registered"
+        )
+        self.assertEqual(
+            self.p.parse_modes('#', ['+o', 'MyUiD']),
+            [],
+            "+o on wrong case UID should be ignored"
+        )
+        self.assertEqual(
+            self.p.parse_modes('#', ['+o', 'MyNick']),
+            [('+o', 'myuid')],
+            "+o on different case nick should be registered"
+        )
+
+        self.assertEqual(
+            self.p.parse_modes('#', ['-o', 'myuid']),
+            [('-o', 'myuid')],
+            "-o on UID should be registered"
+        )
+        self.assertEqual(
+            self.p.parse_modes('#', ['-o', 'mynick']),
+            [('-o', 'myuid')],
+            "-o on nick should be registered"
+        )
+
+    # TODO: check the output of parse_modes() here too
+    def test_parse_modes_channel_key(self):
+        c = self.p.channels['#pylink'] = Channel(self.p, name='#pylink')
+        c.modes = {('k', 'foobar')}
+
+        modes = self.p.parse_modes('#pylink', ['-k', 'foobar'])
+        self.assertEqual(modes, [('-k', 'foobar')], "Parse result should include -k")
+
+        modes = self.p.parse_modes('#pylink', ['-k', 'aBcDeF'])
+        self.assertEqual(modes, [], "Incorrect key removal should be ignored")
+
+        modes = self.p.parse_modes('#pylink', ['+k', 'aBcDeF'])
+        self.assertEqual(modes, [('+k', 'aBcDeF')], "Parse result should include +k (replace key)")
+
+        # Mismatched case - treat this as remove
+        # Some IRCds allow this (Unreal, P10), others do not (InspIRCd). However, if such a message
+        # makes actually its way to us it is most likely valid.
+        # Note: Charybdis and ngIRCd do -k * removal instead so this case will never happen there
+        modes = self.p.parse_modes('#pylink', ['-k', 'FooBar'])
+        self.assertEqual(modes, [('-k', 'foobar')], "Parse result should include -k (different case)")
+
+        modes = self.p.parse_modes('#pylink', ['-k', '*'])
+        self.assertEqual(modes, [('-k', 'foobar')], "Parse result should include -k (-k *)")
+
+    def test_parse_modes_user_rfc(self):
+        u = self._make_user('testuser', uid='100')
+
+        self.assertEqual(
+            self.p.parse_modes('100', ['+i-w+x']),
+            [('+i', None), ('-w', None), ('+x', None)]
+        )
+        self.assertEqual(
+            # Sloppy syntax, but OK
+            self.p.parse_modes('100', ['wx']),
+            [('+w', None), ('+x', None)]
+        )
+
+    def test_apply_modes_channel_simple(self):
+        c = self.p.channels['#'] = Channel(self.p, name='#')
+
+        self.p.apply_modes('#', [('+m', None)])
+        self.assertEqual(c.modes, {('m', None)})
+
+        self.p.apply_modes('#', [])  # No-op
+        self.assertEqual(c.modes, {('m', None)})
+
+        self.p.apply_modes('#', [('-m', None)])
+        self.assertFalse(c.modes)  # assert is empty
+
+    def test_apply_modes_channel_add_remove(self):
+        c = self.p.channels['#'] = Channel(self.p, name='#')
+
+        self.p.apply_modes('#', [('+i', None)])
+        self.assertEqual(c.modes, {('i', None)})
+
+        self.p.apply_modes('#', [('+n', None), ('-i', None)])
+        self.assertEqual(c.modes, {('n', None)})
+
+        c = self.p.channels['#Magic'] = Channel(self.p, name='#Magic')
+        self.p.apply_modes('#Magic', [('+m', None), ('+n', None), ('+i', None)])
+        self.assertEqual(c.modes, {('m', None), ('n', None), ('i', None)}, "Modes should be added")
+
+        self.p.apply_modes('#Magic', [('-i', None), ('-n', None)])
+        self.assertEqual(c.modes, {('m', None)}, "Modes should be removed")
+
+    def test_apply_modes_channel_remove_nonexistent(self):
+        c = self.p.channels['#abc'] = Channel(self.p, name='#abc')
+        self.p.apply_modes('#abc', [('+t', None)])
+        self.assertEqual(c.modes, {('t', None)})
+
+        self.p.apply_modes('#abc', [('-n', None), ('-i', None)])
+        self.assertEqual(c.modes, {('t', None)})
+
+    def test_apply_modes_channel_limit(self):
+        c = self.p.channels['#abc'] = Channel(self.p, name='#abc')
+        self.p.apply_modes('#abc', [('+t', None), ('+l', '30')])
+        self.assertEqual(c.modes, {('t', None), ('l', '30')})
+
+        self.p.apply_modes('#abc', [('+l', '55')])
+        self.assertEqual(c.modes, {('t', None), ('l', '55')})
+
+        self.p.apply_modes('#abc', [('-l', None)])
+        self.assertEqual(c.modes, {('t', None)})
+
+    def test_apply_modes_channel_key(self):
+        c = self.p.channels['#pylink'] = Channel(self.p, name='#pylink')
+        self.p.apply_modes('#pylink', [('+k', 'password123'), ('+s', None)])
+        self.assertEqual(c.modes, {('s', None), ('k', 'password123')})
+
+        self.p.apply_modes('#pylink', [('+k', 'qwerty')])
+        self.assertEqual(c.modes, {('s', None), ('k', 'qwerty')})
+
+        self.p.apply_modes('#pylink', [('-k', 'abcdef')])
+        # Trying to remove with wrong key is a no-op
+        self.assertEqual(c.modes, {('s', None), ('k', 'qwerty')})
+
+        self.p.apply_modes('#pylink', [('-k', 'qwerty')])
+        self.assertEqual(c.modes, {('s', None)})
+
+        self.p.apply_modes('#pylink', [('+k', 'qwerty')])
+        self.assertEqual(c.modes, {('s', None), ('k', 'qwerty')})
+        self.p.apply_modes('#pylink', [('-k', 'QWERTY')])  # Remove with different case
+        self.assertEqual(c.modes, {('s', None)})
+
+        self.p.apply_modes('#pylink', [('+k', '12345')])  # Replace existing key
+        self.assertEqual(c.modes, {('s', None), ('k', '12345')})
+
+    def test_apply_modes_channel_ban(self):
+        c = self.p.channels['#Magic'] = Channel(self.p, name='#Magic')
+        self.p.apply_modes('#Magic', [('+b', '*!*@test.host'), ('+b', '*!*@best.host')])
+        self.assertEqual(c.modes, {('b', '*!*@test.host'), ('b', '*!*@best.host')}, "Bans should be added")
+
+        # This should be a no-op
+        self.p.apply_modes('#Magic', [('-b', '*!*non-existent')])
+        self.assertEqual(c.modes, {('b', '*!*@test.host'), ('b', '*!*@best.host')}, "Trying to unset non-existent ban should be no-op")
+
+        # Simple removal
+        self.p.apply_modes('#Magic', [('-b', '*!*@test.host')])
+        self.assertEqual(c.modes, {('b', '*!*@best.host')}, "Ban on *!*@test.host be removed (same case as original)")
+
+        # Removal but different case than original
+        self.p.apply_modes('#Magic', [('-b', '*!*@BEST.HOST')])
+        self.assertFalse(c.modes, "Ban on *!*@best.host should be removed (different case)")
+
+    def test_apply_modes_channel_ban_multiple(self):
+        c = self.p.channels['#Magic'] = Channel(self.p, name='#Magic')
+        self.p.apply_modes('#Magic', [('+b', '*!*@test.host'), ('+b', '*!*@best.host'), ('+b', '*!*@guest.host')])
+        self.assertEqual(c.modes, {('b', '*!*@test.host'), ('b', '*!*@best.host'), ('b', '*!*@guest.host')},
+                         "Bans should be added")
+
+        self.p.apply_modes('#Magic', [('-b', '*!*@best.host'), ('-b', '*!*@guest.host'), ('-b', '*!*@test.host')])
+        self.assertEqual(c.modes, set(), "Bans should be removed")
+
+    def test_apply_modes_channel_mode_cycle(self):
+        c = self.p.channels['#Magic'] = Channel(self.p, name='#Magic')
+        self.p.apply_modes('#Magic', [('+b', '*!*@example.net'), ('-b', '*!*@example.net')])
+        self.assertEqual(c.modes, set(), "Ban should have been removed (same case)")
+
+        self.p.apply_modes('#Magic', [('+b', '*!*@example.net'), ('-b', '*!*@Example.net')])
+        self.assertEqual(c.modes, set(), "Ban should have been removed (different case)")
+
+        u = self._make_user('nick', uid='user')
+        c.users.add(u.uid)
+        u.channels.add(c)
+
+        self.p.apply_modes('#Magic', [('+o', 'user'), ('-o', 'user')])
+        self.assertEqual(c.modes, set(), "No prefixmodes should have been set")
+        self.assertFalse(c.is_op('user'))
+        self.assertFalse(c.get_prefix_modes('user'))
+
+    def test_apply_modes_channel_prefixmodes(self):
+        # Make some users
+        c = self.p.channels['#staff'] = Channel(self.p, name='#staff')
+        u1 = self._make_user('user100', uid='100')
+        u2 = self._make_user('user101', uid='101')
+        c.users.add(u1.uid)
+        u1.channels.add(c)
+        c.users.add(u2.uid)
+        u2.channels.add(c)
+
+        # Set modes
+        self.p.apply_modes('#staff', [('+o', '100'), ('+v', '101'), ('+t', None)])
+        self.assertEqual(c.modes, {('t', None)})
+
+        # State checks, lots of them... TODO: move this into Channel class tests
+        self.assertTrue(c.is_op('100'))
+        self.assertTrue(c.is_op_plus('100'))
+        self.assertFalse(c.is_halfop('100'))
+        self.assertTrue(c.is_halfop_plus('100'))
+        self.assertFalse(c.is_voice('100'))
+        self.assertTrue(c.is_voice_plus('100'))
+        self.assertEqual(c.get_prefix_modes('100'), ['op'])
+
+        self.assertTrue(c.is_voice('101'))
+        self.assertTrue(c.is_voice_plus('101'))
+        self.assertFalse(c.is_halfop('101'))
+        self.assertFalse(c.is_halfop_plus('101'))
+        self.assertFalse(c.is_op('101'))
+        self.assertFalse(c.is_op_plus('101'))
+        self.assertEqual(c.get_prefix_modes('101'), ['voice'])
+
+        self.assertFalse(c.prefixmodes['owner'])
+        self.assertFalse(c.prefixmodes['admin'])
+        self.assertEqual(c.prefixmodes['op'], {'100'})
+        self.assertFalse(c.prefixmodes['halfop'])
+        self.assertEqual(c.prefixmodes['voice'], {'101'})
+
+        self.p.apply_modes('#staff', [('-o', '100')])
+        self.assertEqual(c.modes, {('t', None)})
+        self.assertFalse(c.get_prefix_modes('100'))
+        self.assertEqual(c.get_prefix_modes('101'), ['voice'])
+
+    def test_apply_modes_user(self):
+        u = self._make_user('nick', uid='user')
+        self.p.apply_modes('user', [('+o', None), ('+w', None)])
+        self.assertEqual(u.modes, {('o', None), ('w', None)})
+        self.p.apply_modes('user', [('-o', None), ('+i', None)])
+        self.assertEqual(u.modes, {('i', None), ('w', None)})
+
+    def test_reverse_modes_simple(self):
+        c = self.p.channels['#foobar'] = Channel(self.p, name='#foobar')
+        c.modes = {('m', None), ('n', None)}
+
+        # This function supports both strings and mode lists
+
+        # Base cases
+        for inp in {'', '+', '-'}:
+            self.assertEqual(self.p.reverse_modes('#foobar', inp), '+')
+        out = self.p.reverse_modes('#foobar', [])
+        self.assertEqual(out, [])
+
+        # One simple
+        out = self.p.reverse_modes('#foobar', '+t')
+        self.assertEqual(out, '-t')
+        out = self.p.reverse_modes('#foobar', [('+t', None)])
+        self.assertEqual(out, [('-t', None)])
+
+        out = self.p.reverse_modes('#foobar', '+m')
+        self.assertEqual(out, '+', 'Calling reverse_modes() on an already set mode is a no-op')
+
+        out = self.p.reverse_modes('#foobar', [('+m', None)])
+        self.assertEqual(out, [], 'Calling reverse_modes() on an already set mode is a no-op')
+
+        out = self.p.reverse_modes('#foobar', [('-i', None)])
+        self.assertEqual(out, [], 'Calling reverse_modes() on non-existent mode is a no-op')
+
+    def test_reverse_modes_multi(self):
+        c = self.p.channels['#foobar'] = Channel(self.p, name='#foobar')
+        c.modes = {('t', None), ('n', None)}
+
+        # reverse_modes should ignore modes that were already set
+        out = self.p.reverse_modes('#foobar', '+nt')
+        self.assertEqual(out, '+')
+
+        out = self.p.reverse_modes('#foobar', [('+m', None), ('+i', None)])
+        self.assertEqual(out, [('-m', None), ('-i', None)])
+
+        out = self.p.reverse_modes('#foobar', '+mint')
+        self.assertEqual(out, '-mi')  # Ignore +nt since it already exists
+
+        out = self.p.reverse_modes('#foobar', '+m-t')
+        self.assertEqual(out, '-m+t')
+
+        out = self.p.reverse_modes('#foobar', '+mn-t')
+        self.assertEqual(out, '-m+t')
+
+        out = self.p.reverse_modes('#foobar', [('+m', None), ('+n', None), ('-t', None)])
+        self.assertEqual(out, [('-m', None), ('+t', None)])
+
+    def test_reverse_modes_bans(self):
+        c = self.p.channels['#foobar'] = Channel(self.p, name='#foobar')
+        c.modes = {('t', None), ('n', None), ('b', '*!*@example.com')}
+
+        out = self.p.reverse_modes('#foobar', [('+b', '*!*@test')])
+        self.assertEqual(out, [('-b', '*!*@test')], "non-existent ban should be removed")
+        out = self.p.reverse_modes('#foobar', '+b *!*@example.com')
+        self.assertEqual(out, '+', "+b existing ban should be no-op")
+
+        out = self.p.reverse_modes('#foobar', '+b *!*@Example.com')
+        self.assertEqual(out, '+', "Should ignore attempt to change case of ban mode")
+
+        out = self.p.reverse_modes('#foobar', '-b *!*@example.com')
+        self.assertEqual(out, '+b *!*@example.com', "-b existing ban should reset it")
+
+        out = self.p.reverse_modes('#foobar', '-b *!*@Example.com')
+        self.assertEqual(out, '+b *!*@example.com', "-b existing ban should reset it using original case")
+
+        out = self.p.reverse_modes('#foobar', '-b *!*@*')
+        self.assertEqual(out, '+', "Removing non-existent ban is no-op")
+
+        out = self.p.reverse_modes('#foobar', '+bbbm 1 2 3')
+        self.assertEqual(out, '-bbbm 1 2 3')
+
+    def test_reverse_modes_limit(self):
+        c = self.p.channels['#foobar'] = Channel(self.p, name='#foobar')
+        c.modes = {('t', None), ('n', None), ('l', '50')}
+
+        out = self.p.reverse_modes('#foobar', [('+l', '100')])
+        self.assertEqual(out, [('+l', '50')], "Setting +l should reset original mode")
+
+        out = self.p.reverse_modes('#foobar', [('-l', None)])
+        self.assertEqual(out, [('+l', '50')], "Unsetting +l should reset original mode")
+
+        out = self.p.reverse_modes('#foobar', [('+l', '50')])
+        self.assertEqual(out, [], "Setting +l with original value is no-op")
+
+        c.modes.clear()
+        out = self.p.reverse_modes('#foobar', [('+l', '111'), ('+m', None)])
+        self.assertEqual(out, [('-l', None), ('-m', None)], "Setting +l on channel without it should remove")
+
+    def test_reverse_modes_prefixmodes(self):
+        c = self.p.channels['#foobar'] = Channel(self.p, name='#foobar')
+        c.modes = {('t', None), ('n', None)}
+        u = self._make_user('nick', uid='user')
+        u.channels.add(c)
+        c.users.add(u)
+        c.prefixmodes['op'].add(u.uid)
+
+        out = self.p.reverse_modes('#foobar', '-o user')
+        self.assertEqual(out, '+o user')
+        out = self.p.reverse_modes('#foobar', '+o user')
+        self.assertEqual(out, '+')
+        out = self.p.reverse_modes('#foobar', '+ov user user')
+        self.assertEqual(out, '-v user')  # ignore +o
+        out = self.p.reverse_modes('#foobar', '-ovt user user')
+        self.assertEqual(out, '+ot user')  # ignore -v
+
+    def test_reverse_modes_cycle_simple(self):
+        c = self.p.channels['#weirdstuff'] = Channel(self.p, name='#weirdstuff')
+        c.modes = {('t', None), ('n', None)}
+
+        out = self.p.reverse_modes('#weirdstuff', '+n-n')  # +- cycle existing mode
+        self.assertEqual(out, '+n')
+        out = self.p.reverse_modes('#weirdstuff', '-n+n')  # -+ cycle existing mode
+        self.assertEqual(out, '+n')  # Ugly but OK
+
+        out = self.p.reverse_modes('#weirdstuff', '+m-m')  # +- cycle non-existent mode
+        self.assertEqual(out, '-m')
+        out = self.p.reverse_modes('#weirdstuff', '-m+m')  # -+ cycle non-existent mode
+        self.assertEqual(out, '-m')  # Ugly but OK
+
+    def test_reverse_modes_cycle_bans(self):
+        c = self.p.channels['#weirdstuff'] = Channel(self.p, name='#weirdstuff')
+        c.modes = {('t', None), ('n', None), ('b', '*!*@test.host')}
+
+        out = self.p.reverse_modes('#weirdstuff', '+b-b *!*@test.host *!*@test.host')  # +- cycle existing ban
+        self.assertEqual(out, '+b *!*@test.host')
+        out = self.p.reverse_modes('#weirdstuff', '-b+b *!*@test.host *!*@test.host')  # -+ cycle existing ban
+        self.assertEqual(out, '+b *!*@test.host')  # Ugly but OK
+
+        out = self.p.reverse_modes('#weirdstuff', '+b-b *!*@* *!*@*')  # +- cycle existing ban
+        self.assertEqual(out, '-b *!*@*')
+        out = self.p.reverse_modes('#weirdstuff', '-b+b *!*@* *!*@*')  # -+ cycle existing ban
+        self.assertEqual(out, '-b *!*@*')  # Ugly but OK
+
+    def test_reverse_modes_cycle_arguments(self):
+        # All of these cases are ugly, sometimes unsetting modes that don't exist...
+        c = self.p.channels['#weirdstuff'] = Channel(self.p, name='#weirdstuff')
+
+        out = self.p.reverse_modes('#weirdstuff', '+l-l 30')
+        self.assertEqual(out, '-l')
+        out = self.p.reverse_modes('#weirdstuff', '-l+l 30')
+        self.assertEqual(out, '-l')
+
+        out = self.p.reverse_modes('#weirdstuff', '+k-k aaaaaaaaaaaa aaaaaaaaaaaa')
+        self.assertEqual(out, '-k aaaaaaaaaaaa')
+        out = self.p.reverse_modes('#weirdstuff', '-k+k aaaaaaaaaaaa aaaaaaaaaaaa')
+        self.assertEqual(out, '-k aaaaaaaaaaaa')
+
+        c.modes = {('l', '555'), ('k', 'NO-PLEASE')}
+        out = self.p.reverse_modes('#weirdstuff', '+l-l 30')
+        self.assertEqual(out, '+l 555')
+        out = self.p.reverse_modes('#weirdstuff', '-l+l 30')
+        self.assertEqual(out, '+l 555')
+
+        out = self.p.reverse_modes('#weirdstuff', '+k-k aaaaaaaaaaaa aaaaaaaaaaaa')
+        self.assertEqual(out, '+k NO-PLEASE')
+        out = self.p.reverse_modes('#weirdstuff', '-k+k aaaaaaaaaaaa aaaaaaaaaaaa')
+        self.assertEqual(out, '+k NO-PLEASE')
+
+    def test_reverse_modes_cycle_prefixmodes(self):
+        # All of these cases are ugly, sometimes unsetting modes that don't exist...
+        c = self.p.channels['#weirdstuff'] = Channel(self.p, name='#weirdstuff')
+        u = self._make_user('nick', uid='user')
+        u.channels.add(c)
+        c.users.add(u)
+
+        # user not already opped
+        out = self.p.reverse_modes('#weirdstuff', '+o-o user user')
+        self.assertEqual(out, '-o user')
+        out = self.p.reverse_modes('#weirdstuff', '-o+o user user')
+        self.assertEqual(out, '-o user')
+
+        c.prefixmodes['op'].add(u.uid)
+
+        # user was opped
+        out = self.p.reverse_modes('#weirdstuff', '+o-o user user')
+        self.assertEqual(out, '+o user')
+        out = self.p.reverse_modes('#weirdstuff', '-o+o user user')
+        self.assertEqual(out, '+o user')
+
+    def test_join_modes(self):
+        # join_modes operates independently of state; the input just has to be valid modepairs
+        check = lambda inp, expected, sort=False: self.assertEqual(self.p.join_modes(inp, sort=sort), expected)
+
+        check([], '+')  # base case
+
+        check([('+b', '*!*@test')], '+b *!*@test')
+        check([('-S', None)], '-S')
+        check([('+n', None), ('+t', None)], '+nt')
+        check([('+t', None), ('+n', None)], '+tn')
+        check([('+t', None), ('+n', None)], '+nt', sort=True)
+
+        check([('-n', None), ('-s', None)], '-ns')
+
+        check([('+q', '*'), ('-q', '*')], '+q-q * *')
+        check([('+l', '5'), ('-n', None), ('+R', None)], '+l-n+R 5')
+
+        # Sloppy syntax: assume leading mode is + if not otherwise stated
+        check([('o', '100AAAAAC'), ('m', None), ('-v', '100AAAAAC')], '+om-v 100AAAAAC 100AAAAAC')
+
+    def test_wrap_modes_small(self):
+        # wrap_modes is also state independent; it only calls join_modes
+
+        N_USERS = 5
+        modes = [('+m', None)] + [('-b', 'user%s!*@example.org' % num) for num in range(N_USERS)]
+        wr = self.p.wrap_modes(modes, 200)
+        log.debug('wrap_modes input: %s', modes)
+        log.debug('wrap_modes output: %s', wr)
+
+        self.assertEqual(len(wr), 1)  # no split should have occurred
+        self.assertEqual(wr[0], self.p.join_modes(modes))
+
+    def test_wrap_modes_split_length_limit(self):
+        # wrap_modes is also state independent; it only calls join_modes
+        N_USERS = 50
+
+        modes = [('+o', 'user%s' % num) for num in range(N_USERS)]
+        wr = self.p.wrap_modes(modes, 120)
+        log.debug('wrap_modes input: %s', modes)
+        log.debug('wrap_modes output: %s', wr)
+
+        self.assertTrue(len(wr) > 1)  # we should have induced a split
+
+        for s in wr:
+            # Check that each split item starts with the right mode char
+            self.assertTrue(s.startswith('+oooo'))
+
+        all_args = itertools.chain.from_iterable(s.split() for s in wr)
+        for num in range(N_USERS):
+            # Check that no users are missing
+            self.assertIn('user%s' % num, all_args)
+
+    def test_wrap_modes_split_max_modes(self):
+        # wrap_modes is also state independent; it only calls join_modes
+        N_USERS = 50
+        N_MAX_PER_MSG = 8
+
+        modes = [('+v', 'user%s' % num) for num in range(N_USERS)]
+        wr = self.p.wrap_modes(modes, 200, N_MAX_PER_MSG)
+        log.debug('wrap_modes input: %s', modes)
+        log.debug('wrap_modes output: %s', wr)
+
+        self.assertTrue(len(wr) > 1)  # we should have induced a split
+
+        splits = [s.split() for s in wr]
+        for s in splits:
+            # Check that each message sets <= N_MAX_PER_MSG modes
+            self.assertTrue(len(s[0]) <= (N_MAX_PER_MSG + 1))  # add 1 to account for leading +
+            self.assertTrue(s[0].startswith('+'))
+            self.assertTrue(s[0].endswith('v'))
+
+        all_args = itertools.chain.from_iterable(splits)
+        for num in range(N_USERS):
+            # Check that no users are missing
+            self.assertIn('user%s' % num, all_args)
+
+    # TODO: test type coersion if channel or mode targets are ints

test/test_irc_parsers.py

@@ -0,0 +1,118 @@
+"""
+Runs IRC parser tests from ircdocs/parser-tests.
+
+This test suite runs static code only.
+"""
+from pathlib import Path
+import unittest
+
+import yaml
+
+PARSER_DATA_PATH = Path(__file__).parent.resolve() / 'parser-tests' / 'tests'
+print(PARSER_DATA_PATH)
+
+from pylinkirc import utils
+from pylinkirc.protocols.ircs2s_common import IRCCommonProtocol
+
+class MessageParserTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        with open(PARSER_DATA_PATH / 'msg-split.yaml') as f:
+            cls.MESSAGE_SPLIT_TEST_DATA = yaml.safe_load(f)
+        with open(PARSER_DATA_PATH / 'userhost-split.yaml') as f:
+            cls.USER_HOST_SPLIT_TEST_DATA = yaml.safe_load(f)
+        with open(PARSER_DATA_PATH / 'mask-match.yaml') as f:
+            cls.MASK_MATCH_TEST_DATA = yaml.safe_load(f)
+        with open(PARSER_DATA_PATH / 'validate-hostname.yaml') as f:
+            cls.VALIDATE_HOSTNAME_TEST_DATA = yaml.safe_load(f)
+
+    def testMessageSplit(self):
+        for testdata in self.MESSAGE_SPLIT_TEST_DATA['tests']:
+            inp = testdata['input']
+            atoms = testdata['atoms']
+
+            with self.subTest():
+                expected = []
+                has_source = False
+                if 'source' in atoms:
+                    has_source = True
+                    expected.append(atoms['source'])
+
+                if 'verb' in atoms:
+                    expected.append(atoms['verb'])
+
+                if 'params' in atoms:
+                    expected.extend(atoms['params'])
+
+                if 'tags' in atoms:
+                    # Remove message tags before parsing
+                    _, inp = inp.split(" ", 1)
+
+                if has_source:
+                    parts = IRCCommonProtocol.parse_prefixed_args(inp)
+                else:
+                    parts = IRCCommonProtocol.parse_args(inp)
+                self.assertEqual(expected, parts, "Parse test failed for string: %r" % inp)
+
+    @unittest.skip("Not quite working yet")
+    def testMessageTags(self):
+        for testdata in self.MESSAGE_SPLIT_TEST_DATA['tests']:
+            inp = testdata['input']
+            atoms = testdata['atoms']
+
+            with self.subTest():
+                if 'tags' in atoms:
+                    self.assertEqual(atoms['tags'], IRCCommonProtocol.parse_message_tags(inp.split(" ")),
+                                     "Parse test failed for message tags: %r" % inp)
+
+    def testUserHostSplit(self):
+        for test in self.USER_HOST_SPLIT_TEST_DATA['tests']:
+            inp = test['source']
+            atoms = test['atoms']
+
+            with self.subTest():
+                if 'nick' not in atoms or 'user' not in atoms or 'host' not in atoms:
+                    # Trying to parse a hostmask with missing atoms is an error in split_hostmask()
+                    with self.assertRaises(ValueError):
+                        utils.split_hostmask(inp)
+                else:
+                    expected = [atoms['nick'], atoms['user'], atoms['host']]
+                    self.assertEqual(expected, utils.split_hostmask(inp))
+
+    def testHostMatch(self):
+        for test in self.MASK_MATCH_TEST_DATA['tests']:
+            mask = test['mask']
+
+            # N.B.: utils.match_text() does Unicode case-insensitive match by default,
+            # which might not be the right thing to do on IRC.
+            # But irc.to_lower() isn't a static function so we're not testing it here...
+            for match in test['matches']:
+                with self.subTest():
+                    self.assertTrue(utils.match_text(mask, match))
+
+            for fail in test['fails']:
+                with self.subTest():
+                    self.assertFalse(utils.match_text(mask, fail))
+
+    def testValidateHostname(self):
+        for test in self.VALIDATE_HOSTNAME_TEST_DATA['tests']:
+            with self.subTest():
+                self.assertEqual(test['valid'], IRCCommonProtocol.is_server_name(test['host']),
+                                 "Failed test for %r; should be %s" % (test['host'], test['valid']))
+
+
+    # N.B. skipping msg-join tests because PyLink doesn't think about messages that way
+
+    ### Custom test cases
+    def testMessageSplitSpaces(self):
+        # Test that tokenization ignores empty fields, but doesn't strip away other types of whitespace
+        f = IRCCommonProtocol.parse_prefixed_args
+        self.assertEqual(f(":foo PRIVMSG  #test :message"), ["foo", "PRIVMSG", "#test", "message"])
+        self.assertEqual(f(":123LOLWUT NICK cursed\u3000nickname"), ["123LOLWUT", "NICK", "cursed\u3000nickname"])
+        self.assertEqual(f(":123LOLWUT MODE ## +ov \x1f checking"),
+                         ["123LOLWUT", "MODE", "##", "+ov", "\x1f", "checking"])
+        self.assertEqual(f(":123LOLWUT MODE  ## +ov  \u3000  checking"),
+                         ["123LOLWUT", "MODE", "##", "+ov", "\u3000", "checking"])
+
+if __name__ == '__main__':
+    unittest.main()