GitHub/ergo
3
0
Fork 0
mirror of https://github.com/ergochat/ergo.git synced 2024-11-25 21:39:25 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #405 from slingamn/language_loading.2

Browse Source 
fix #384
This commit is contained in:
Daniel Oaks 2019-02-26 21:14:43 +10:00 committed by GitHub
commit 9ec4f76c60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 224 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
irc/client.go
View File

@ -131,6 +131,7 @@ func NewClient(server *Server, conn net.Conn, isTLS bool) {
channels: make(ChannelSet),
ctime: now,
flags: modes.NewModeSet(),
languages: server.Languages().Default(),
loginThrottle: connection_limits.GenericThrottle{
Duration: config.Accounts.LoginThrottling.Duration,
Limit: config.Accounts.LoginThrottling.MaxAttempts,
@ -143,7 +144,6 @@ func NewClient(server *Server, conn net.Conn, isTLS bool) {
nickMaskString: "*", // * is used until actual nick is given
history: history.NewHistoryBuffer(config.History.ClientLength),
}
client.languages = server.languages.Default()
remoteAddr := conn.RemoteAddr()
client.realIP = utils.AddrToIP(remoteAddr)

122
irc/config.go
View File

@ -7,13 +7,11 @@ package irc
import (
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net"
"os"
"path/filepath"
"regexp"
"strings"
"time"
@ -283,9 +281,10 @@ type Config struct {
Enabled bool
Path string
Default string
Data map[string]languages.LangData
}
languageManager *languages.Manager
Datastore struct {
Path string
AutoUpgrade bool
@ -638,120 +637,9 @@ func LoadConfig(filename string) (config *Config, err error) {
}
config.Server.MaxSendQBytes = int(maxSendQBytes)
// get language files
config.Languages.Data = make(map[string]languages.LangData)
if config.Languages.Enabled {
files, err := ioutil.ReadDir(config.Languages.Path)
if err != nil {
return nil, fmt.Errorf("Could not load language files: %s", err.Error())
}
for _, f := range files {
// skip dirs
if f.IsDir() {
continue
}
// only load core .lang.yaml file, and ignore help/irc files
name := f.Name()
lowerName := strings.ToLower(name)
if !strings.HasSuffix(lowerName, ".lang.yaml") {
continue
}
// don't load our example files in practice
if strings.HasPrefix(lowerName, "example") {
continue
}
// load core info file
data, err = ioutil.ReadFile(filepath.Join(config.Languages.Path, name))
if err != nil {
return nil, fmt.Errorf("Could not load language file [%s]: %s", name, err.Error())
}
var langInfo languages.LangData
err = yaml.Unmarshal(data, &langInfo)
if err != nil {
return nil, fmt.Errorf("Could not parse language file [%s]: %s", name, err.Error())
}
langInfo.Translations = make(map[string]string)
// load actual translation files
var tlList map[string]string
// load irc strings file
ircName := strings.TrimSuffix(name, ".lang.yaml") + "-irc.lang.json"
data, err = ioutil.ReadFile(filepath.Join(config.Languages.Path, ircName))
if err == nil {
err = json.Unmarshal(data, &tlList)
if err != nil {
return nil, fmt.Errorf("Could not parse language's irc file [%s]: %s", ircName, err.Error())
}
for key, value := range tlList {
// because of how crowdin works, this is how we skip untranslated lines
if key == value || value == "" {
continue
}
langInfo.Translations[key] = value
}
}
// load help strings file
helpName := strings.TrimSuffix(name, ".lang.yaml") + "-help.lang.json"
data, err = ioutil.ReadFile(filepath.Join(config.Languages.Path, helpName))
if err == nil {
err = json.Unmarshal(data, &tlList)
if err != nil {
return nil, fmt.Errorf("Could not parse language's help file [%s]: %s", helpName, err.Error())
}
for key, value := range tlList {
// because of how crowdin works, this is how we skip untranslated lines
if key == value || value == "" {
continue
}
langInfo.Translations[key] = value
}
}
// confirm that values are correct
if langInfo.Code == "en" {
return nil, fmt.Errorf("Cannot have language file with code 'en' (this is the default language using strings inside the server code). If you're making an English variant, name it with a more specific code")
}
if len(langInfo.Translations) == 0 {
// skip empty translations
continue
}
if langInfo.Code == "" || langInfo.Name == "" || langInfo.Contributors == "" {
return nil, fmt.Errorf("Code, name or contributors is empty in language file [%s]", name)
}
// check for duplicate languages
_, exists := config.Languages.Data[strings.ToLower(langInfo.Code)]
if exists {
return nil, fmt.Errorf("Language code [%s] defined twice", langInfo.Code)
}
// and insert into lang info
config.Languages.Data[strings.ToLower(langInfo.Code)] = langInfo
}
// confirm that default language exists
if config.Languages.Default == "" {
config.Languages.Default = "en"
} else {
config.Languages.Default = strings.ToLower(config.Languages.Default)
}
_, exists := config.Languages.Data[config.Languages.Default]
if config.Languages.Default != "en" && !exists {
return nil, fmt.Errorf("Cannot find default language [%s]", config.Languages.Default)
}
config.languageManager, err = languages.NewManager(config.Languages.Enabled, config.Languages.Path, config.Languages.Default)
if err != nil {
return nil, fmt.Errorf("Could not load languages: %s", err.Error())
}
// RecoverFromErrors defaults to true

25
irc/getters.go
View File

@ -5,13 +5,15 @@ package irc
import (
"github.com/oragono/oragono/irc/isupport"
"github.com/oragono/oragono/irc/languages"
"github.com/oragono/oragono/irc/modes"
)
func (server *Server) Config() *Config {
func (server *Server) Config() (config *Config) {
server.configurableStateMutex.RLock()
defer server.configurableStateMutex.RUnlock()
return server.config
config = server.config
server.configurableStateMutex.RUnlock()
return
}
func (server *Server) ISupport() *isupport.List {
@ -58,6 +60,10 @@ func (server *Server) GetOperator(name string) (oper *Oper) {
return server.config.operators[name]
}
func (server *Server) Languages() (lm *languages.Manager) {
return server.Config().languageManager
}
func (client *Client) Nick() string {
client.stateMutex.RLock()
defer client.stateMutex.RUnlock()
@ -191,6 +197,19 @@ func (client *Client) SetAccountName(account string) (changed bool) {
return
}
func (client *Client) Languages() (languages []string) {
client.stateMutex.RLock()
languages = client.languages
client.stateMutex.RUnlock()
return languages
}
func (client *Client) SetLanguages(languages []string) {
client.stateMutex.Lock()
client.languages = languages
client.stateMutex.Unlock()
}
func (client *Client) HasMode(mode modes.Mode) bool {
// client.flags has its own synch
return client.flags.HasMode(mode)

38
irc/handlers.go
View File

@ -988,11 +988,7 @@ Get an explanation of <argument>, or "index" for a list of help topics.`), rb)
// handle index
if argument == "index" {
if client.HasMode(modes.Operator) {
client.sendHelp("HELP", GetHelpIndex(client.languages, HelpIndexOpers), rb)
} else {
client.sendHelp("HELP", GetHelpIndex(client.languages, HelpIndex), rb)
}
client.sendHelp("HELP", server.helpIndexManager.GetIndex(client.Languages(), client.HasMode(modes.Operator)), rb)
return false
}
@ -1088,7 +1084,7 @@ func infoHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Resp
rb.Add(nil, server.name, RPL_INFO, client.nick, line)
}
// show translators for languages other than good ole' regular English
tlines := server.languages.Translators()
tlines := server.Languages().Translators()
if 0 < len(tlines) {
rb.Add(nil, server.name, RPL_INFO, client.nick, client.t("Translators:"))
for _, line := range tlines {
@ -1422,12 +1418,14 @@ func klineHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Res
// LANGUAGE <code>{ <code>}
func languageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
nick := client.Nick()
alreadyDoneLanguages := make(map[string]bool)
var appliedLanguages []string
supportedLanguagesCount := server.languages.Count()
lm := server.Languages()
supportedLanguagesCount := lm.Count()
if supportedLanguagesCount < len(msg.Params) {
rb.Add(nil, client.server.name, ERR_TOOMANYLANGUAGES, client.nick, strconv.Itoa(supportedLanguagesCount), client.t("You specified too many languages"))
rb.Add(nil, client.server.name, ERR_TOOMANYLANGUAGES, nick, strconv.Itoa(supportedLanguagesCount), client.t("You specified too many languages"))
return false
}
@ -1441,9 +1439,9 @@ func languageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *
continue
}
_, exists := server.languages.Info[value]
_, exists := lm.Languages[value]
if !exists {
rb.Add(nil, client.server.name, ERR_NOLANGUAGE, client.nick, client.t("Languages are not supported by this server"))
rb.Add(nil, client.server.name, ERR_NOLANGUAGE, nick, fmt.Sprintf(client.t("Language %s is not supported by this server"), value))
return false
}
@ -1456,20 +1454,16 @@ func languageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *
appliedLanguages = append(appliedLanguages, value)
}
client.stateMutex.Lock()
if len(appliedLanguages) == 1 && appliedLanguages[0] == "en" {
// premature optimisation ahoy!
client.languages = []string{}
} else {
client.languages = appliedLanguages
var langsToSet []string
if !(len(appliedLanguages) == 1 && appliedLanguages[0] == "en") {
langsToSet = appliedLanguages
}
client.stateMutex.Unlock()
client.SetLanguages(langsToSet)
params := []string{client.nick}
for _, lang := range appliedLanguages {
params = append(params, lang)
}
params = append(params, client.t("Language preferences have been set"))
params := make([]string, len(appliedLanguages)+2)
params[0] = nick
copy(params[1:], appliedLanguages)
params[len(params)-1] = client.t("Language preferences have been set")
rb.Add(nil, client.server.name, RPL_YOURLANGUAGESARE, params...)

63
irc/help.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"sort"
"strings"
"sync"
"github.com/oragono/oragono/irc/languages"
)
@ -603,13 +604,15 @@ func modesTextGenerator(client *Client) string {
return client.t(cmodeHelpText) + "\n\n" + client.t(umodeHelpText)
}
// HelpIndex contains the list of all help topics for regular users.
var HelpIndex map[string]string
type HelpIndexManager struct {
sync.RWMutex // tier 1
// HelpIndexOpers contains the list of all help topics for opers.
var HelpIndexOpers map[string]string
langToIndex map[string]string
langToOperIndex map[string]string
}
// GenerateHelpIndex is used to generate HelpIndex.
// Returns: a map from language code to the help index in that language.
func GenerateHelpIndex(lm *languages.Manager, forOpers bool) map[string]string {
// generate the help entry lists
var commands, isupport, information []string
@ -658,10 +661,7 @@ Information:
newHelpIndex["en"] = fmt.Sprintf(defaultHelpIndex, commandsString, isupportString, informationString)
lm.RLock()
defer lm.RUnlock()
for langCode := range lm.Info {
for langCode := range lm.Languages {
translatedHelpIndex := lm.Translate([]string{langCode}, defaultHelpIndex)
if translatedHelpIndex != defaultHelpIndex {
newHelpIndex[langCode] = fmt.Sprintf(translatedHelpIndex, commandsString, isupportString, informationString)
@ -671,22 +671,16 @@ Information:
return newHelpIndex
}
// GenerateHelpIndices generates our help indexes and confirms we have HELP entries for every command.
func GenerateHelpIndices(lm *languages.Manager) error {
// startup check that we have HELP entries for every command
if len(HelpIndex) == 0 && len(HelpIndexOpers) == 0 {
for name := range Commands {
_, exists := Help[strings.ToLower(name)]
if !exists {
return fmt.Errorf("Help entry does not exist for command %s", name)
}
}
}
// GenerateIndices regenerates our help indexes for each currently enabled language.
func (hm *HelpIndexManager) GenerateIndices(lm *languages.Manager) {
// generate help indexes
HelpIndex = GenerateHelpIndex(lm, false)
HelpIndexOpers = GenerateHelpIndex(lm, true)
return nil
langToIndex := GenerateHelpIndex(lm, false)
langToOperIndex := GenerateHelpIndex(lm, true)
hm.Lock()
defer hm.Unlock()
hm.langToIndex = langToIndex
hm.langToOperIndex = langToOperIndex
}
// sendHelp sends the client help of the given string.
@ -709,13 +703,30 @@ func (client *Client) sendHelp(name string, text string, rb *ResponseBuffer) {
}
// GetHelpIndex returns the help index for the given language.
func GetHelpIndex(languages []string, helpIndex map[string]string) string {
func (hm *HelpIndexManager) GetIndex(languages []string, oper bool) string {
hm.RLock()
langToIndex := hm.langToIndex
if oper {
langToIndex = hm.langToOperIndex
}
hm.RUnlock()
for _, lang := range languages {
index, exists := helpIndex[lang]
index, exists := langToIndex[lang]
if exists {
return index
}
}
// 'en' always exists
return helpIndex["en"]
return langToIndex["en"]
}
func init() {
// startup check that we have HELP entries for every command
for name := range Commands {
_, exists := Help[strings.ToLower(name)]
if !exists {
panic(fmt.Sprintf("Help entry does not exist for command %s", name))
}
}
}

172
irc/languages/languages.go
View File

@ -4,10 +4,25 @@
package languages
import (
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"gopkg.in/yaml.v2"
)
const (
// for a language (e.g., `fi-FI`) to be supported
// it must have a metadata file named, e.g., `fi-FI.lang.yaml`
metadataFileSuffix = ".lang.yaml"
)
var (
stringsFileSuffixes = []string{"-irc.lang.json", "-help.lang.json", "-nickserv.lang.json", "-hostserv.lang.json", "-chanserv.lang.json"}
)
// LangData is the data contained in a language file.
@ -16,76 +31,144 @@ type LangData struct {
Code string
Contributors string
Incomplete bool
Translations map[string]string
}
// Manager manages our languages and provides translation abilities.
type Manager struct {
sync.RWMutex
Info map[string]LangData
Languages map[string]LangData
translations map[string]map[string]string
defaultLang string
}
// NewManager returns a new Manager.
func NewManager(defaultLang string, languageData map[string]LangData) *Manager {
lm := Manager{
Info: make(map[string]LangData),
func NewManager(enabled bool, path string, defaultLang string) (lm *Manager, err error) {
lm = &Manager{
Languages: make(map[string]LangData),
translations: make(map[string]map[string]string),
defaultLang: defaultLang,
}
// make fake "en" info
lm.Info["en"] = LangData{
lm.Languages["en"] = LangData{
Code: "en",
Name: "English",
Contributors: "Oragono contributors and the IRC community",
}
// load language data
for name, data := range languageData {
lm.Info[name] = data
// make sure we don't include empty translations
lm.translations[name] = make(map[string]string)
for key, value := range data.Translations {
if strings.TrimSpace(value) == "" {
continue
if enabled {
err = lm.loadData(path)
if err == nil {
// successful load, check that defaultLang is sane
_, ok := lm.Languages[lm.defaultLang]
if !ok {
err = fmt.Errorf("Cannot find default language [%s]", lm.defaultLang)
}
lm.translations[name][key] = value
}
} else {
lm.defaultLang = "en"
}
return &lm
return
}
func (lm *Manager) loadData(path string) (err error) {
files, err := ioutil.ReadDir(path)
if err != nil {
return
}
// 1. for each language that has a ${langcode}.lang.yaml in the languages path
// 2. load ${langcode}.lang.yaml
// 3. load ${langcode}-irc.lang.json and friends as the translations
for _, f := range files {
if f.IsDir() {
continue
}
// glob up *.lang.yaml in the directory
name := f.Name()
if !strings.HasSuffix(name, metadataFileSuffix) {
continue
}
prefix := strings.TrimSuffix(name, metadataFileSuffix)
// load, e.g., `zh-CN.lang.yaml`
var data []byte
data, err = ioutil.ReadFile(filepath.Join(path, name))
if err != nil {
return
}
var langInfo LangData
err = yaml.Unmarshal(data, &langInfo)
if err != nil {
return err
}
if langInfo.Code == "en" {
return fmt.Errorf("Cannot have language file with code 'en' (this is the default language using strings inside the server code). If you're making an English variant, name it with a more specific code")
}
// check for duplicate languages
_, exists := lm.Languages[strings.ToLower(langInfo.Code)]
if exists {
return fmt.Errorf("Language code [%s] defined twice", langInfo.Code)
}
// slurp up all translation files with `prefix` into a single translation map
translations := make(map[string]string)
for _, translationSuffix := range stringsFileSuffixes {
stringsFilePath := filepath.Join(path, prefix+translationSuffix)
data, err = ioutil.ReadFile(stringsFilePath)
if err != nil {
continue // skip missing paths
}
var tlList map[string]string
err = json.Unmarshal(data, &tlList)
if err != nil {
return fmt.Errorf("invalid json for translation file %s: %s", stringsFilePath, err.Error())
}
for key, value := range tlList {
// because of how crowdin works, this is how we skip untranslated lines
if key == value || strings.TrimSpace(value) == "" {
continue
}
translations[key] = value
}
}
if len(translations) == 0 {
// skip empty translations
continue
}
// sanity check the language definition from the yaml file
if langInfo.Code == "" || langInfo.Name == "" || langInfo.Contributors == "" {
return fmt.Errorf("Code, name or contributors is empty in language file [%s]", name)
}
key := strings.ToLower(langInfo.Code)
lm.Languages[key] = langInfo
lm.translations[key] = translations
}
return nil
}
// Default returns the default languages.
func (lm *Manager) Default() []string {
lm.RLock()
defer lm.RUnlock()
if lm.defaultLang == "" {
return []string{}
}
return []string{lm.defaultLang}
}
// Count returns how many languages we have.
func (lm *Manager) Count() int {
lm.RLock()
defer lm.RUnlock()
return len(lm.Info)
return len(lm.Languages)
}
// Translators returns the languages we have and the translators.
func (lm *Manager) Translators() []string {
lm.RLock()
defer lm.RUnlock()
var tlist sort.StringSlice
for _, info := range lm.Info {
for _, info := range lm.Languages {
if info.Code == "en" {
continue
}
@ -98,12 +181,9 @@ func (lm *Manager) Translators() []string {
// Codes returns the proper language codes for the given casefolded language codes.
func (lm *Manager) Codes(codes []string) []string {
lm.RLock()
defer lm.RUnlock()
var newCodes []string
for _, code := range codes {
info, exists := lm.Info[code]
info, exists := lm.Languages[code]
if exists {
newCodes = append(newCodes, info.Code)
}
@ -123,9 +203,6 @@ func (lm *Manager) Translate(languages []string, originalString string) string {
return originalString
}
lm.RLock()
defer lm.RUnlock()
for _, lang := range languages {
lang = strings.ToLower(lang)
if lang == "en" {
@ -149,3 +226,18 @@ func (lm *Manager) Translate(languages []string, originalString string) string {
// didn't find any translation
return originalString
}
func (lm *Manager) CapValue() string {
langCodes := make([]string, len(lm.Languages)+1)
langCodes[0] = strconv.Itoa(len(lm.Languages))
i := 1
for _, info := range lm.Languages {
codeToken := info.Code
if info.Incomplete {
codeToken = "~" + info.Code
}
langCodes[i] = codeToken
i += 1
}
return strings.Join(langCodes, ",")
}

45
irc/server.go
View File

@ -25,7 +25,6 @@ import (
"github.com/oragono/oragono/irc/caps"
"github.com/oragono/oragono/irc/connection_limits"
"github.com/oragono/oragono/irc/isupport"
"github.com/oragono/oragono/irc/languages"
"github.com/oragono/oragono/irc/logger"
"github.com/oragono/oragono/irc/modes"
"github.com/oragono/oragono/irc/sno"
@ -76,9 +75,9 @@ type Server struct {
connectionThrottler *connection_limits.Throttler
ctime time.Time
dlines *DLineManager
helpIndexManager HelpIndexManager
isupport *isupport.List
klines *KLineManager
languages *languages.Manager
listeners map[string]*ListenerWrapper
logger *logger.Manager
monitorManager *MonitorManager
@ -119,7 +118,6 @@ func NewServer(config *Config, logger *logger.Manager) (*Server, error) {
clients: NewClientManager(),
connectionLimiter: connection_limits.NewLimiter(),
connectionThrottler: connection_limits.NewThrottler(),
languages: languages.NewManager(config.Languages.Default, config.Languages.Data),
listeners: make(map[string]*ListenerWrapper),
logger: logger,
monitorManager: NewMonitorManager(),
@ -137,11 +135,6 @@ func NewServer(config *Config, logger *logger.Manager) (*Server, error) {
return nil, err
}
// generate help info
if err := GenerateHelpIndices(server.languages); err != nil {
return nil, err
}
// Attempt to clean up when receiving these signals.
signal.Notify(server.signals, ServerExitSignals...)
signal.Notify(server.rehashSignal, syscall.SIGHUP)
@ -468,11 +461,9 @@ func (server *Server) tryRegister(c *Client) {
// t returns the translated version of the given string, based on the languages configured by the client.
func (client *Client) t(originalString string) string {
// grab this mutex to protect client.languages
client.stateMutex.RLock()
defer client.stateMutex.RUnlock()
return client.server.languages.Translate(client.languages, originalString)
// TODO(slingamn) investigate a fast path for this, using an atomic load to see if translation is disabled
languages := client.Languages()
return client.server.Languages().Translate(languages, originalString)
}
// MOTD serves the Message of the Day.
@ -536,9 +527,10 @@ func (client *Client) getWhoisOf(target *Client, rb *ResponseBuffer) {
rb.Add(nil, client.server.name, RPL_WHOISBOT, cnick, tnick, ircfmt.Unescape(fmt.Sprintf(client.t("is a $bBot$b on %s"), client.server.Config().Network.Name)))
}
if 0 < len(target.languages) {
tLanguages := target.Languages()
if 0 < len(tLanguages) {
params := []string{cnick, tnick}
for _, str := range client.server.languages.Codes(target.languages) {
for _, str := range client.server.Languages().Codes(tLanguages) {
params = append(params, str)
}
params = append(params, client.t("can speak these languages"))
@ -655,31 +647,16 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) {
updatedCaps := caps.NewSet()
// Translations
server.logger.Debug("server", "Regenerating HELP indexes for new languages")
server.helpIndexManager.GenerateIndices(config.languageManager)
currentLanguageValue, _ := CapValues.Get(caps.Languages)
langCodes := []string{strconv.Itoa(len(config.Languages.Data) + 1), "en"}
for _, info := range config.Languages.Data {
if info.Incomplete {
langCodes = append(langCodes, "~"+info.Code)
} else {
langCodes = append(langCodes, info.Code)
}
}
newLanguageValue := strings.Join(langCodes, ",")
server.logger.Debug("server", "Languages:", newLanguageValue)
newLanguageValue := config.languageManager.CapValue()
if currentLanguageValue != newLanguageValue {
updatedCaps.Add(caps.Languages)
CapValues.Set(caps.Languages, newLanguageValue)
}
lm := languages.NewManager(config.Languages.Default, config.Languages.Data)
server.logger.Debug("server", "Regenerating HELP indexes for new languages")
GenerateHelpIndices(lm)
server.languages = lm
// SASL
authPreviouslyEnabled := oldConfig != nil && oldConfig.Accounts.AuthenticationEnabled
if config.Accounts.AuthenticationEnabled && !authPreviouslyEnabled {