diff --git a/irc/client.go b/irc/client.go index af91f03e..99d8df2e 100644 --- a/irc/client.go +++ b/irc/client.go @@ -131,6 +131,7 @@ func NewClient(server *Server, conn net.Conn, isTLS bool) { channels: make(ChannelSet), ctime: now, flags: modes.NewModeSet(), + languages: server.Languages().Default(), loginThrottle: connection_limits.GenericThrottle{ Duration: config.Accounts.LoginThrottling.Duration, Limit: config.Accounts.LoginThrottling.MaxAttempts, @@ -143,7 +144,6 @@ func NewClient(server *Server, conn net.Conn, isTLS bool) { nickMaskString: "*", // * is used until actual nick is given history: history.NewHistoryBuffer(config.History.ClientLength), } - client.languages = server.languages.Default() remoteAddr := conn.RemoteAddr() client.realIP = utils.AddrToIP(remoteAddr) diff --git a/irc/config.go b/irc/config.go index 112a5387..7adbe678 100644 --- a/irc/config.go +++ b/irc/config.go @@ -7,13 +7,11 @@ package irc import ( "crypto/tls" - "encoding/json" "fmt" "io/ioutil" "log" "net" "os" - "path/filepath" "regexp" "strings" "time" @@ -283,9 +281,10 @@ type Config struct { Enabled bool Path string Default string - Data map[string]languages.LangData } + languageManager *languages.Manager + Datastore struct { Path string AutoUpgrade bool @@ -638,120 +637,9 @@ func LoadConfig(filename string) (config *Config, err error) { } config.Server.MaxSendQBytes = int(maxSendQBytes) - // get language files - config.Languages.Data = make(map[string]languages.LangData) - if config.Languages.Enabled { - files, err := ioutil.ReadDir(config.Languages.Path) - if err != nil { - return nil, fmt.Errorf("Could not load language files: %s", err.Error()) - } - - for _, f := range files { - // skip dirs - if f.IsDir() { - continue - } - - // only load core .lang.yaml file, and ignore help/irc files - name := f.Name() - lowerName := strings.ToLower(name) - if !strings.HasSuffix(lowerName, ".lang.yaml") { - continue - } - // don't load our example files in practice - if strings.HasPrefix(lowerName, "example") { - continue - } - - // load core info file - data, err = ioutil.ReadFile(filepath.Join(config.Languages.Path, name)) - if err != nil { - return nil, fmt.Errorf("Could not load language file [%s]: %s", name, err.Error()) - } - - var langInfo languages.LangData - err = yaml.Unmarshal(data, &langInfo) - if err != nil { - return nil, fmt.Errorf("Could not parse language file [%s]: %s", name, err.Error()) - } - langInfo.Translations = make(map[string]string) - - // load actual translation files - var tlList map[string]string - - // load irc strings file - ircName := strings.TrimSuffix(name, ".lang.yaml") + "-irc.lang.json" - - data, err = ioutil.ReadFile(filepath.Join(config.Languages.Path, ircName)) - if err == nil { - err = json.Unmarshal(data, &tlList) - if err != nil { - return nil, fmt.Errorf("Could not parse language's irc file [%s]: %s", ircName, err.Error()) - } - - for key, value := range tlList { - // because of how crowdin works, this is how we skip untranslated lines - if key == value || value == "" { - continue - } - langInfo.Translations[key] = value - } - } - - // load help strings file - helpName := strings.TrimSuffix(name, ".lang.yaml") + "-help.lang.json" - - data, err = ioutil.ReadFile(filepath.Join(config.Languages.Path, helpName)) - if err == nil { - err = json.Unmarshal(data, &tlList) - if err != nil { - return nil, fmt.Errorf("Could not parse language's help file [%s]: %s", helpName, err.Error()) - } - - for key, value := range tlList { - // because of how crowdin works, this is how we skip untranslated lines - if key == value || value == "" { - continue - } - langInfo.Translations[key] = value - } - } - - // confirm that values are correct - if langInfo.Code == "en" { - return nil, fmt.Errorf("Cannot have language file with code 'en' (this is the default language using strings inside the server code). If you're making an English variant, name it with a more specific code") - } - - if len(langInfo.Translations) == 0 { - // skip empty translations - continue - } - - if langInfo.Code == "" || langInfo.Name == "" || langInfo.Contributors == "" { - return nil, fmt.Errorf("Code, name or contributors is empty in language file [%s]", name) - } - - // check for duplicate languages - _, exists := config.Languages.Data[strings.ToLower(langInfo.Code)] - if exists { - return nil, fmt.Errorf("Language code [%s] defined twice", langInfo.Code) - } - - // and insert into lang info - config.Languages.Data[strings.ToLower(langInfo.Code)] = langInfo - } - - // confirm that default language exists - if config.Languages.Default == "" { - config.Languages.Default = "en" - } else { - config.Languages.Default = strings.ToLower(config.Languages.Default) - } - - _, exists := config.Languages.Data[config.Languages.Default] - if config.Languages.Default != "en" && !exists { - return nil, fmt.Errorf("Cannot find default language [%s]", config.Languages.Default) - } + config.languageManager, err = languages.NewManager(config.Languages.Enabled, config.Languages.Path, config.Languages.Default) + if err != nil { + return nil, fmt.Errorf("Could not load languages: %s", err.Error()) } // RecoverFromErrors defaults to true diff --git a/irc/getters.go b/irc/getters.go index 1f6ff9cb..863137b7 100644 --- a/irc/getters.go +++ b/irc/getters.go @@ -5,13 +5,15 @@ package irc import ( "github.com/oragono/oragono/irc/isupport" + "github.com/oragono/oragono/irc/languages" "github.com/oragono/oragono/irc/modes" ) -func (server *Server) Config() *Config { +func (server *Server) Config() (config *Config) { server.configurableStateMutex.RLock() - defer server.configurableStateMutex.RUnlock() - return server.config + config = server.config + server.configurableStateMutex.RUnlock() + return } func (server *Server) ISupport() *isupport.List { @@ -58,6 +60,10 @@ func (server *Server) GetOperator(name string) (oper *Oper) { return server.config.operators[name] } +func (server *Server) Languages() (lm *languages.Manager) { + return server.Config().languageManager +} + func (client *Client) Nick() string { client.stateMutex.RLock() defer client.stateMutex.RUnlock() @@ -191,6 +197,19 @@ func (client *Client) SetAccountName(account string) (changed bool) { return } +func (client *Client) Languages() (languages []string) { + client.stateMutex.RLock() + languages = client.languages + client.stateMutex.RUnlock() + return languages +} + +func (client *Client) SetLanguages(languages []string) { + client.stateMutex.Lock() + client.languages = languages + client.stateMutex.Unlock() +} + func (client *Client) HasMode(mode modes.Mode) bool { // client.flags has its own synch return client.flags.HasMode(mode) diff --git a/irc/handlers.go b/irc/handlers.go index 90f345f1..dc46eb1b 100644 --- a/irc/handlers.go +++ b/irc/handlers.go @@ -988,11 +988,7 @@ Get an explanation of , or "index" for a list of help topics.`), rb) // handle index if argument == "index" { - if client.HasMode(modes.Operator) { - client.sendHelp("HELP", GetHelpIndex(client.languages, HelpIndexOpers), rb) - } else { - client.sendHelp("HELP", GetHelpIndex(client.languages, HelpIndex), rb) - } + client.sendHelp("HELP", server.helpIndexManager.GetIndex(client.Languages(), client.HasMode(modes.Operator)), rb) return false } @@ -1088,7 +1084,7 @@ func infoHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Resp rb.Add(nil, server.name, RPL_INFO, client.nick, line) } // show translators for languages other than good ole' regular English - tlines := server.languages.Translators() + tlines := server.Languages().Translators() if 0 < len(tlines) { rb.Add(nil, server.name, RPL_INFO, client.nick, client.t("Translators:")) for _, line := range tlines { @@ -1422,12 +1418,14 @@ func klineHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Res // LANGUAGE { } func languageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool { + nick := client.Nick() alreadyDoneLanguages := make(map[string]bool) var appliedLanguages []string - supportedLanguagesCount := server.languages.Count() + lm := server.Languages() + supportedLanguagesCount := lm.Count() if supportedLanguagesCount < len(msg.Params) { - rb.Add(nil, client.server.name, ERR_TOOMANYLANGUAGES, client.nick, strconv.Itoa(supportedLanguagesCount), client.t("You specified too many languages")) + rb.Add(nil, client.server.name, ERR_TOOMANYLANGUAGES, nick, strconv.Itoa(supportedLanguagesCount), client.t("You specified too many languages")) return false } @@ -1441,9 +1439,9 @@ func languageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb * continue } - _, exists := server.languages.Info[value] + _, exists := lm.Languages[value] if !exists { - rb.Add(nil, client.server.name, ERR_NOLANGUAGE, client.nick, client.t("Languages are not supported by this server")) + rb.Add(nil, client.server.name, ERR_NOLANGUAGE, nick, fmt.Sprintf(client.t("Language %s is not supported by this server"), value)) return false } @@ -1456,20 +1454,16 @@ func languageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb * appliedLanguages = append(appliedLanguages, value) } - client.stateMutex.Lock() - if len(appliedLanguages) == 1 && appliedLanguages[0] == "en" { - // premature optimisation ahoy! - client.languages = []string{} - } else { - client.languages = appliedLanguages + var langsToSet []string + if !(len(appliedLanguages) == 1 && appliedLanguages[0] == "en") { + langsToSet = appliedLanguages } - client.stateMutex.Unlock() + client.SetLanguages(langsToSet) - params := []string{client.nick} - for _, lang := range appliedLanguages { - params = append(params, lang) - } - params = append(params, client.t("Language preferences have been set")) + params := make([]string, len(appliedLanguages)+2) + params[0] = nick + copy(params[1:], appliedLanguages) + params[len(params)-1] = client.t("Language preferences have been set") rb.Add(nil, client.server.name, RPL_YOURLANGUAGESARE, params...) diff --git a/irc/help.go b/irc/help.go index e2a94e0f..b8c17d04 100644 --- a/irc/help.go +++ b/irc/help.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" "strings" + "sync" "github.com/oragono/oragono/irc/languages" ) @@ -603,13 +604,15 @@ func modesTextGenerator(client *Client) string { return client.t(cmodeHelpText) + "\n\n" + client.t(umodeHelpText) } -// HelpIndex contains the list of all help topics for regular users. -var HelpIndex map[string]string +type HelpIndexManager struct { + sync.RWMutex // tier 1 -// HelpIndexOpers contains the list of all help topics for opers. -var HelpIndexOpers map[string]string + langToIndex map[string]string + langToOperIndex map[string]string +} // GenerateHelpIndex is used to generate HelpIndex. +// Returns: a map from language code to the help index in that language. func GenerateHelpIndex(lm *languages.Manager, forOpers bool) map[string]string { // generate the help entry lists var commands, isupport, information []string @@ -658,10 +661,7 @@ Information: newHelpIndex["en"] = fmt.Sprintf(defaultHelpIndex, commandsString, isupportString, informationString) - lm.RLock() - defer lm.RUnlock() - - for langCode := range lm.Info { + for langCode := range lm.Languages { translatedHelpIndex := lm.Translate([]string{langCode}, defaultHelpIndex) if translatedHelpIndex != defaultHelpIndex { newHelpIndex[langCode] = fmt.Sprintf(translatedHelpIndex, commandsString, isupportString, informationString) @@ -671,22 +671,16 @@ Information: return newHelpIndex } -// GenerateHelpIndices generates our help indexes and confirms we have HELP entries for every command. -func GenerateHelpIndices(lm *languages.Manager) error { - // startup check that we have HELP entries for every command - if len(HelpIndex) == 0 && len(HelpIndexOpers) == 0 { - for name := range Commands { - _, exists := Help[strings.ToLower(name)] - if !exists { - return fmt.Errorf("Help entry does not exist for command %s", name) - } - } - } - +// GenerateIndices regenerates our help indexes for each currently enabled language. +func (hm *HelpIndexManager) GenerateIndices(lm *languages.Manager) { // generate help indexes - HelpIndex = GenerateHelpIndex(lm, false) - HelpIndexOpers = GenerateHelpIndex(lm, true) - return nil + langToIndex := GenerateHelpIndex(lm, false) + langToOperIndex := GenerateHelpIndex(lm, true) + + hm.Lock() + defer hm.Unlock() + hm.langToIndex = langToIndex + hm.langToOperIndex = langToOperIndex } // sendHelp sends the client help of the given string. @@ -709,13 +703,30 @@ func (client *Client) sendHelp(name string, text string, rb *ResponseBuffer) { } // GetHelpIndex returns the help index for the given language. -func GetHelpIndex(languages []string, helpIndex map[string]string) string { +func (hm *HelpIndexManager) GetIndex(languages []string, oper bool) string { + hm.RLock() + langToIndex := hm.langToIndex + if oper { + langToIndex = hm.langToOperIndex + } + hm.RUnlock() + for _, lang := range languages { - index, exists := helpIndex[lang] + index, exists := langToIndex[lang] if exists { return index } } // 'en' always exists - return helpIndex["en"] + return langToIndex["en"] +} + +func init() { + // startup check that we have HELP entries for every command + for name := range Commands { + _, exists := Help[strings.ToLower(name)] + if !exists { + panic(fmt.Sprintf("Help entry does not exist for command %s", name)) + } + } } diff --git a/irc/languages/languages.go b/irc/languages/languages.go index 39d43e73..34a33621 100644 --- a/irc/languages/languages.go +++ b/irc/languages/languages.go @@ -4,10 +4,25 @@ package languages import ( + "encoding/json" "fmt" + "io/ioutil" + "path/filepath" "sort" + "strconv" "strings" - "sync" + + "gopkg.in/yaml.v2" +) + +const ( + // for a language (e.g., `fi-FI`) to be supported + // it must have a metadata file named, e.g., `fi-FI.lang.yaml` + metadataFileSuffix = ".lang.yaml" +) + +var ( + stringsFileSuffixes = []string{"-irc.lang.json", "-help.lang.json", "-nickserv.lang.json", "-hostserv.lang.json", "-chanserv.lang.json"} ) // LangData is the data contained in a language file. @@ -16,76 +31,144 @@ type LangData struct { Code string Contributors string Incomplete bool - Translations map[string]string } // Manager manages our languages and provides translation abilities. type Manager struct { - sync.RWMutex - Info map[string]LangData + Languages map[string]LangData translations map[string]map[string]string defaultLang string } // NewManager returns a new Manager. -func NewManager(defaultLang string, languageData map[string]LangData) *Manager { - lm := Manager{ - Info: make(map[string]LangData), +func NewManager(enabled bool, path string, defaultLang string) (lm *Manager, err error) { + lm = &Manager{ + Languages: make(map[string]LangData), translations: make(map[string]map[string]string), defaultLang: defaultLang, } // make fake "en" info - lm.Info["en"] = LangData{ + lm.Languages["en"] = LangData{ Code: "en", Name: "English", Contributors: "Oragono contributors and the IRC community", } - // load language data - for name, data := range languageData { - lm.Info[name] = data - - // make sure we don't include empty translations - lm.translations[name] = make(map[string]string) - for key, value := range data.Translations { - if strings.TrimSpace(value) == "" { - continue + if enabled { + err = lm.loadData(path) + if err == nil { + // successful load, check that defaultLang is sane + _, ok := lm.Languages[lm.defaultLang] + if !ok { + err = fmt.Errorf("Cannot find default language [%s]", lm.defaultLang) } - lm.translations[name][key] = value } + } else { + lm.defaultLang = "en" } - return &lm + return +} + +func (lm *Manager) loadData(path string) (err error) { + files, err := ioutil.ReadDir(path) + if err != nil { + return + } + + // 1. for each language that has a ${langcode}.lang.yaml in the languages path + // 2. load ${langcode}.lang.yaml + // 3. load ${langcode}-irc.lang.json and friends as the translations + for _, f := range files { + if f.IsDir() { + continue + } + // glob up *.lang.yaml in the directory + name := f.Name() + if !strings.HasSuffix(name, metadataFileSuffix) { + continue + } + prefix := strings.TrimSuffix(name, metadataFileSuffix) + + // load, e.g., `zh-CN.lang.yaml` + var data []byte + data, err = ioutil.ReadFile(filepath.Join(path, name)) + if err != nil { + return + } + var langInfo LangData + err = yaml.Unmarshal(data, &langInfo) + if err != nil { + return err + } + + if langInfo.Code == "en" { + return fmt.Errorf("Cannot have language file with code 'en' (this is the default language using strings inside the server code). If you're making an English variant, name it with a more specific code") + } + + // check for duplicate languages + _, exists := lm.Languages[strings.ToLower(langInfo.Code)] + if exists { + return fmt.Errorf("Language code [%s] defined twice", langInfo.Code) + } + + // slurp up all translation files with `prefix` into a single translation map + translations := make(map[string]string) + for _, translationSuffix := range stringsFileSuffixes { + stringsFilePath := filepath.Join(path, prefix+translationSuffix) + data, err = ioutil.ReadFile(stringsFilePath) + if err != nil { + continue // skip missing paths + } + var tlList map[string]string + err = json.Unmarshal(data, &tlList) + if err != nil { + return fmt.Errorf("invalid json for translation file %s: %s", stringsFilePath, err.Error()) + } + + for key, value := range tlList { + // because of how crowdin works, this is how we skip untranslated lines + if key == value || strings.TrimSpace(value) == "" { + continue + } + translations[key] = value + } + } + + if len(translations) == 0 { + // skip empty translations + continue + } + + // sanity check the language definition from the yaml file + if langInfo.Code == "" || langInfo.Name == "" || langInfo.Contributors == "" { + return fmt.Errorf("Code, name or contributors is empty in language file [%s]", name) + } + + key := strings.ToLower(langInfo.Code) + lm.Languages[key] = langInfo + lm.translations[key] = translations + } + + return nil } // Default returns the default languages. func (lm *Manager) Default() []string { - lm.RLock() - defer lm.RUnlock() - - if lm.defaultLang == "" { - return []string{} - } return []string{lm.defaultLang} } // Count returns how many languages we have. func (lm *Manager) Count() int { - lm.RLock() - defer lm.RUnlock() - - return len(lm.Info) + return len(lm.Languages) } // Translators returns the languages we have and the translators. func (lm *Manager) Translators() []string { - lm.RLock() - defer lm.RUnlock() - var tlist sort.StringSlice - for _, info := range lm.Info { + for _, info := range lm.Languages { if info.Code == "en" { continue } @@ -98,12 +181,9 @@ func (lm *Manager) Translators() []string { // Codes returns the proper language codes for the given casefolded language codes. func (lm *Manager) Codes(codes []string) []string { - lm.RLock() - defer lm.RUnlock() - var newCodes []string for _, code := range codes { - info, exists := lm.Info[code] + info, exists := lm.Languages[code] if exists { newCodes = append(newCodes, info.Code) } @@ -123,9 +203,6 @@ func (lm *Manager) Translate(languages []string, originalString string) string { return originalString } - lm.RLock() - defer lm.RUnlock() - for _, lang := range languages { lang = strings.ToLower(lang) if lang == "en" { @@ -149,3 +226,18 @@ func (lm *Manager) Translate(languages []string, originalString string) string { // didn't find any translation return originalString } + +func (lm *Manager) CapValue() string { + langCodes := make([]string, len(lm.Languages)+1) + langCodes[0] = strconv.Itoa(len(lm.Languages)) + i := 1 + for _, info := range lm.Languages { + codeToken := info.Code + if info.Incomplete { + codeToken = "~" + info.Code + } + langCodes[i] = codeToken + i += 1 + } + return strings.Join(langCodes, ",") +} diff --git a/irc/server.go b/irc/server.go index fdf4f01e..1f2b78f8 100644 --- a/irc/server.go +++ b/irc/server.go @@ -25,7 +25,6 @@ import ( "github.com/oragono/oragono/irc/caps" "github.com/oragono/oragono/irc/connection_limits" "github.com/oragono/oragono/irc/isupport" - "github.com/oragono/oragono/irc/languages" "github.com/oragono/oragono/irc/logger" "github.com/oragono/oragono/irc/modes" "github.com/oragono/oragono/irc/sno" @@ -76,9 +75,9 @@ type Server struct { connectionThrottler *connection_limits.Throttler ctime time.Time dlines *DLineManager + helpIndexManager HelpIndexManager isupport *isupport.List klines *KLineManager - languages *languages.Manager listeners map[string]*ListenerWrapper logger *logger.Manager monitorManager *MonitorManager @@ -119,7 +118,6 @@ func NewServer(config *Config, logger *logger.Manager) (*Server, error) { clients: NewClientManager(), connectionLimiter: connection_limits.NewLimiter(), connectionThrottler: connection_limits.NewThrottler(), - languages: languages.NewManager(config.Languages.Default, config.Languages.Data), listeners: make(map[string]*ListenerWrapper), logger: logger, monitorManager: NewMonitorManager(), @@ -137,11 +135,6 @@ func NewServer(config *Config, logger *logger.Manager) (*Server, error) { return nil, err } - // generate help info - if err := GenerateHelpIndices(server.languages); err != nil { - return nil, err - } - // Attempt to clean up when receiving these signals. signal.Notify(server.signals, ServerExitSignals...) signal.Notify(server.rehashSignal, syscall.SIGHUP) @@ -468,11 +461,9 @@ func (server *Server) tryRegister(c *Client) { // t returns the translated version of the given string, based on the languages configured by the client. func (client *Client) t(originalString string) string { - // grab this mutex to protect client.languages - client.stateMutex.RLock() - defer client.stateMutex.RUnlock() - - return client.server.languages.Translate(client.languages, originalString) + // TODO(slingamn) investigate a fast path for this, using an atomic load to see if translation is disabled + languages := client.Languages() + return client.server.Languages().Translate(languages, originalString) } // MOTD serves the Message of the Day. @@ -536,9 +527,10 @@ func (client *Client) getWhoisOf(target *Client, rb *ResponseBuffer) { rb.Add(nil, client.server.name, RPL_WHOISBOT, cnick, tnick, ircfmt.Unescape(fmt.Sprintf(client.t("is a $bBot$b on %s"), client.server.Config().Network.Name))) } - if 0 < len(target.languages) { + tLanguages := target.Languages() + if 0 < len(tLanguages) { params := []string{cnick, tnick} - for _, str := range client.server.languages.Codes(target.languages) { + for _, str := range client.server.Languages().Codes(tLanguages) { params = append(params, str) } params = append(params, client.t("can speak these languages")) @@ -655,31 +647,16 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) { updatedCaps := caps.NewSet() // Translations + server.logger.Debug("server", "Regenerating HELP indexes for new languages") + server.helpIndexManager.GenerateIndices(config.languageManager) + currentLanguageValue, _ := CapValues.Get(caps.Languages) - - langCodes := []string{strconv.Itoa(len(config.Languages.Data) + 1), "en"} - for _, info := range config.Languages.Data { - if info.Incomplete { - langCodes = append(langCodes, "~"+info.Code) - } else { - langCodes = append(langCodes, info.Code) - } - } - newLanguageValue := strings.Join(langCodes, ",") - server.logger.Debug("server", "Languages:", newLanguageValue) - + newLanguageValue := config.languageManager.CapValue() if currentLanguageValue != newLanguageValue { updatedCaps.Add(caps.Languages) CapValues.Set(caps.Languages, newLanguageValue) } - lm := languages.NewManager(config.Languages.Default, config.Languages.Data) - - server.logger.Debug("server", "Regenerating HELP indexes for new languages") - GenerateHelpIndices(lm) - - server.languages = lm - // SASL authPreviouslyEnabled := oldConfig != nil && oldConfig.Accounts.AuthenticationEnabled if config.Accounts.AuthenticationEnabled && !authPreviouslyEnabled {