#!/usr/bin/env python # -*- coding: utf-8 -*- from re import compile as re_compile from json import dumps as json_encode from json import loads as json_decode from random import randint as random_integer if "LibreTranslatePlus" not in globals(): class LibreTranslatePlus:pass # KyMAN I18N JSON Files class Anonymous(LibreTranslatePlus.Abstracts.Models): # LibreTranslatePlus.Models.KJSON re_encode = re_compile(r'[\']') re_decode = re_compile(r'[`]') def _build(self, _input): self.__paths = self.settings("paths", _input) self.__cache = [] self.__main_language = self.settings(("main", "main_language"), _input) self.__print_variables["main_language"] = self.__main_language self.__main_key_language = self.ltp.i18n.get_key(self.__main_language)[0] self.__connection_key = self.settings(("connection_key", "connection"), _input) self.__separators = self.settings(("models_kjson_separators", "separators"), _input) self.__indent = self.settings(("models_kjson_indent", "indent"), _input) self.__ascii = self.settings(("models_kjson_ascii", "ascii"), _input) self.__cache_directory = self.ltp.get_root_directory() + "/" + self.settings(("models_kjson_cache_directory", "cache_directory"), _input) protected_fragments = self.settings(("models_kjson_protected_fragments", "protected_fragments"), _input) default_variables_pattern = self.settings(("models_kjson_default_variables_pattern", "default_variables_pattern"), _input) self.__key_name = self.settings(("models_kjson_key_name", "key_name"), _input) self.__splitted_sentences_length = self.settings(("models_kjson_splitted_sentences_minimum_length", "splitted_sentences_minimum_length"), _input) self.__fragment_maximum_length = self.settings(("models_kjson_fragment_maximum_length", "fragment_maximum_length"), _input) self.__default_language_key = self.settings(("models_kjson_default_language_key", "default_language_key"), _input) self.__language_names_keys = {} self.__re_variables = ( protected_fragments + r'|' if protected_fragments and isinstance(protected_fragments, str) else r'|'.join(protected_fragments) + r'|' if isinstance(protected_fragments, (list, tuple)) and len(protected_fragments) else r'' ) + ( default_variables_pattern if default_variables_pattern and isinstance(default_variables_pattern, str) else r'|'.join(default_variables_pattern) if isinstance(default_variables_pattern, (list, tuple)) and len(default_variables_pattern) else r'' ) self.__key_name_length = self.settings(("models_kjson_key_name_length", "key_name_length"), _input) self.__key_name_characters = self.settings(("models_kjson_key_name_characters", "key_name_characters"), _input) if self.__re_variables[-1] == "|": self.__re_variables = self.__re_variables[:-1] self.__re_variables = re_compile(self.__re_variables) self.__allow_blocks = self.settings(("models_kjson_allow_blocks", "allow_blocks"), _input) self.__show_load_cache_exception = self.settings(("models_kjson_show_load_cache_exception_message", "show_exception_message")) self.__show_load_cache_error = self.settings(("models_kjson_show_load_cache_error_message", "show_error_message")) self.__show_load_cache_ok = self.settings(("models_kjson_show_load_cache_ok_message", "show_ok_message")) self.__show_load_cache_info = self.settings(("models_kjson_show_load_cache_info_message", "show_info_message")) self.__show_key_processing = self.settings(("models_kjson_show_key_processing_message", "show_info_message")) self.__show_only_key_with_changes = self.settings("models_kjson_show_only_key_with_changes") self.__show_are_changes = self.settings(("models_kjson_show_are_changes_message", "show_info_message")) self.__show_saving_language = self.settings(("models_kjson_show_saving_language_message", "show_info_message")) self.__show_saving_cache = self.settings(("models_kjson_show_saving_cache_message", "show_info_message")) (self.__paths, error) = self.ltp.models.validate_paths(self.__paths, self.__main_key_language, False)[0:2] error |= ( (self.ltp.connections.has(self.__connection_key)[1] << 4) | (( 1 << 0 if self.__separators == None else 1 << 1 if not isinstance(self.__separators, (list, tuple)) else 1 << 2 if not len(self.__separators) else 1 << 3 if len(self.__separators) != 2 else (( (( 1 << 0 if self.__separators[0] == None else 1 << 1 if not isinstance(self.__separators[0], str) else 1 << 2 if not self.__separators[0] else 0) << 0) | (( 1 << 0 if self.__separators[1] == None else 1 << 1 if not isinstance(self.__separators[1], str) else 1 << 2 if not self.__separators[1] else 0) << 6) | 0) << 4) | 0) << 9) | (( 1 << 0 if self.__indent == None else 1 << 1 if not isinstance(self.__indent, int) else 1 << 2 if self.__indent < 0 else 0) << 19) | (( 1 << 0 if self.__ascii == None else 1 << 1 if not isinstance(self.__ascii, bool) else 0) << 22) | (( 1 << 0 if self.__main_language == None else 1 << 1 if not isinstance(self.__main_language, str) else 1 << 2 if not self.__main_language else 0) << 24) | (( 1 << 0 if self.__cache_directory == None else 1 << 1 if not isinstance(self.__cache_directory, str) else 1 << 2 if not self.__cache_directory else # 1 << 3 if not LibreTranslatePlus.path_exists(self.__cache_directory) else 0) << 27) | 0) self.__error |= error << 9 self.__error_messages += [ "paths_null", "paths_bad_type", "paths_empty", "paths_with_errors", "connection_key_null", "connection_key_not_string", "connection_key_empty", "connection_key_unknown", "connection_key_deleted", "separators_null", "separators_not_list", "separators_empty", "separators_not_2", "separator_item_null", "separator_item_not_string", "separator_item_empty", "separator_key_null", "separator_key_not_string", "separator_key_empty", "indent_null", "indent_not_integer", "indent_lower_0", "ascii_null", "ascii_not_boolean", "main_language_null", "main_language_not_string", "main_language_empty", "cache_directory_null", "cache_directory_not_string", "cache_directory_empty", "cache_directory_not_exists" ] if not self.__error: for i, path in enumerate(self.__paths): path = self.__cache_directory + "/" + self.__key + "." + str(i) + "." + self.__main_key_language + ".json" if LibreTranslatePlus.path_exists(path): main_data = self.ltp.load_file(path)[0] self.__cache.append({ "main_data" : main_data, self.__main_key_language : json_decode(main_data)[self.__main_key_language] }) self.__main_paths = [] if not self.__error: for path in self.__paths: self.__main_paths.append(LibreTranslatePlus.string_variables(path, {"key" : self.__main_key_language})) self.__load_caches() def __load_caches(self): self.__show_load_cache_info and self._print("info", "ltp_models_kjson_loading_caches", self.__print_variables) if self.__error: self.__show_load_cache_error and self._print("warn", "ltp_models_kjson_loading_caches_error", self.__print_variables) return languages = self.ltp.i18n.get_keys() for i, path in enumerate(self.__paths): full_path = self.__cache_directory + "/" + self.__key + "." + str(i) + "." + self.__main_key_language + ".json" error = ( 1 << 0 if full_path == None else 1 << 1 if not isinstance(full_path, str) else 1 << 2 if not full_path else 1 << 3 if not LibreTranslatePlus.path_exists(full_path) else 0) << 1 if not error: try: main_data = self.ltp.load_file(full_path)[0] self.__cache.append({ "main_data" : main_data, self.__main_key_language : json_decode(main_data)[self.__main_key_language] }) for language in languages: if language != self.__main_key_language: language_path = LibreTranslatePlus.string_variables(path, {"key" : language}) if LibreTranslatePlus.path_exists(language_path): self.__show_load_cache_info and self._print("info", "ltp_models_kjson_load_cache_language", { **self.__print_variables, "language_path" : language_path, "key" : language }) self.__cache[i][language] = self.ltp.load_json(language_path)[0][language] except Exception as exception: error |= 1 << 0 self.exception(exception, self.__show_load_cache_exception and "ltp_models_kjson_load_cache_exception", { **self.__print_variables, "path" : full_path }) self.validate( error, ( "exception", "cache_path_null", "cache_path_not_string", "cache_path_empty", "cache_path_not_exists" ), { **self.__print_variables, "path" : full_path }, self.__show_load_cache_error and "ltp_models_kjson_load_cache_error", self.__show_load_cache_ok and "ltp_models_kjson_load_cache_ok" ) self.__show_load_cache_ok and self._print("ok", "ltp_models_kjson_loaded_caches", self.__print_variables) def __working(self, i): thread = self.ltp.threads.get(i) return thread and thread["working"] def execute(self, thread_i): connection = self.ltp.connections.get(self.__connection_key)[0] for i, path in enumerate(self.__main_paths): if not self.__working(thread_i): break if len(self.__cache) <= i: self.__cache.append({"main_data" : None}) data = self.ltp.load_file(path)[0] if not data or self.__cache[i]["main_data"] == data: continue json = self.ltp.json_decode(data, False)[0] self.__cache[i]["main_data"] = data if not json: continue self.__show_are_changes and self._print("info", "ltp_models_kjson_are_changes", self.__print_variables) created = self.__main_key_language in self.__cache[i] if not created: self.__cache[i][self.__main_key_language] = {} if len(list(json[self.__main_key_language].keys())): sentences = json[self.__main_key_language].items() total = len(tuple(json[self.__main_key_language].keys())) languages = [(language["code"], language["key"]) for language in connection.languages()[0] if language["code"] != self.__main_language] new_state = {} j = 0 for key, sentence in sentences: if not self.__working(thread_i): break changes = ( key not in self.__cache[i][self.__main_key_language] or self.__cache[i][self.__main_key_language][key] != sentence ) self.__show_key_processing and self._print("info", "ltp_models_kjson_key_processing", { **self.__print_variables, "i18n_key" : key, "length" : ( len(sentence) if isinstance(sentence, str) else len("".join(sentence)) if isinstance(sentence, (list, tuple)) else None), "total" : total, "i" : j, "percentaje" : round(100.0 * j / float(total), 2) }) j += 1 for language_code, language_key in languages: if not self.__working(thread_i): break if language_key not in new_state: new_state[language_key] = {} if language_key not in self.__language_names_keys: self.__language_names_keys[language_key] = connection.translate(self.__key_name, self.__main_language, language_code)[0] new_state[language_key][key] = ( sentence if not isinstance(sentence, (str, list, tuple)) else self.__translate(connection, sentence, language_code, language_key) if ( changes or language_key not in self.__cache[i] or key not in self.__cache[i][language_key] ) else self.__cache[i][language_key][key] ) for language_code, language_key in languages: if not self.__working(thread_i): break if self.__show_saving_language and (changes or not self.__show_only_key_with_changes): self._print("info", "ltp_models_kjson_saving_language", { **self.__print_variables, "language_code" : language_code, "language_key" : language_key }) encoded = json_encode( {language_key : new_state[language_key]}, separators = self.__separators, indent = self.__indent, ensure_ascii = self.__ascii ) self.__cache[i][language_key] = new_state[language_key] if self.__allow_blocks: encoded = re_compile(r'("' + language_key + '" \: \{|_end" : null(, *)?)').sub(r'\1\n', encoded) self.ltp.save_file(LibreTranslatePlus.string_variables(self.__paths[i], { "key" : language_key }), encoded) self.__show_key_processing and self._print("ok", "ltp_models_kjson_keys_processed", self.__print_variables) if self.__working(thread_i): self.__show_saving_cache and self._print("info", "ltp_models_kjson_saving_cache", self.__print_variables) self.ltp.save_file(self.__cache_directory + "/" + self.__key + "." + str(i) + "." + self.__main_key_language + ".json", self.__cache[i]["main_data"]) self.__cache[i][self.__main_key_language] = json[self.__main_key_language] def __encode(self, sentence): variables = {} keys_used = [] characters_added = self.__key_name_length - len(self.__key_name) def variables_match(matches): # groups = matches.groups() key = "" l = len(self.__key_name_characters) - 1 while True: key = "" + self.__key_name while len(key) < self.__key_name_length: character = self.__key_name_characters[random_integer(0, l)] if key[-1] != character: key += character if key not in keys_used: keys_used.append(key) break variables[key] = matches.group(0) # return key + ("." if groups[-1] == "." else "") return key if isinstance(sentence, (list, tuple)): sentence = "".join(sentence) sentence = self.__re_variables.sub(variables_match, sentence) # if sentence[-len(self.__key_name):] == self.__key_name: # sentence += "." sentence = re_compile(r'([^\(\[\¡\¿"\'\s\r\n])(' + self.__key_name + r'[' + self.__key_name_characters + r']{' + str(characters_added) + r'})').sub(r'\1 \2', sentence) sentence = LibreTranslatePlus.Models.KJSON.re_encode.sub(lambda matches:{ "'" : "`" }[matches.group(0)], sentence) return (variables, sentence) def __decode(self, sentence, variables, language_key): characters_added = self.__key_name_length - len(self.__key_name) def variables_match(matches): key = matches.group(0) if key in variables: return variables[key] for subkey in tuple(variables.keys()): if key in subkey: return variables[subkey] return key sentence = LibreTranslatePlus.Models.KJSON.re_decode.sub(lambda matches:{ "`" : "'" }[matches.group(0)], sentence) sentence = re_compile(r'' + self.__key_name + r'[' + self.__key_name_characters + r']{0,' + str(characters_added) + r'}').sub(variables_match, sentence) return sentence def __fragment(self, sentence): if len(sentence) > self.__splitted_sentences_length: fragment = "" l = 0 total = [] for block in sentence.split(" "): m = len(block) if l + len(block) + 1 > self.__fragment_maximum_length: total.append(fragment) fragment = "" l = 0 fragment += block + " " l += m + 1 sentence = total + [fragment] if sentence[-1][-1] == " ": sentence[-1] = sentence[-1][:-1] return sentence def __translate(self, connection, sentence, code, key): (local_variables, encoded) = self.__encode(sentence) translated = connection.translate(encoded, self.__main_language, code)[0] or "" decoded = self.__decode(translated, local_variables, key) return self.__fragment(decoded) if not hasattr(LibreTranslatePlus, "Models"): class Subanonymous:pass LibreTranslatePlus.Models = Subanonymous del globals()["Subanonymous"] LibreTranslatePlus.Models.KJSON = Anonymous del globals()["Anonymous"]