diff --git a/scripts/addons_core/ui_translate/update_repo.py b/scripts/addons_core/ui_translate/update_repo.py index 6ba68da5f63..299ca3df3b7 100644 --- a/scripts/addons_core/ui_translate/update_repo.py +++ b/scripts/addons_core/ui_translate/update_repo.py @@ -28,18 +28,6 @@ import tempfile # Operators ################################################################### -def i18n_updatetranslation_work_repo_callback(pot, lng, settings): - if not lng['use']: - return - if os.path.isfile(lng['po_path']): - po = utils_i18n.I18nMessages(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings) - po.update(pot) - else: - po = pot - po.write(kind="PO", dest=lng['po_path']) - print("{} PO written!".format(lng['uid'])) - - class UI_OT_i18n_updatetranslation_work_repo(Operator): """Update i18n working repository (po files)""" bl_idname = "ui.i18n_updatetranslation_work_repo" @@ -57,7 +45,8 @@ class UI_OT_i18n_updatetranslation_work_repo(Operator): i18n_sett = context.window_manager.i18n_update_settings self.settings.FILE_NAME_POT = i18n_sett.pot_path - context.window_manager.progress_begin(0, len(i18n_sett.langs) + 1) + num_langs = len(i18n_sett.langs) + context.window_manager.progress_begin(0, num_langs + 1) context.window_manager.progress_update(0) if not self.use_skip_pot_gen: env = os.environ.copy() @@ -86,16 +75,26 @@ class UI_OT_i18n_updatetranslation_work_repo(Operator): return {'CANCELLED'} # Now we should have a valid POT file, we have to merge it in all languages po's... + # NOTE: While on linux sub-processes are `os.fork`ed by default, + # on Windows and OSX they are `spawn`ed. + # See https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods + # This is a problem because spawned processes do not inherit the whole environment + # of the current (Blender-customized) python. In pratice, the `bpy` module won't load e.g. + # So care must be taken that the callback passed to the executor does not rely on any + # Blender-specific modules etc. This is why it is using a class method from `bl_i18n_utils` + # module, rather than a local function of this current Blender-only module. with concurrent.futures.ProcessPoolExecutor() as exctr: pot = utils_i18n.I18nMessages(kind='PO', src=self.settings.FILE_NAME_POT, settings=self.settings) - num_langs = len(i18n_sett.langs) - for progress, _ in enumerate(exctr.map(i18n_updatetranslation_work_repo_callback, - (pot,) * num_langs, - [dict(lng.items()) for lng in i18n_sett.langs], - (self.settings,) * num_langs, - chunksize=4)): + for progress, _ in enumerate( + exctr.map(utils_i18n.I18nMessages.update_from_pot_callback, + (pot,) * num_langs, + [dict(lng.items()) for lng in i18n_sett.langs], + (self.settings,) * num_langs, + chunksize=4)): context.window_manager.progress_update(progress + 2) + context.window_manager.progress_end() + print("", flush=True) return {'FINISHED'} def invoke(self, context, event): @@ -103,19 +102,6 @@ class UI_OT_i18n_updatetranslation_work_repo(Operator): return wm.invoke_props_dialog(self) -def i18n_cleanuptranslation_work_repo_callback(lng, settings): - if not lng['use']: - print("Skipping {} language ({}).".format(lng['name'], lng['uid'])) - return - po = utils_i18n.I18nMessages(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings) - errs = po.check(fix=True) - cleanedup_commented = po.clean_commented() - po.write(kind="PO", dest=lng['po_path']) - print("Processing {} language ({}).\n" - "Cleaned up {} commented messages.\n".format(lng['name'], lng['uid'], cleanedup_commented) + - ("Errors in this po, solved as best as possible!\n\t" + "\n\t".join(errs) if errs else "") + "\n") - - class UI_OT_i18n_cleanuptranslation_work_repo(Operator): """Clean up i18n working repository (po files)""" bl_idname = "ui.i18n_cleanuptranslation_work_repo" @@ -128,43 +114,24 @@ class UI_OT_i18n_cleanuptranslation_work_repo(Operator): # 'DEFAULT' and en_US are always valid, fully-translated "languages"! stats = {"DEFAULT": 1.0, "en_US": 1.0} - context.window_manager.progress_begin(0, len(i18n_sett.langs) + 1) + num_langs = len(i18n_sett.langs) + context.window_manager.progress_begin(0, num_langs + 1) context.window_manager.progress_update(0) + # NOTE: See comment in #UI_OT_i18n_updatetranslation_work_repo `execute` function about usage caveats + # of the `ProcessPoolExecutor`. with concurrent.futures.ProcessPoolExecutor() as exctr: - num_langs = len(i18n_sett.langs) - for progress, _ in enumerate(exctr.map(i18n_cleanuptranslation_work_repo_callback, - [dict(lng.items()) for lng in i18n_sett.langs], - (self.settings,) * num_langs, - chunksize=4)): + for progress, _ in enumerate( + exctr.map(utils_i18n.I18nMessages.cleanup_callback, + [dict(lng.items()) for lng in i18n_sett.langs], + (self.settings,) * num_langs, + chunksize=4)): context.window_manager.progress_update(progress + 1) context.window_manager.progress_end() - + print("", flush=True) return {'FINISHED'} -def i18n_updatetranslation_blender_repo_callback(lng, settings): - reports = [] - if lng['uid'] in settings.IMPORT_LANGUAGES_SKIP: - reports.append( - "Skipping {} language ({}), edit settings if you want to enable it.".format( - lng['name'], lng['uid'])) - return lng['uid'], 0.0, reports - if not lng['use']: - reports.append("Skipping {} language ({}).".format(lng['name'], lng['uid'])) - return lng['uid'], 0.0, reports - po = utils_i18n.I18nMessages(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings) - errs = po.check(fix=True) - reports.append("Processing {} language ({}).\n" - "Cleaned up {} commented messages.\n".format(lng['name'], lng['uid'], po.clean_commented()) + - ("Errors in this po, solved as best as possible!\n\t" + "\n\t".join(errs) if errs else "")) - if lng['uid'] in settings.IMPORT_LANGUAGES_RTL: - po.rtl_process() - po.write(kind="PO_COMPACT", dest=lng['po_path_blender']) - po.update_info() - return lng['uid'], po.nbr_trans_msgs / po.nbr_msgs, reports - - class UI_OT_i18n_updatetranslation_blender_repo(Operator): """Update i18n data (po files) in Blender source code repository""" bl_idname = "ui.i18n_updatetranslation_blender_repo" @@ -177,17 +144,22 @@ class UI_OT_i18n_updatetranslation_blender_repo(Operator): # 'DEFAULT' and en_US are always valid, fully-translated "languages"! stats = {"DEFAULT": 1.0, "en_US": 1.0} - context.window_manager.progress_begin(0, len(i18n_sett.langs) + 1) + num_langs = len(i18n_sett.langs) + context.window_manager.progress_begin(0, num_langs + 1) context.window_manager.progress_update(0) + # NOTE: See comment in #UI_OT_i18n_updatetranslation_work_repo `execute` function about usage caveats + # of the `ProcessPoolExecutor`. with concurrent.futures.ProcessPoolExecutor() as exctr: - num_langs = len(i18n_sett.langs) - for progress, (lng_uid, stats_val, reports) in enumerate(exctr.map(i18n_updatetranslation_blender_repo_callback, [ - dict(lng.items()) for lng in i18n_sett.langs], (self.settings,) * num_langs, chunksize=4)): + for progress, (lng_uid, stats_val, reports) in enumerate( + exctr.map(utils_i18n.I18nMessages.update_to_blender_repo_callback, + [dict(lng.items()) for lng in i18n_sett.langs], + (self.settings,) * num_langs, + chunksize=4)): context.window_manager.progress_update(progress + 1) stats[lng_uid] = stats_val print("".join(reports) + "\n") - print("Generating languages' menu...") + print("Generating languages' menu...", flush=True) context.window_manager.progress_update(progress + 2) languages_menu_lines = utils_languages_menu.gen_menu_file(stats, self.settings) with open(os.path.join(self.settings.BLENDER_I18N_ROOT, self.settings.LANGUAGES_FILE), 'w', encoding="utf8") as f: @@ -239,8 +211,9 @@ class UI_OT_i18n_updatetranslation_statistics(Operator): data = data + "\n" + buff.getvalue() text.from_string(data) self.report({'INFO'}, "Info written to %s text datablock!" % self.report_name) - context.window_manager.progress_end() + context.window_manager.progress_end() + print("", flush=True) return {'FINISHED'} def invoke(self, context, event): diff --git a/scripts/modules/bl_i18n_utils/bl_extract_messages.py b/scripts/modules/bl_i18n_utils/bl_extract_messages.py index d295e3fa479..15bca61f8f7 100644 --- a/scripts/modules/bl_i18n_utils/bl_extract_messages.py +++ b/scripts/modules/bl_i18n_utils/bl_extract_messages.py @@ -896,6 +896,7 @@ def dump_src_messages(msgs, reports, settings): rel_path = os.path.relpath(path, settings.SOURCE_DIR) except ValueError: rel_path = path + rel_path = PurePath(rel_path).as_posix() if rel_path in forbidden: continue elif rel_path not in forced: diff --git a/scripts/modules/bl_i18n_utils/settings.py b/scripts/modules/bl_i18n_utils/settings.py index be5d90e352a..b5d5302a167 100644 --- a/scripts/modules/bl_i18n_utils/settings.py +++ b/scripts/modules/bl_i18n_utils/settings.py @@ -14,10 +14,10 @@ import os import sys import types +# Only do soft-dependency on `bpy` module, not real strong need for it currently. try: import bpy except ModuleNotFoundError: - print("Could not import bpy, some features are not available when not run from Blender.") bpy = None ############################################################################### @@ -93,7 +93,7 @@ LANGUAGES = ( (54, "Slovenian (Slovenščina)", "sl"), ) -# Default context, in py (keep in sync with `BLT_translation.h`)! +# Default context, in py (keep in sync with `BLT_translation.hh`)! if bpy is not None: assert bpy.app.translations.contexts.default == "*" DEFAULT_CONTEXT = "*" diff --git a/scripts/modules/bl_i18n_utils/utils.py b/scripts/modules/bl_i18n_utils/utils.py index 0f75146f4e5..ec3f811caf3 100644 --- a/scripts/modules/bl_i18n_utils/utils.py +++ b/scripts/modules/bl_i18n_utils/utils.py @@ -6,6 +6,7 @@ import collections import os +import platform import re import struct import tempfile @@ -1047,7 +1048,9 @@ class I18nMessages: msgstr_lines.append(line) else: self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope")) - # self.parsing_errors += (str(comment_lines), str(msgctxt_lines), str(msgid_lines), str(msgstr_lines)) + # self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope:\n\t\t{}" + # "\n\tComments:{}\n\tContext:{}\n\tKey:{}\n\tMessage:{}".format( + # line, comment_lines, msgctxt_lines, msgid_lines, msgstr_lines))) # If no final empty line, last message is not finalized! if reading_msgstr: @@ -1201,6 +1204,80 @@ class I18nMessages: "MO": write_messages_to_mo, } + @classmethod + def update_from_pot_callback(cls, pot, lng, settings): + """ + Update or create a single PO file (specified by a filepath) from the given POT `I18nMessages` data. + + Callback usable in a context where Blender specific modules (like `bpy`) are not available. + """ + import sys + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") + if not lng['use']: + return + if os.path.isfile(lng['po_path']): + po = cls(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings) + po.update(pot) + else: + po = pot + po.write(kind="PO", dest=lng['po_path']) + print("{} PO written!".format(lng['uid'])) + + @classmethod + def cleanup_callback(cls, lng, settings): + """ + Cleanup a single PO file (specified by a filepath). + + Callback usable in a context where Blender specific modules (like `bpy`) are not available. + """ + import sys + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") + if not lng['use']: + return + po = cls(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings) + errs = po.check(fix=True) + cleanedup_commented = po.clean_commented() + po.write(kind="PO", dest=lng['po_path']) + print("Processing {} language ({}).\n" + "Cleaned up {} commented messages.\n".format(lng['name'], lng['uid'], cleanedup_commented) + + ("Errors in this po, solved as best as possible!\n\t" + "\n\t".join(errs) if errs else "") + "\n") + + @classmethod + def update_to_blender_repo_callback(cls, lng, settings): + """ + Cleanup and write a single PO file (specified by a filepath) into the relevant Blender source 'compact' PO file. + + Callback usable in a context where Blender specific modules (like `bpy`) are not available. + """ + import sys + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") + reports = [] + if lng['uid'] in settings.IMPORT_LANGUAGES_SKIP: + reports.append( + "Skipping {} language ({}), edit settings if you want to enable it.".format( + lng['name'], lng['uid'])) + return lng['uid'], 0.0, reports + if not lng['use']: + reports.append("Skipping {} language ({}).".format(lng['name'], lng['uid'])) + return lng['uid'], 0.0, reports + po = cls(uid=lng['uid'], kind='PO', src=lng['po_path'], settings=settings) + errs = po.check(fix=True) + reports.append("Processing {} language ({}).\n" + "Cleaned up {} commented messages.\n".format(lng['name'], lng['uid'], po.clean_commented()) + + ("Errors in this po, solved as best as possible!\n\t" + "\n\t".join(errs) if errs else "")) + if lng['uid'] in settings.IMPORT_LANGUAGES_RTL: + if platform.system not in {'Linux'}: + reports.append("Skipping RtL processing of {} language ({}), " + "this is only supported on Linux currently.".format(lng['name'], lng['uid'])) + else: + po.rtl_process() + po.write(kind="PO_COMPACT", dest=lng['po_path_blender']) + po.update_info() + return lng['uid'], po.nbr_trans_msgs / po.nbr_msgs, reports + class I18n: """