Sergey Sharybin 03806d0b67 Re-design of submodules used in blender.git
This commit implements described in the #104573.

The goal is to fix the confusion of the submodule hashes change, which are not
ideal for any of the supported git-module configuration (they are either always
visible causing confusion, or silently staged and committed, also causing
confusion).

This commit replaces submodules with a checkout of addons and addons_contrib,
covered by the .gitignore, and locale and developer tools are moved to the
main repository.

This also changes the paths:
- /release/scripts are moved to the /scripts
- /source/tools are moved to the /tools
- /release/datafiles/locale is moved to /locale

This is done to avoid conflicts when using bisect, and also allow buildbot to
automatically "recover" wgen building older or newer branches/patches.

Running `make update` will initialize the local checkout to the changed
repository configuration.

Another aspect of the change is that the make update will support Github style
of remote organization (origin remote pointing to thy fork, upstream remote
pointing to the upstream blender/blender.git).

Pull Request #104755
2023-02-21 16:39:58 +01:00

176 lines
5.5 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-or-later
# Pre-process right-to-left languages.
# You can use it either standalone, or through import_po_from_branches or
# update_trunk.
#
# Notes: This has been tested on Linux, not 100% it will work nicely on
# Windows or OsX.
# This uses ctypes, as there is no py3 binding for fribidi currently.
# This implies you only need the compiled C library to run it.
# Finally, note that it handles some formatting/escape codes (like
# \", %s, %x12, %.4f, etc.), protecting them from ugly (evil) fribidi,
# which seems completely unaware of such things (as unicode is...).
import ctypes
import re
# define FRIBIDI_MASK_NEUTRAL 0x00000040L /* Is neutral */
FRIBIDI_PAR_ON = 0x00000040
# define FRIBIDI_FLAG_SHAPE_MIRRORING 0x00000001
# define FRIBIDI_FLAG_REORDER_NSM 0x00000002
# define FRIBIDI_FLAG_SHAPE_ARAB_PRES 0x00000100
# define FRIBIDI_FLAG_SHAPE_ARAB_LIGA 0x00000200
# define FRIBIDI_FLAG_SHAPE_ARAB_CONSOLE 0x00000400
# define FRIBIDI_FLAG_REMOVE_BIDI 0x00010000
# define FRIBIDI_FLAG_REMOVE_JOINING 0x00020000
# define FRIBIDI_FLAG_REMOVE_SPECIALS 0x00040000
# define FRIBIDI_FLAGS_DEFAULT ( \
# FRIBIDI_FLAG_SHAPE_MIRRORING | \
# FRIBIDI_FLAG_REORDER_NSM | \
# FRIBIDI_FLAG_REMOVE_SPECIALS )
# define FRIBIDI_FLAGS_ARABIC ( \
# FRIBIDI_FLAG_SHAPE_ARAB_PRES | \
# FRIBIDI_FLAG_SHAPE_ARAB_LIGA )
FRIBIDI_FLAG_SHAPE_MIRRORING = 0x00000001
FRIBIDI_FLAG_REORDER_NSM = 0x00000002
FRIBIDI_FLAG_REMOVE_SPECIALS = 0x00040000
FRIBIDI_FLAG_SHAPE_ARAB_PRES = 0x00000100
FRIBIDI_FLAG_SHAPE_ARAB_LIGA = 0x00000200
FRIBIDI_FLAGS_DEFAULT = FRIBIDI_FLAG_SHAPE_MIRRORING | FRIBIDI_FLAG_REORDER_NSM | FRIBIDI_FLAG_REMOVE_SPECIALS
FRIBIDI_FLAGS_ARABIC = FRIBIDI_FLAG_SHAPE_ARAB_PRES | FRIBIDI_FLAG_SHAPE_ARAB_LIGA
MENU_DETECT_REGEX = re.compile("%x\\d+\\|")
##### Kernel processing funcs. #####
def protect_format_seq(msg):
"""
Find some specific escaping/formatting sequences (like \", %s, etc.,
and protect them from any modification!
"""
# LRM = "\u200E"
# RLM = "\u200F"
LRE = "\u202A"
# RLE = "\u202B"
PDF = "\u202C"
LRO = "\u202D"
# RLO = "\u202E"
# uctrl = {LRE, RLE, PDF, LRO, RLO}
# Most likely incomplete, but seems to cover current needs.
format_codes = set("tslfd")
digits = set(".0123456789")
if not msg:
return msg
elif MENU_DETECT_REGEX.search(msg):
# An ugly "menu" message, just force it whole LRE if not yet done.
if msg[0] not in {LRE, LRO}:
msg = LRE + msg
idx = 0
ret = []
ln = len(msg)
while idx < ln:
dlt = 1
# # If we find a control char, skip any additional protection!
# if msg[idx] in uctrl:
# ret.append(msg[idx:])
# break
# \" or \'
if idx < (ln - 1) and msg[idx] == '\\' and msg[idx + 1] in "\"\'":
dlt = 2
# %x12|
elif idx < (ln - 2) and msg[idx] == '%' and msg[idx + 1] in "x" and msg[idx + 2] in digits:
dlt = 2
while (idx + dlt) < ln and msg[idx + dlt] in digits:
dlt += 1
if (idx + dlt) < ln and msg[idx + dlt] == '|':
dlt += 1
# %.4f
elif idx < (ln - 3) and msg[idx] == '%' and msg[idx + 1] in digits:
dlt = 2
while (idx + dlt) < ln and msg[idx + dlt] in digits:
dlt += 1
if (idx + dlt) < ln and msg[idx + dlt] in format_codes:
dlt += 1
else:
dlt = 1
# %s
elif idx < (ln - 1) and msg[idx] == '%' and msg[idx + 1] in format_codes:
dlt = 2
if dlt > 1:
ret.append(LRE)
ret += msg[idx:idx + dlt]
idx += dlt
if dlt > 1:
ret.append(PDF)
return "".join(ret)
def log2vis(msgs, settings):
"""
Globally mimics deprecated fribidi_log2vis.
msgs should be an iterable of messages to rtl-process.
"""
fbd = ctypes.CDLL(settings.FRIBIDI_LIB)
for msg in msgs:
msg = protect_format_seq(msg)
fbc_str = ctypes.create_unicode_buffer(msg)
ln = len(fbc_str) - 1
# print(fbc_str.value, ln)
btypes = (ctypes.c_int * ln)()
embed_lvl = (ctypes.c_uint8 * ln)()
pbase_dir = ctypes.c_int(FRIBIDI_PAR_ON)
jtypes = (ctypes.c_uint8 * ln)()
flags = FRIBIDI_FLAGS_DEFAULT | FRIBIDI_FLAGS_ARABIC
# Find out direction of each char.
fbd.fribidi_get_bidi_types(fbc_str, ln, ctypes.byref(btypes))
# print(*btypes)
fbd.fribidi_get_par_embedding_levels(btypes, ln,
ctypes.byref(pbase_dir),
embed_lvl)
# print(*embed_lvl)
# Joinings for arabic chars.
fbd.fribidi_get_joining_types(fbc_str, ln, jtypes)
# print(*jtypes)
fbd.fribidi_join_arabic(btypes, ln, embed_lvl, jtypes)
# print(*jtypes)
# Final Shaping!
fbd.fribidi_shape(flags, embed_lvl, ln, jtypes, fbc_str)
# print(fbc_str.value)
# print(*(ord(c) for c in fbc_str))
# And now, the reordering.
# Note that here, we expect a single line, so no need to do
# fancy things...
fbd.fribidi_reorder_line(flags, btypes, ln, 0, pbase_dir, embed_lvl,
fbc_str, None)
# print(fbc_str.value)
# print(*(ord(c) for c in fbc_str))
yield fbc_str.value