1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ruby/prism] Add GB1988 encoding

Browse Source 
https://github.com/ruby/prism/commit/78d3fa7172
This commit is contained in:
Orhan Toy 2023-11-19 00:37:29 +01:00 committed by git
parent 71a30b7596
commit 5d2d0962cc
4 changed files with 38 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
prism/enc/pm_encoding.h
View File

@ -164,6 +164,7 @@ extern pm_encoding_t pm_encoding_cp850;
extern pm_encoding_t pm_encoding_cp852;
extern pm_encoding_t pm_encoding_cp855;
extern pm_encoding_t pm_encoding_euc_jp;
extern pm_encoding_t pm_encoding_gb1988;
extern pm_encoding_t pm_encoding_gbk;
extern pm_encoding_t pm_encoding_ibm437;
extern pm_encoding_t pm_encoding_ibm720;

35
prism/enc/pm_tables.c
View File

@ -96,6 +96,30 @@ static uint8_t pm_encoding_cp855_table[256] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Fx
};
/**
* Each element of the following table contains a bitfield that indicates a
* piece of information about the corresponding GB1988 character.
*/
static uint8_t pm_encoding_gb1988_table[256] = {
// 0 1 2 3 4 5 6 7 8 9 A B C D E F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2x
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, // 3x
0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // 4x
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, 0, 0, // 5x
0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 6x
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, // 7x
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8x
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9x
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Ax
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Bx
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Cx
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Dx
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Ex
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Fx
};
/**
* Each element of the following table contains a bitfield that indicates a
* piece of information about the corresponding IBM437 character.
@ -1268,6 +1292,7 @@ pm_encoding_koi8_r_char_width(const uint8_t *b, PRISM_ATTRIBUTE_UNUSED ptrdiff_t
PRISM_ENCODING_TABLE(cp850)
PRISM_ENCODING_TABLE(cp852)
PRISM_ENCODING_TABLE(cp855)
PRISM_ENCODING_TABLE(gb1988)
PRISM_ENCODING_TABLE(ibm437)
PRISM_ENCODING_TABLE(ibm720)
PRISM_ENCODING_TABLE(ibm737)
@ -1367,6 +1392,16 @@ pm_encoding_t pm_encoding_cp855 = {
.multibyte = false
};
/** GB1988 */
pm_encoding_t pm_encoding_gb1988 = {
.name = "GB1988",
.char_width = pm_encoding_single_char_width,
.alnum_char = pm_encoding_gb1988_alnum_char,
.alpha_char = pm_encoding_gb1988_alpha_char,
.isupper_char = pm_encoding_gb1988_isupper_char,
.multibyte = false
};
/** IBM437 */
pm_encoding_t pm_encoding_ibm437 = {
.name = "IBM437",

1
prism/prism.c
View File

@ -6101,6 +6101,7 @@ parser_lex_magic_comment_encoding_value(pm_parser_t *parser, const uint8_t *star
ENCODING1("filesystem", pm_encoding_utf_8);
break;
case 'G': case 'g':
ENCODING1("GB1988", pm_encoding_gb1988);
ENCODING1("GBK", pm_encoding_gbk);
break;
case 'I': case 'i':

1
test/prism/encoding_test.rb
View File

@ -12,6 +12,7 @@ module Prism
Encoding::CP850 => 0x00...0x100,
Encoding::CP852 => 0x00...0x100,
Encoding::CP855 => 0x00...0x100,
Encoding::GB1988 => 0x00...0x100,
Encoding::IBM437 => 0x00...0x100,
Encoding::IBM720 => 0x00...0x100,
Encoding::IBM737 => 0x00...0x100,