2020-04-10 14:11:40 +09:00
|
|
|
#ifndef INTERNAL_PARSE_H /*-*-C-*-vi:se ft=c:*/
|
|
|
|
#define INTERNAL_PARSE_H
|
|
|
|
/**
|
2020-04-08 13:28:13 +09:00
|
|
|
* @author Ruby developers <ruby-core@ruby-lang.org>
|
2019-11-29 15:18:34 +09:00
|
|
|
* @copyright This file is a part of the programming language Ruby.
|
|
|
|
* Permission is hereby granted, to either redistribute and/or
|
|
|
|
* modify this file, provided that the conditions mentioned in the
|
|
|
|
* file COPYING are met. Consult the file for details.
|
2020-04-08 13:28:13 +09:00
|
|
|
* @brief Internal header for the parser.
|
2019-11-29 15:18:34 +09:00
|
|
|
*/
|
2023-05-28 20:00:20 +09:00
|
|
|
#include <limits.h>
|
|
|
|
#include "rubyparser.h"
|
|
|
|
#include "internal/static_assert.h"
|
|
|
|
|
2024-08-27 19:15:37 -04:00
|
|
|
// The default parser to use for Ruby code.
|
2024-10-02 19:08:54 +09:00
|
|
|
typedef enum {
|
|
|
|
RB_DEFAULT_PARSER_PARSE_Y,
|
|
|
|
RB_DEFAULT_PARSER_PRISM,
|
|
|
|
} ruby_default_parser_enum;
|
|
|
|
|
|
|
|
ruby_default_parser_enum rb_ruby_default_parser(void);
|
|
|
|
void rb_ruby_default_parser_set(ruby_default_parser_enum parser);
|
|
|
|
|
|
|
|
#define rb_ruby_prism_p() (rb_ruby_default_parser() == RB_DEFAULT_PARSER_PRISM)
|
2024-08-27 19:15:37 -04:00
|
|
|
|
2023-05-28 20:00:20 +09:00
|
|
|
#ifdef UNIVERSAL_PARSER
|
Constify encoding type in universal parser
Fixed warning about discarding modifiers.
```
../src/ruby_parser.c:677:48: warning: passing 'rb_encoding *' (aka 'const struct OnigEncodingTypeST *') to parameter of type 'void *' discards qualifiers [-Wincompatible-pointer-types-discards-qualifiers]
677 | ast = rb_parser_compile(p, gets, ptr, len, enc, input, line);
| ^~~
../src/internal/parse.h:58:128: note: passing argument to parameter 'fname_enc' here
58 | rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, const char *fname_ptr, long fname_len, rb_encoding *fname_enc, rb_parser_input_data input, int line);
| ^
```
2024-05-12 23:12:02 +09:00
|
|
|
#define rb_encoding const void
|
2023-05-28 20:00:20 +09:00
|
|
|
#endif
|
|
|
|
|
2019-12-03 14:47:38 +09:00
|
|
|
struct rb_iseq_struct; /* in vm_core.h */
|
|
|
|
|
2023-05-28 20:00:20 +09:00
|
|
|
/* structs for managing terminator of string literal and heredocment */
|
|
|
|
typedef struct rb_strterm_literal_struct {
|
2023-10-13 03:33:48 +09:00
|
|
|
long nest;
|
2023-10-13 19:13:26 +09:00
|
|
|
int func; /* STR_FUNC_* (e.g., STR_FUNC_ESCAPE and STR_FUNC_EXPAND) */
|
|
|
|
int paren; /* '(' of `%q(...)` */
|
|
|
|
int term; /* ')' of `%q(...)` */
|
2023-05-28 20:00:20 +09:00
|
|
|
} rb_strterm_literal_t;
|
|
|
|
|
|
|
|
typedef struct rb_strterm_heredoc_struct {
|
2024-01-12 22:26:54 +09:00
|
|
|
rb_parser_string_t *lastline; /* the string of line that contains `<<"END"` */
|
2023-05-28 20:00:20 +09:00
|
|
|
long offset; /* the column of END in `<<"END"` */
|
|
|
|
int sourceline; /* lineno of the line that contains `<<"END"` */
|
2023-10-13 03:33:48 +09:00
|
|
|
unsigned length; /* the length of END in `<<"END"` */
|
2023-05-28 20:00:20 +09:00
|
|
|
uint8_t quote;
|
|
|
|
uint8_t func;
|
|
|
|
} rb_strterm_heredoc_t;
|
|
|
|
|
2023-10-13 03:33:48 +09:00
|
|
|
#define HERETERM_LENGTH_MAX UINT_MAX
|
|
|
|
|
2023-05-28 20:00:20 +09:00
|
|
|
typedef struct rb_strterm_struct {
|
2024-04-01 15:46:58 +09:00
|
|
|
bool heredoc;
|
2023-05-28 20:00:20 +09:00
|
|
|
union {
|
|
|
|
rb_strterm_literal_t literal;
|
|
|
|
rb_strterm_heredoc_t heredoc;
|
|
|
|
} u;
|
|
|
|
} rb_strterm_t;
|
|
|
|
|
2019-12-03 14:47:38 +09:00
|
|
|
/* parse.y */
|
2023-05-28 20:00:20 +09:00
|
|
|
void rb_ruby_parser_mark(void *ptr);
|
|
|
|
size_t rb_ruby_parser_memsize(const void *ptr);
|
|
|
|
|
|
|
|
void rb_ruby_parser_set_options(rb_parser_t *p, int print, int loop, int chomp, int split);
|
|
|
|
rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_struct *base, int main);
|
2024-03-28 10:26:42 +09:00
|
|
|
void rb_ruby_parser_set_script_lines(rb_parser_t *p);
|
2023-05-28 20:00:20 +09:00
|
|
|
void rb_ruby_parser_error_tolerant(rb_parser_t *p);
|
|
|
|
void rb_ruby_parser_keep_tokens(rb_parser_t *p);
|
2024-04-25 16:09:42 +09:00
|
|
|
typedef rb_parser_string_t*(rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int);
|
2024-05-31 17:30:21 +09:00
|
|
|
rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line);
|
2023-06-12 20:06:19 +09:00
|
|
|
|
|
|
|
RUBY_SYMBOL_EXPORT_BEGIN
|
|
|
|
|
2024-04-23 11:27:42 +09:00
|
|
|
rb_encoding *rb_ruby_parser_encoding(rb_parser_t *p);
|
2023-05-28 20:00:20 +09:00
|
|
|
int rb_ruby_parser_end_seen_p(rb_parser_t *p);
|
|
|
|
int rb_ruby_parser_set_yydebug(rb_parser_t *p, int flag);
|
2024-01-12 22:26:54 +09:00
|
|
|
rb_parser_string_t *rb_str_to_parser_string(rb_parser_t *p, VALUE str);
|
2024-09-23 13:55:37 -04:00
|
|
|
void rb_parser_string_free(rb_parser_t *p, rb_parser_string_t *str);
|
2023-05-28 20:00:20 +09:00
|
|
|
|
[Feature #20257] Rearchitect Ripper
Introduce another semantic value stack for Ripper so that
Ripper can manage both Node and Ruby Object separately.
This rearchitectutre of Ripper solves these issues.
Therefore adding test cases for them.
* [Bug 10436] https://bugs.ruby-lang.org/issues/10436
* [Bug 18988] https://bugs.ruby-lang.org/issues/18988
* [Bug 20055] https://bugs.ruby-lang.org/issues/20055
Checked the differences of `Ripper.sexp` for files under `/test/ruby`
are only on test_pattern_matching.rb.
The differences comes from the differences between
`new_hash_pattern_tail` functions between parser and Ripper.
Ripper `new_hash_pattern_tail` didn’t call `assignable` then
`kw_rest_arg` wasn’t marked as local variable.
This is also fixed by this commit.
```
--- a/./tmp/before/test_pattern_matching.rb
+++ b/./tmp/after/test_pattern_matching.rb
@@ -3607,7 +3607,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “a”, [984, 13]]]],
[[:binary,
- [:vcall, [:@ident, “a”, [985, 10]]],
+ [:var_ref, [:@ident, “a”, [985, 10]]],
:==,
[:hash, nil]]],
nil]]],
@@ -3662,7 +3662,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “a”, [993, 13]]]],
[[:binary,
- [:vcall, [:@ident, “a”, [994, 10]]],
+ [:var_ref, [:@ident, “a”, [994, 10]]],
:==,
[:hash,
[:assoclist_from_args,
@@ -3813,7 +3813,7 @@
[:command,
[:@ident, “raise”, [1022, 10]],
[:args_add_block,
- [[:vcall, [:@ident, “b”, [1022, 16]]]],
+ [[:var_ref, [:@ident, “b”, [1022, 16]]]],
false]]],
[:else, [[:var_ref, [:@kw, “true”, [1024, 10]]]]]]]],
nil,
@@ -3876,7 +3876,7 @@
[:@int, “0”, [1033, 15]]],
:“&&“,
[:binary,
- [:vcall, [:@ident, “b”, [1033, 20]]],
+ [:var_ref, [:@ident, “b”, [1033, 20]]],
:==,
[:hash, nil]]]],
nil]]],
@@ -3946,7 +3946,7 @@
[:@int, “0”, [1042, 15]]],
:“&&“,
[:binary,
- [:vcall, [:@ident, “b”, [1042, 20]]],
+ [:var_ref, [:@ident, “b”, [1042, 20]]],
:==,
[:hash,
[:assoclist_from_args,
@@ -5206,7 +5206,7 @@
[[:assoc_new,
[:@label, “c:“, [1352, 22]],
[:@int, “0”, [1352, 25]]]]]],
- [:vcall, [:@ident, “r”, [1352, 29]]]],
+ [:var_ref, [:@ident, “r”, [1352, 29]]]],
false]]],
[:binary,
[:call,
@@ -5299,7 +5299,7 @@
[:assoc_new,
[:@label, “c:“, [1367, 34]],
[:@int, “0”, [1367, 37]]]]]],
- [:vcall, [:@ident, “r”, [1367, 41]]]],
+ [:var_ref, [:@ident, “r”, [1367, 41]]]],
false]]],
[:binary,
[:call,
@@ -5931,7 +5931,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “r”, [1533, 11]]]],
[[:binary,
- [:vcall, [:@ident, “r”, [1534, 8]]],
+ [:var_ref, [:@ident, “r”, [1534, 8]]],
:==,
[:hash,
[:assoclist_from_args,
```
2024-01-12 10:46:17 +09:00
|
|
|
int rb_parser_dvar_defined_ref(struct parser_params*, ID, ID**);
|
|
|
|
ID rb_parser_internal_id(struct parser_params*);
|
2024-12-28 11:25:57 +09:00
|
|
|
typedef void (*rb_parser_reg_fragment_error_func)(struct parser_params *, VALUE);
|
|
|
|
int rb_parser_reg_fragment_check(struct parser_params*, rb_parser_string_t*, int, rb_parser_reg_fragment_error_func);
|
2024-12-19 23:20:09 +09:00
|
|
|
int rb_reg_named_capture_assign_iter_impl(struct parser_params *p, const char *s, long len, rb_encoding *enc, NODE **succ_block, const rb_code_location_t *loc, rb_parser_assignable_func assignable);
|
[Feature #20257] Rearchitect Ripper
Introduce another semantic value stack for Ripper so that
Ripper can manage both Node and Ruby Object separately.
This rearchitectutre of Ripper solves these issues.
Therefore adding test cases for them.
* [Bug 10436] https://bugs.ruby-lang.org/issues/10436
* [Bug 18988] https://bugs.ruby-lang.org/issues/18988
* [Bug 20055] https://bugs.ruby-lang.org/issues/20055
Checked the differences of `Ripper.sexp` for files under `/test/ruby`
are only on test_pattern_matching.rb.
The differences comes from the differences between
`new_hash_pattern_tail` functions between parser and Ripper.
Ripper `new_hash_pattern_tail` didn’t call `assignable` then
`kw_rest_arg` wasn’t marked as local variable.
This is also fixed by this commit.
```
--- a/./tmp/before/test_pattern_matching.rb
+++ b/./tmp/after/test_pattern_matching.rb
@@ -3607,7 +3607,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “a”, [984, 13]]]],
[[:binary,
- [:vcall, [:@ident, “a”, [985, 10]]],
+ [:var_ref, [:@ident, “a”, [985, 10]]],
:==,
[:hash, nil]]],
nil]]],
@@ -3662,7 +3662,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “a”, [993, 13]]]],
[[:binary,
- [:vcall, [:@ident, “a”, [994, 10]]],
+ [:var_ref, [:@ident, “a”, [994, 10]]],
:==,
[:hash,
[:assoclist_from_args,
@@ -3813,7 +3813,7 @@
[:command,
[:@ident, “raise”, [1022, 10]],
[:args_add_block,
- [[:vcall, [:@ident, “b”, [1022, 16]]]],
+ [[:var_ref, [:@ident, “b”, [1022, 16]]]],
false]]],
[:else, [[:var_ref, [:@kw, “true”, [1024, 10]]]]]]]],
nil,
@@ -3876,7 +3876,7 @@
[:@int, “0”, [1033, 15]]],
:“&&“,
[:binary,
- [:vcall, [:@ident, “b”, [1033, 20]]],
+ [:var_ref, [:@ident, “b”, [1033, 20]]],
:==,
[:hash, nil]]]],
nil]]],
@@ -3946,7 +3946,7 @@
[:@int, “0”, [1042, 15]]],
:“&&“,
[:binary,
- [:vcall, [:@ident, “b”, [1042, 20]]],
+ [:var_ref, [:@ident, “b”, [1042, 20]]],
:==,
[:hash,
[:assoclist_from_args,
@@ -5206,7 +5206,7 @@
[[:assoc_new,
[:@label, “c:“, [1352, 22]],
[:@int, “0”, [1352, 25]]]]]],
- [:vcall, [:@ident, “r”, [1352, 29]]]],
+ [:var_ref, [:@ident, “r”, [1352, 29]]]],
false]]],
[:binary,
[:call,
@@ -5299,7 +5299,7 @@
[:assoc_new,
[:@label, “c:“, [1367, 34]],
[:@int, “0”, [1367, 37]]]]]],
- [:vcall, [:@ident, “r”, [1367, 41]]]],
+ [:var_ref, [:@ident, “r”, [1367, 41]]]],
false]]],
[:binary,
[:call,
@@ -5931,7 +5931,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “r”, [1533, 11]]]],
[[:binary,
- [:vcall, [:@ident, “r”, [1534, 8]]],
+ [:var_ref, [:@ident, “r”, [1534, 8]]],
:==,
[:hash,
[:assoclist_from_args,
```
2024-01-12 10:46:17 +09:00
|
|
|
int rb_parser_local_defined(struct parser_params *p, ID id, const struct rb_iseq_struct *iseq);
|
2024-12-19 23:20:09 +09:00
|
|
|
NODE *rb_parser_assignable(struct parser_params *p, ID id, NODE *val, const YYLTYPE *loc);
|
[Feature #20257] Rearchitect Ripper
Introduce another semantic value stack for Ripper so that
Ripper can manage both Node and Ruby Object separately.
This rearchitectutre of Ripper solves these issues.
Therefore adding test cases for them.
* [Bug 10436] https://bugs.ruby-lang.org/issues/10436
* [Bug 18988] https://bugs.ruby-lang.org/issues/18988
* [Bug 20055] https://bugs.ruby-lang.org/issues/20055
Checked the differences of `Ripper.sexp` for files under `/test/ruby`
are only on test_pattern_matching.rb.
The differences comes from the differences between
`new_hash_pattern_tail` functions between parser and Ripper.
Ripper `new_hash_pattern_tail` didn’t call `assignable` then
`kw_rest_arg` wasn’t marked as local variable.
This is also fixed by this commit.
```
--- a/./tmp/before/test_pattern_matching.rb
+++ b/./tmp/after/test_pattern_matching.rb
@@ -3607,7 +3607,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “a”, [984, 13]]]],
[[:binary,
- [:vcall, [:@ident, “a”, [985, 10]]],
+ [:var_ref, [:@ident, “a”, [985, 10]]],
:==,
[:hash, nil]]],
nil]]],
@@ -3662,7 +3662,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “a”, [993, 13]]]],
[[:binary,
- [:vcall, [:@ident, “a”, [994, 10]]],
+ [:var_ref, [:@ident, “a”, [994, 10]]],
:==,
[:hash,
[:assoclist_from_args,
@@ -3813,7 +3813,7 @@
[:command,
[:@ident, “raise”, [1022, 10]],
[:args_add_block,
- [[:vcall, [:@ident, “b”, [1022, 16]]]],
+ [[:var_ref, [:@ident, “b”, [1022, 16]]]],
false]]],
[:else, [[:var_ref, [:@kw, “true”, [1024, 10]]]]]]]],
nil,
@@ -3876,7 +3876,7 @@
[:@int, “0”, [1033, 15]]],
:“&&“,
[:binary,
- [:vcall, [:@ident, “b”, [1033, 20]]],
+ [:var_ref, [:@ident, “b”, [1033, 20]]],
:==,
[:hash, nil]]]],
nil]]],
@@ -3946,7 +3946,7 @@
[:@int, “0”, [1042, 15]]],
:“&&“,
[:binary,
- [:vcall, [:@ident, “b”, [1042, 20]]],
+ [:var_ref, [:@ident, “b”, [1042, 20]]],
:==,
[:hash,
[:assoclist_from_args,
@@ -5206,7 +5206,7 @@
[[:assoc_new,
[:@label, “c:“, [1352, 22]],
[:@int, “0”, [1352, 25]]]]]],
- [:vcall, [:@ident, “r”, [1352, 29]]]],
+ [:var_ref, [:@ident, “r”, [1352, 29]]]],
false]]],
[:binary,
[:call,
@@ -5299,7 +5299,7 @@
[:assoc_new,
[:@label, “c:“, [1367, 34]],
[:@int, “0”, [1367, 37]]]]]],
- [:vcall, [:@ident, “r”, [1367, 41]]]],
+ [:var_ref, [:@ident, “r”, [1367, 41]]]],
false]]],
[:binary,
[:call,
@@ -5931,7 +5931,7 @@
[:in,
[:hshptn, nil, [], [:var_field, [:@ident, “r”, [1533, 11]]]],
[[:binary,
- [:vcall, [:@ident, “r”, [1534, 8]]],
+ [:var_ref, [:@ident, “r”, [1534, 8]]],
:==,
[:hash,
[:assoclist_from_args,
```
2024-01-12 10:46:17 +09:00
|
|
|
|
|
|
|
RUBY_SYMBOL_EXPORT_END
|
2023-05-28 20:00:20 +09:00
|
|
|
|
2024-04-20 11:59:04 +09:00
|
|
|
#ifndef UNIVERSAL_PARSER
|
|
|
|
rb_parser_t *rb_ruby_parser_allocate(void);
|
|
|
|
rb_parser_t *rb_ruby_parser_new(void);
|
|
|
|
#endif
|
|
|
|
|
2023-05-28 20:00:20 +09:00
|
|
|
#ifdef RIPPER
|
|
|
|
void ripper_parser_mark(void *ptr);
|
|
|
|
void ripper_parser_free(void *ptr);
|
|
|
|
size_t ripper_parser_memsize(const void *ptr);
|
|
|
|
void ripper_error(struct parser_params *p);
|
|
|
|
VALUE ripper_value(struct parser_params *p);
|
|
|
|
int rb_ruby_parser_get_yydebug(rb_parser_t *p);
|
|
|
|
void rb_ruby_parser_set_value(rb_parser_t *p, VALUE value);
|
|
|
|
int rb_ruby_parser_error_p(rb_parser_t *p);
|
|
|
|
VALUE rb_ruby_parser_debug_output(rb_parser_t *p);
|
|
|
|
void rb_ruby_parser_set_debug_output(rb_parser_t *p, VALUE output);
|
|
|
|
VALUE rb_ruby_parser_parsing_thread(rb_parser_t *p);
|
|
|
|
void rb_ruby_parser_set_parsing_thread(rb_parser_t *p, VALUE parsing_thread);
|
2024-04-21 09:54:23 +09:00
|
|
|
void rb_ruby_parser_ripper_initialize(rb_parser_t *p, rb_parser_lex_gets_func *gets, rb_parser_input_data input, VALUE sourcefile_string, const char *sourcefile, int sourceline);
|
2023-05-28 20:00:20 +09:00
|
|
|
VALUE rb_ruby_parser_result(rb_parser_t *p);
|
|
|
|
rb_encoding *rb_ruby_parser_enc(rb_parser_t *p);
|
|
|
|
VALUE rb_ruby_parser_ruby_sourcefile_string(rb_parser_t *p);
|
|
|
|
int rb_ruby_parser_ruby_sourceline(rb_parser_t *p);
|
|
|
|
int rb_ruby_parser_lex_state(rb_parser_t *p);
|
|
|
|
void rb_ruby_ripper_parse0(rb_parser_t *p);
|
2024-09-21 08:39:42 +09:00
|
|
|
int rb_ruby_ripper_dedent_string(rb_parser_t *p, rb_parser_string_t *string, int width);
|
2023-05-28 20:00:20 +09:00
|
|
|
int rb_ruby_ripper_initialized_p(rb_parser_t *p);
|
|
|
|
void rb_ruby_ripper_parser_initialize(rb_parser_t *p);
|
|
|
|
long rb_ruby_ripper_column(rb_parser_t *p);
|
|
|
|
long rb_ruby_ripper_token_len(rb_parser_t *p);
|
2024-01-12 22:26:54 +09:00
|
|
|
rb_parser_string_t *rb_ruby_ripper_lex_lastline(rb_parser_t *p);
|
2023-05-28 20:00:20 +09:00
|
|
|
VALUE rb_ruby_ripper_lex_state_name(struct parser_params *p, int state);
|
Fix Ripper memory allocation size when enabled Universal Parser
The size of `struct parser_params` is 8 bytes difference in `ripper_s_allocate` and `rb_ruby_parser_allocate` when the universal parser is
enabled.
This causes a situation where `*r->p` is not fully initialized in `ripper_s_allocate` as shown below.
```console
(gdb) p *r->p
$2 = {heap = 0x0, lval = 0x0, yylloc = 0x0, lex = {strterm = 0x0, gets = 0x0, input = 0, string_buffer = {head = 0x0, last = 0x0}, lastlin
e = 0x0,
nextline = 0x0, pbeg = 0x0, pcur = 0x0, pend = 0x0, ptok = 0x0, gets_ = {ptr = 0, call = 0x0}, state = EXPR_NONE, paren_nest = 0, lpar
_seen = 0,
debug = 0, has_shebang = 0, token_seen = 0, token_info_enabled = 0, error_p = 0, cr_seen = 0, value = 0, result = 0, parsing_thread = 0, s_value = 0,
s_lvalue = 0, s_value_stack = 2097}
````
This seems to cause `double free or corruption (!prev)` and SEGV.
So, fixing this by introduce `rb_ripper_parser_params_allocate` and `rb_ruby_parser_config` functions for Ripper, and `struct parser_params` same size is returned.
2024-03-20 00:26:54 +09:00
|
|
|
#ifdef UNIVERSAL_PARSER
|
|
|
|
rb_parser_t *rb_ripper_parser_params_allocate(const rb_parser_config_t *config);
|
|
|
|
#endif
|
2023-05-28 20:00:20 +09:00
|
|
|
struct parser_params *rb_ruby_ripper_parser_allocate(void);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef UNIVERSAL_PARSER
|
|
|
|
#undef rb_encoding
|
|
|
|
#endif
|
2019-11-29 15:18:34 +09:00
|
|
|
|
|
|
|
#endif /* INTERNAL_PARSE_H */
|