Mirrors/fish-shell
2
0
Fork 0
mirror of https://github.com/fish-shell/fish-shell synced 2024-12-26 12:53:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

Improvements to new parser. All functions and completions now parse.

Browse source
This commit is contained in:
ridiculousfish 2013-07-22 18:26:15 -07:00
parent 77b6b0a9b2
commit 3e3eefc2dc
13 changed files with 1080 additions and 879 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
builtin.cpp
View file

@ -3946,53 +3946,53 @@ static int builtin_history(parser_t &parser, wchar_t **argv)
struct parse_execution_simulator_t : public parse_execution_visitor_t
{
wcstring_list_t result;
wcstring &back()
{
assert(! result.empty());
return result.back();
}
void append_src(node_offset_t idx)
{
wcstring tmp;
context->get_source(idx, &tmp);
back().append(tmp);
}
void append(const wchar_t *s)
{
back().append(s);
}
bool enter_job_list(void)
{
return true;
}
bool enter_job(void)
{
result.resize(result.size() + 1);
return true;
}
void visit_statement(void)
{
}
virtual void visit_boolean_statement(void)
{
}
virtual void enter_if_clause(const exec_if_clause_t &statement)
{
}
virtual void exit_if_clause(const exec_if_clause_t &statement)
{
append_format(back(), L"\nIF successful jump to %lu", (unsigned long)statement.body);
}
void visit_basic_statement(const exec_basic_statement_t &statement)
{
wcstring &line = this->back();
@ -4005,34 +4005,35 @@ struct parse_execution_simulator_t : public parse_execution_visitor_t
case exec_basic_statement_t::decoration_builtin:
line.append(L"<builtin> ");
break;
case exec_basic_statement_t::decoration_command:
line.append(L"<command> ");
break;
default:
break;
break;
}
line.append(L"cmd:");
this->append_src(statement.command_idx);
for (size_t i=0; i < statement.arguments().size(); i++)
{
const exec_argument_t &arg = statement.arguments().at(i);
append(L" ");
append(L"arg:");
append(L"arg:");
append_src(arg.parse_node_idx);
}
}
void visit_function(const exec_function_header_t &function) {
void visit_function(const exec_function_header_t &function)
{
wcstring &line = this->back();
line.append(L"define function: ");
wcstring tmp;
context->get_source(function.name_idx, &tmp);
line.append(tmp);
}
void exit_job_list(void)
{
}
@ -4074,17 +4075,19 @@ int builtin_parse(parser_t &parser, wchar_t **argv)
}
else
{
parse_execution_context_t ctx(parse_tree, src);
parse_execution_simulator_t sim;
sim.context = &ctx;
while (ctx.visit_next_node(&sim))
{
}
stdout_buffer.append(L"Simulating execution:\n");
for (size_t i=0; i < sim.result.size(); i++)
{
stdout_buffer.append(sim.result.at(i));
stdout_buffer.push_back(L'\n');
if (0) {
parse_execution_context_t ctx(parse_tree, src);
parse_execution_simulator_t sim;
sim.context = &ctx;
while (ctx.visit_next_node(&sim))
{
}
stdout_buffer.append(L"Simulating execution:\n");
for (size_t i=0; i < sim.result.size(); i++)
{
stdout_buffer.append(sim.result.at(i));
stdout_buffer.push_back(L'\n');
}
}
}
}

2
builtin_printf.cpp
View file

@ -632,7 +632,7 @@ int builtin_printf_state_t::print_formatted(const wchar_t *format, int argc, wch
}
break;
}
modify_allowed_format_specifiers(ok, "aAcdeEfFgGiosuxX", true);
for (;; f++, direc_length++)

16
exec.cpp
View file

@ -537,7 +537,7 @@ static bool can_use_posix_spawn_for_job(const job_t *job, const process_t *proce
return false;
}
}
/* Now see if we have a redirection involving a file. The only one we allow is /dev/null, which we assume will not fail. */
bool result = true;
for (size_t idx = 0; idx < job->io.size(); idx++)
@ -545,8 +545,8 @@ static bool can_use_posix_spawn_for_job(const job_t *job, const process_t *proce
const shared_ptr<const io_data_t> &io = job->io.at(idx);
if (redirection_is_to_real_file(io.get()))
{
result = false;
break;
result = false;
break;
}
}
return result;
@ -1192,16 +1192,16 @@ void exec(parser_t &parser, job_t *j)
forking is expensive, fish tries to avoid it when
possible.
*/
bool fork_was_skipped = false;
const shared_ptr<io_data_t> stdout_io = io_chain_get(j->io, STDOUT_FILENO);
const shared_ptr<io_data_t> stderr_io = io_chain_get(j->io, STDERR_FILENO);
/* If we are outputting to a file, we have to actually do it, even if we have no output, so that we can truncate the file. Does not apply to /dev/null. */
bool must_fork = redirection_is_to_real_file(stdout_io.get()) || redirection_is_to_real_file(stderr_io.get());
if (! must_fork)
{
{
if (p->next == NULL)
{
const bool stdout_is_to_buffer = stdout_io && stdout_io->io_mode == IO_BUFFER;
@ -1250,7 +1250,7 @@ void exec(parser_t &parser, job_t *j)
}
}
}
if (fork_was_skipped)
{

2
fish.xcodeproj/project.pbxproj
View file

@ -465,6 +465,7 @@
D0D2693C159835CA005D9B9C /* fish */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish; sourceTree = BUILT_PRODUCTS_DIR; };
D0F3373A1506DE3C00ECEFC0 /* builtin_test.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_test.cpp; sourceTree = "<group>"; };
D0F5E28415A7A32D00315DFF /* config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = config.h; sourceTree = "<group>"; };
D0FE8EE6179CA8A5008C9F21 /* parse_tree_construction.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_tree_construction.h; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@ -589,6 +590,7 @@
D0A0850C13B3ACEE0099B651 /* expand.h */,
D0A0853D13B3ACEE0099B651 /* expand.cpp */,
D0C52F361765284C00BFAB82 /* parse_tree.h */,
D0FE8EE6179CA8A5008C9F21 /* parse_tree_construction.h */,
D0C52F351765284C00BFAB82 /* parse_tree.cpp */,
D0C52F341765281F00BFAB82 /* parse_exec.h */,
D0C52F331765281F00BFAB82 /* parse_exec.cpp */,

6
fish_tests.cpp
View file

@ -534,7 +534,7 @@ static void test_utils()
{
say(L"Testing utils");
const wchar_t *a = L"echo (echo (echo hi";
const wchar_t *begin = NULL, *end = NULL;
parse_util_cmdsubst_extent(a, 0, &begin, &end);
if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
@ -544,7 +544,7 @@ static void test_utils()
if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
parse_util_cmdsubst_extent(a, 3, &begin, &end);
if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
parse_util_cmdsubst_extent(a, 8, &begin, &end);
if (begin != a + wcslen(L"echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
@ -1842,7 +1842,7 @@ int main(int argc, char **argv)
builtin_init();
reader_init();
env_init();
test_new_parser();
return 0;

170
parse_exec.cpp
View file

@ -6,11 +6,11 @@ struct exec_node_t
node_offset_t parse_node_idx;
node_offset_t body_parse_node_idx;
bool visited;
explicit exec_node_t(node_offset_t pni) : parse_node_idx(pni), body_parse_node_idx(NODE_OFFSET_INVALID), visited(false)
{
}
explicit exec_node_t(node_offset_t pni, node_offset_t body_pni) : parse_node_idx(pni), body_parse_node_idx(body_pni), visited(false)
{
}
@ -18,7 +18,7 @@ struct exec_node_t
exec_basic_statement_t::exec_basic_statement_t() : command_idx(0), decoration(decoration_plain)
{
}
@ -26,21 +26,21 @@ class parse_exec_t
{
parse_node_tree_t parse_tree;
wcstring src;
/* The stack of nodes as we execute them */
std::vector<exec_node_t> exec_nodes;
/* The stack of commands being built */
std::vector<exec_basic_statement_t> assembling_statements;
/* Current visitor (very transient) */
struct parse_execution_visitor_t * visitor;
const parse_node_t &get_child(const parse_node_t &parent, node_offset_t which) const
{
return parse_tree.at(parent.child_offset(which));
}
void pop_push_specific(node_offset_t idx1, node_offset_t idx2 = NODE_OFFSET_INVALID, node_offset_t idx3 = NODE_OFFSET_INVALID, node_offset_t idx4 = NODE_OFFSET_INVALID, node_offset_t idx5 = NODE_OFFSET_INVALID)
{
PARSE_ASSERT(! exec_nodes.empty());
@ -48,10 +48,10 @@ class parse_exec_t
exec_node_t &top = exec_nodes.back();
const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx);
node_offset_t child_node_idx = parse_node.child_start;
// Remove the top node
exec_nodes.pop_back();
// Append the given children, backwards
const node_offset_t idxs[] = {idx5, idx4, idx3, idx2, idx1};
for (size_t q=0; q < sizeof idxs / sizeof *idxs; q++)
@ -65,18 +65,18 @@ class parse_exec_t
}
}
void push(node_offset_t global_idx)
{
exec_nodes.push_back(exec_node_t(global_idx));
}
void push(const exec_node_t &node)
{
exec_nodes.push_back(node);
}
void pop_push(node_offset_t child_idx, node_offset_t child_count = 1)
{
PARSE_ASSERT(! exec_nodes.empty());
@ -92,10 +92,10 @@ class parse_exec_t
const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx);
PARSE_ASSERT(child_idx < parse_node.child_count);
node_offset_t child_node_idx = parse_node.child_start + child_idx;
// Remove the top node
exec_nodes.pop_back();
// Append the given children, backwards
node_offset_t cursor = child_count;
while (cursor--)
@ -104,20 +104,20 @@ class parse_exec_t
}
}
}
void pop()
{
PARSE_ASSERT(! exec_nodes.empty());
exec_nodes.pop_back();
}
void pop_push_all()
{
exec_node_t &top = exec_nodes.back();
const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx);
pop_push(0, parse_node.child_count);
}
void assemble_1_argument_or_redirection(node_offset_t idx, exec_arguments_and_redirections_t *output) const
{
const parse_node_t &node = parse_tree.at(idx);
@ -130,28 +130,28 @@ class parse_exec_t
{
case parse_token_type_string:
// Argument
{
exec_argument_t arg = exec_argument_t();
arg.parse_node_idx = child_idx;
output->arguments.push_back(arg);
}
break;
{
exec_argument_t arg = exec_argument_t();
arg.parse_node_idx = child_idx;
output->arguments.push_back(arg);
}
break;
case parse_token_type_redirection:
// Redirection
{
exec_redirection_t redirect = exec_redirection_t();
redirect.parse_node_idx = child_idx;
output->redirections.push_back(redirect);
}
break;
{
exec_redirection_t redirect = exec_redirection_t();
redirect.parse_node_idx = child_idx;
output->redirections.push_back(redirect);
}
break;
default:
PARSER_DIE();
break;
}
}
void assemble_arguments_and_redirections(node_offset_t start_idx, exec_arguments_and_redirections_t *output) const
{
node_offset_t idx = start_idx;
@ -173,7 +173,7 @@ class parse_exec_t
}
}
}
void assemble_command_for_plain_statement(node_offset_t idx, parse_keyword_t decoration)
{
const parse_node_t &node = parse_tree.at(idx);
@ -185,10 +185,10 @@ class parse_exec_t
assemble_arguments_and_redirections(node.child_offset(1), &statement.arguments_and_redirections);
visitor->visit_basic_statement(statement);
}
void assemble_block_statement(node_offset_t parse_node_idx)
{
const parse_node_t &node = parse_tree.at(parse_node_idx);
PARSE_ASSERT(node.type == symbol_block_statement);
PARSE_ASSERT(node.child_count == 5);
@ -196,24 +196,24 @@ class parse_exec_t
// Fetch arguments and redirections. These ought to be evaluated before the job list
exec_block_statement_t statement;
assemble_arguments_and_redirections(node.child_offset(4), &statement.arguments_and_redirections);
// Generic visit
visitor->enter_block_statement(statement);
// Dig into the header to discover the type
const parse_node_t &header_parent = parse_tree.at(node.child_offset(0));
PARSE_ASSERT(header_parent.type == symbol_block_header);
PARSE_ASSERT(header_parent.child_count == 1);
PARSE_ASSERT(header_parent.child_count == 1);
const node_offset_t header_idx = header_parent.child_offset(0);
// Fetch body (job list)
node_offset_t body_idx = node.child_offset(2);
PARSE_ASSERT(parse_tree.at(body_idx).type == symbol_job_list);
pop();
push(exec_node_t(header_idx, body_idx));
}
/* which: 0 -> if, 1 -> else if, 2 -> else */
void assemble_if_else_clause(exec_node_t &exec_node, const parse_node_t &node, int which)
{
@ -227,7 +227,7 @@ class parse_exec_t
PARSE_ASSERT(node.type == symbol_else_continuation);
PARSE_ASSERT(node.child_count == 2);
}
struct exec_if_clause_t clause;
if (which == 0)
{
@ -252,7 +252,7 @@ class parse_exec_t
pop();
}
}
void assemble_arguments(node_offset_t start_idx, exec_argument_list_t *output) const
{
node_offset_t idx = start_idx;
@ -282,24 +282,24 @@ class parse_exec_t
}
}
}
void assemble_1_case_item(exec_switch_statement_t *statement, node_offset_t node_idx)
{
const parse_node_t &node = parse_tree.at(node_idx);
PARSE_ASSERT(node.type == symbol_case_item);
// add a new case
size_t len = statement->cases.size();
statement->cases.resize(len + 1);
exec_switch_case_t &new_case = statement->cases.back();
// assemble it
new_case.body = node.child_offset(3);
assemble_arguments(node.child_offset(1), &new_case.arguments);
}
void assemble_case_item_list(exec_switch_statement_t *statement, node_offset_t node_idx)
{
const parse_node_t &node = parse_tree.at(node_idx);
@ -311,21 +311,21 @@ class parse_exec_t
assemble_case_item_list(statement, node.child_offset(1));
}
}
void assemble_switch_statement(const exec_node_t &exec_node, const parse_node_t &parse_node)
{
PARSE_ASSERT(parse_node.type == symbol_switch_statement);
exec_switch_statement_t statement;
statement.argument.parse_node_idx = parse_node.child_offset(1);
assemble_case_item_list(&statement, parse_node.child_offset(3));
visitor->visit_switch_statement(statement);
// pop off the switch
pop();
}
void assemble_function_header(const exec_node_t &exec_node, const parse_node_t &header)
{
PARSE_ASSERT(header.type == symbol_function_header);
@ -336,17 +336,17 @@ class parse_exec_t
function_info.body_idx = exec_node.body_parse_node_idx;
assemble_arguments(header.child_offset(2), &function_info.arguments);
visitor->visit_function(function_info);
// Always pop
pop();
}
void enter_parse_node(size_t idx);
void run_top_node(void);
public:
public:
void get_node_string(node_offset_t idx, wcstring *output) const
{
const parse_node_t &node = parse_tree.at(idx);
@ -354,9 +354,9 @@ class parse_exec_t
PARSE_ASSERT(node.source_start + node.source_length <= src.size());
output->assign(src, node.source_start, node.source_length);
}
bool visit_next_node(parse_execution_visitor_t *v);
parse_exec_t(const parse_node_tree_t &tree, const wcstring &s) : parse_tree(tree), src(s), visitor(NULL)
{
if (! parse_tree.empty())
@ -373,7 +373,7 @@ void parse_exec_t::run_top_node()
const node_offset_t parse_node_idx = exec_node.parse_node_idx;
const parse_node_t &parse_node = parse_tree.at(exec_node.parse_node_idx);
bool log = true;
if (log)
{
wcstring tmp;
@ -381,7 +381,7 @@ void parse_exec_t::run_top_node()
tmp.append(parse_node.describe());
printf("%ls\n", tmp.c_str());
}
switch (parse_node.type)
{
case symbol_job_list:
@ -404,7 +404,7 @@ void parse_exec_t::run_top_node()
pop_push(0, 2);
}
break;
case symbol_job:
{
PARSE_ASSERT(parse_node.child_count == 2);
@ -412,7 +412,7 @@ void parse_exec_t::run_top_node()
pop_push_all();
break;
}
case symbol_job_continuation:
PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 3);
if (parse_node.child_count == 0)
@ -426,51 +426,51 @@ void parse_exec_t::run_top_node()
// Skip the pipe
pop_push(1, 2);
}
break;
break;
case symbol_statement:
{
PARSE_ASSERT(parse_node.child_count == 1);
pop_push_all();
break;
}
case symbol_block_statement:
{
PARSE_ASSERT(parse_node.child_count == 5);
assemble_block_statement(parse_node_idx);
break;
}
case symbol_block_header:
{
PARSE_ASSERT(parse_node.child_count == 1);
pop_push_all();
break;
}
case symbol_function_header:
{
PARSE_ASSERT(parse_node.child_count == 3);
assemble_function_header(exec_node, parse_node);
break;
}
case symbol_if_statement:
{
PARSE_ASSERT(parse_node.child_count == 3);
PARSE_ASSERT(parse_node.child_count == 4);
pop_push(0, 2);
break;
}
case symbol_if_clause:
{
PARSE_ASSERT(parse_node.child_count == 4);
assemble_if_else_clause(exec_node, parse_node, 0);
pop();
break;
break;
}
case symbol_else_clause:
{
PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 2);
@ -486,7 +486,7 @@ void parse_exec_t::run_top_node()
}
break;
}
case symbol_else_continuation:
{
// Figure out if this is an else if or a terminating else
@ -505,17 +505,17 @@ void parse_exec_t::run_top_node()
}
break;
}
case symbol_switch_statement:
{
assemble_switch_statement(exec_node, parse_node);
break;
}
case symbol_decorated_statement:
{
PARSE_ASSERT(parse_node.child_count == 1 || parse_node.child_count == 2);
node_offset_t plain_statement_idx = parse_node.child_offset(parse_node.child_count - 1);
parse_keyword_t decoration = static_cast<parse_keyword_t>(parse_node.tag);
assemble_command_for_plain_statement(plain_statement_idx, decoration);
@ -528,20 +528,20 @@ void parse_exec_t::run_top_node()
case symbol_plain_statement:
case symbol_arguments_or_redirections_list:
case symbol_argument_or_redirection:
fprintf(stderr, "Unhandled token type %ls at index %ld\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx);
fprintf(stderr, "Unexpected token type %ls at index %ld. This should have been handled by the parent.\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx);
PARSER_DIE();
break;
case parse_token_type_end:
PARSE_ASSERT(parse_node.child_count == 0);
pop();
break;
default:
fprintf(stderr, "Unhandled token type %ls at index %ld\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx);
PARSER_DIE();
break;
}
}
@ -553,7 +553,7 @@ bool parse_exec_t::visit_next_node(parse_execution_visitor_t *v)
{
return false;
}
visitor = v;
run_top_node();
visitor = NULL;

49
parse_exec.h
View file

@ -13,13 +13,13 @@ class parse_exec_t;
class parse_execution_context_t
{
parse_exec_t *ctx; //owned
public:
public:
parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s);
~parse_execution_context_t();
bool visit_next_node(parse_execution_visitor_t *visitor);
// Gets the source for a node at a given index
void get_source(node_offset_t idx, wcstring *result) const;
};
@ -53,10 +53,10 @@ struct exec_basic_statement_t
{
// Node containing the command
node_offset_t command_idx;
// Arguments
exec_arguments_and_redirections_t arguments_and_redirections;
// Decoration
enum
{
@ -64,9 +64,9 @@ struct exec_basic_statement_t
decoration_command,
decoration_builtin
} decoration;
exec_basic_statement_t();
void set_decoration(uint32_t k)
{
PARSE_ASSERT(k == parse_keyword_none || k == parse_keyword_command || k == parse_keyword_builtin);
@ -86,12 +86,12 @@ struct exec_basic_statement_t
break;
}
}
const exec_argument_list_t &arguments() const
{
return arguments_and_redirections.arguments;
}
const exec_redirection_list_t &redirections() const
{
return arguments_and_redirections.redirections;
@ -102,10 +102,10 @@ struct exec_function_header_t
{
// Node containing the function name
node_offset_t name_idx;
// Node containing the function body
node_offset_t body_idx;
// Arguments
exec_argument_list_t arguments;
};
@ -138,23 +138,32 @@ struct parse_execution_visitor_t
{
node_offset_t node_idx;
parse_execution_context_t *context;
parse_execution_visitor_t() : node_idx(0), context(NULL)
{
}
virtual bool enter_job_list(void) { return true; }
virtual bool enter_job(void) { return true; }
virtual bool enter_job_list(void)
{
return true;
}
virtual bool enter_job(void)
{
return true;
}
virtual void visit_statement(void) { }
virtual void visit_function(const exec_function_header_t &function) { }
virtual bool enter_block_statement(const exec_block_statement_t &statement) { return true; }
virtual bool enter_block_statement(const exec_block_statement_t &statement)
{
return true;
}
virtual void enter_if_clause(const exec_if_clause_t &statement) { }
virtual void exit_if_clause(const exec_if_clause_t &statement) { }
virtual void visit_switch_statement(const exec_switch_statement_t &header) { }
virtual void visit_boolean_statement(void) { }
virtual void visit_basic_statement(const exec_basic_statement_t &statement) { }
virtual void exit_job(void) { }

628
parse_tree.cpp
View file

@ -1,4 +1,4 @@
#include "parse_tree.h"
#include "parse_tree_construction.h"
#include "tokenizer.h"
#include <vector>
@ -11,26 +11,28 @@ wcstring parse_error_t::describe(const wcstring &src) const
{
// Locate the beginning of this line of source
size_t line_start = 0;
// Look for a newline prior to source_start. If we don't find one, start at the beginning of the string; otherwise start one past the newline
size_t newline = src.find_last_of(L'\n', source_start);
fprintf(stderr, "newline: %lu, source_start %lu, source_length %lu\n", newline, source_start, source_length);
if (newline != wcstring::npos)
{
line_start = newline + 1;
line_start = newline;// + 1;
}
size_t line_end = src.find(L'\n', source_start + source_length);
if (line_end == wcstring::npos)
{
line_end = src.size();
}
assert(line_end >= line_start);
fprintf(stderr, "source start: %lu, line start %lu\n", source_start, line_start);
assert(source_start >= line_start);
// Append the line of text
result.push_back(L'\n');
result.append(src, line_start, line_end - line_start);
// Append the caret line
result.push_back(L'\n');
result.append(source_start - line_start, L' ');
@ -43,68 +45,115 @@ wcstring token_type_description(parse_token_type_t type)
{
switch (type)
{
case token_type_invalid: return L"invalid";
case symbol_job_list: return L"job_list";
case symbol_job: return L"job";
case symbol_job_continuation: return L"job_continuation";
case symbol_statement: return L"statement";
case symbol_block_statement: return L"block_statement";
case symbol_block_header: return L"block_header";
case symbol_for_header: return L"for_header";
case symbol_while_header: return L"while_header";
case symbol_begin_header: return L"begin_header";
case symbol_function_header: return L"function_header";
case symbol_if_statement: return L"if_statement";
case symbol_if_clause: return L"if_clause";
case symbol_else_clause: return L"else_clause";
case symbol_else_continuation: return L"else_continuation";
case symbol_switch_statement: return L"switch_statement";
case symbol_case_item_list: return L"case_item_list";
case symbol_case_item: return L"case_item";
case symbol_argument_list_nonempty: return L"argument_list_nonempty";
case symbol_argument_list: return L"argument_list";
case symbol_boolean_statement: return L"boolean_statement";
case symbol_decorated_statement: return L"decorated_statement";
case symbol_plain_statement: return L"plain_statement";
case symbol_arguments_or_redirections_list: return L"arguments_or_redirections_list";
case symbol_argument_or_redirection: return L"argument_or_redirection";
case parse_token_type_string: return L"token_string";
case parse_token_type_pipe: return L"token_pipe";
case parse_token_type_redirection: return L"token_redirection";
case parse_token_background: return L"token_background";
case parse_token_type_end: return L"token_end";
case parse_token_type_terminate: return L"token_terminate";
default: return format_string(L"Unknown token type %ld", static_cast<long>(type));
case token_type_invalid:
return L"invalid";
case symbol_job_list:
return L"job_list";
case symbol_job:
return L"job";
case symbol_job_continuation:
return L"job_continuation";
case symbol_statement:
return L"statement";
case symbol_block_statement:
return L"block_statement";
case symbol_block_header:
return L"block_header";
case symbol_for_header:
return L"for_header";
case symbol_while_header:
return L"while_header";
case symbol_begin_header:
return L"begin_header";
case symbol_function_header:
return L"function_header";
case symbol_if_statement:
return L"if_statement";
case symbol_if_clause:
return L"if_clause";
case symbol_else_clause:
return L"else_clause";
case symbol_else_continuation:
return L"else_continuation";
case symbol_switch_statement:
return L"switch_statement";
case symbol_case_item_list:
return L"case_item_list";
case symbol_case_item:
return L"case_item";
case symbol_argument_list_nonempty:
return L"argument_list_nonempty";
case symbol_argument_list:
return L"argument_list";
case symbol_boolean_statement:
return L"boolean_statement";
case symbol_decorated_statement:
return L"decorated_statement";
case symbol_plain_statement:
return L"plain_statement";
case symbol_arguments_or_redirections_list:
return L"arguments_or_redirections_list";
case symbol_argument_or_redirection:
return L"argument_or_redirection";
case parse_token_type_string:
return L"token_string";
case parse_token_type_pipe:
return L"token_pipe";
case parse_token_type_redirection:
return L"token_redirection";
case parse_token_type_background:
return L"token_background";
case parse_token_type_end:
return L"token_end";
case parse_token_type_terminate:
return L"token_terminate";
case symbol_optional_background:
return L"optional_background";
}
return format_string(L"Unknown token type %ld", static_cast<long>(type));
}
wcstring keyword_description(parse_keyword_t k)
{
switch (k)
{
case parse_keyword_none: return L"none";
case parse_keyword_if: return L"if";
case parse_keyword_else: return L"else";
case parse_keyword_for: return L"for";
case parse_keyword_in: return L"in";
case parse_keyword_while: return L"while";
case parse_keyword_begin: return L"begin";
case parse_keyword_function: return L"function";
case parse_keyword_switch: return L"switch";
case parse_keyword_end: return L"end";
case parse_keyword_and: return L"and";
case parse_keyword_or: return L"or";
case parse_keyword_not: return L"not";
case parse_keyword_command: return L"command";
case parse_keyword_builtin: return L"builtin";
case parse_keyword_none:
return L"none";
case parse_keyword_if:
return L"if";
case parse_keyword_else:
return L"else";
case parse_keyword_for:
return L"for";
case parse_keyword_in:
return L"in";
case parse_keyword_while:
return L"while";
case parse_keyword_begin:
return L"begin";
case parse_keyword_function:
return L"function";
case parse_keyword_switch:
return L"switch";
case parse_keyword_end:
return L"end";
case parse_keyword_and:
return L"and";
case parse_keyword_or:
return L"or";
case parse_keyword_not:
return L"not";
case parse_keyword_command:
return L"command";
case parse_keyword_builtin:
return L"builtin";
default:
return format_string(L"Unknown keyword type %ld", static_cast<long>(k));
}
@ -123,7 +172,7 @@ struct parse_token_t
enum parse_keyword_t keyword; // Any keyword represented by this parser
size_t source_start;
size_t source_length;
wcstring describe() const;
};
@ -147,19 +196,28 @@ static parse_token_t parse_token_from_tokenizer_token(enum token_type tokenizer_
case TOK_STRING:
result.type = parse_token_type_string;
break;
case TOK_PIPE:
result.type = parse_token_type_pipe;
break;
case TOK_END:
result.type = parse_token_type_end;
break;
case TOK_BACKGROUND:
result.type = parse_token_background;
result.type = parse_token_type_background;
break;
case TOK_REDIRECT_OUT:
case TOK_REDIRECT_APPEND:
case TOK_REDIRECT_IN:
case TOK_REDIRECT_FD:
case TOK_REDIRECT_NOCLOB:
result.type = parse_token_type_redirection;
break;
default:
fprintf(stderr, "Bad token type %d passed to %s\n", (int)tokenizer_token_type, __FUNCTION__);
assert(0);
@ -172,15 +230,15 @@ static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring &
{
assert(start < nodes.size());
const parse_node_t &node = nodes.at(start);
const size_t spacesPerIndent = 2;
// unindent statement lists by 1 to flatten them
if (node.type == symbol_job_list || node.type == symbol_arguments_or_redirections_list)
{
if (indent > 0) indent -= 1;
}
append_format(*result, L"%2lu - %l2u ", *line, start);
result->append(indent * spacesPerIndent, L' ');;
result->append(node.describe());
@ -206,7 +264,7 @@ static wcstring dump_tree(const parse_node_tree_t &nodes, const wcstring &src)
{
if (nodes.empty())
return L"(empty!)";
size_t line = 0;
wcstring result;
dump_tree_recursive(nodes, src, 0, 0, &result, &line);
@ -218,15 +276,15 @@ struct parse_stack_element_t
enum parse_token_type_t type;
enum parse_keyword_t keyword;
node_offset_t node_idx;
parse_stack_element_t(parse_token_type_t t) : type(t), keyword(parse_keyword_none), node_idx(-1)
{
}
parse_stack_element_t(parse_keyword_t k) : type(parse_token_type_string), keyword(k), node_idx(-1)
{
}
wcstring describe(void) const
{
wcstring result = token_type_description(type);
@ -242,13 +300,13 @@ struct parse_stack_element_t
class parse_ll_t
{
friend class parse_t;
std::vector<parse_stack_element_t> symbol_stack; // LL parser stack
parse_node_tree_t nodes;
bool fatal_errored;
parse_error_list_t errors;
// Constructor
parse_ll_t() : fatal_errored(false)
{
@ -258,33 +316,30 @@ class parse_ll_t
symbol_stack.push_back(elem); // goal token
nodes.push_back(parse_node_t(symbol_job_list));
}
bool top_node_match_token(parse_token_t token);
// implementation of certain parser constructions
void accept_token(parse_token_t token, const wcstring &src);
void accept_token_job_list(parse_token_t token);
void accept_token_job(parse_token_t token);
void accept_token_job_continuation(parse_token_t token);
void accept_token_statement(parse_token_t token);
void accept_token_block_header(parse_token_t token);
void accept_token_else_clause(parse_token_t token);
void accept_token_else_continuation(parse_token_t token);
void accept_token_boolean_statement(parse_token_t token);
void accept_token_decorated_statement(parse_token_t token);
void accept_token_plain_statement(parse_token_t token);
void accept_token_argument_list(parse_token_t token);
void accept_token_arguments_or_redirections_list(parse_token_t token);
void accept_token_argument_or_redirection(parse_token_t token);
bool accept_token_string(parse_token_t token);
void token_unhandled(parse_token_t token, const char *function);
void parse_error(const wchar_t *expected, parse_token_t token);
void parse_error(parse_token_t token, const wchar_t *format, ...);
void append_error_callout(wcstring &error_message, parse_token_t token);
void dump_stack(void) const;
// Get the node corresponding to the top element of the stack
parse_node_t &node_for_top_symbol()
{
@ -294,17 +349,17 @@ class parse_ll_t
PARSE_ASSERT(top_symbol.node_idx < nodes.size());
return nodes.at(top_symbol.node_idx);
}
parse_token_type_t stack_top_type() const
{
return symbol_stack.back().type;
}
void top_node_set_tag(uint32_t tag)
{
this->node_for_top_symbol().tag = tag;
}
inline void add_child_to_node(size_t parent_node_idx, parse_stack_element_t *tok)
{
PARSE_ASSERT(tok->type != token_type_invalid);
@ -312,19 +367,19 @@ class parse_ll_t
nodes.push_back(parse_node_t(tok->type));
nodes.at(parent_node_idx).child_count += 1;
}
inline void symbol_stack_pop()
{
symbol_stack.pop_back();
}
// Pop from the top of the symbol stack, then push, updating node counts. Note that these are pushed in reverse order, so the first argument will be on the top of the stack.
inline void symbol_stack_pop_push_int(parse_stack_element_t tok1 = token_type_invalid, parse_stack_element_t tok2 = token_type_invalid, parse_stack_element_t tok3 = token_type_invalid, parse_stack_element_t tok4 = token_type_invalid, parse_stack_element_t tok5 = token_type_invalid)
{
// Logging?
if (1)
if (0)
{
fprintf(stderr, "Pop %ls (%lu)\n", token_type_description(symbol_stack.back().type).c_str(), symbol_stack.size());
if (tok5.type != token_type_invalid) fprintf(stderr, "Push %ls\n", tok5.describe().c_str());
@ -333,17 +388,17 @@ class parse_ll_t
if (tok2.type != token_type_invalid) fprintf(stderr, "Push %ls\n", tok2.describe().c_str());
if (tok1.type != token_type_invalid) fprintf(stderr, "Push %ls\n", tok1.describe().c_str());
}
// Get the node for the top symbol and tell it about its children
size_t node_idx = symbol_stack.back().node_idx;
parse_node_t &node = nodes.at(node_idx);
// Should have no children yet
PARSE_ASSERT(node.child_count == 0);
// Tell the node where its children start
node.child_start = nodes.size();
// Add nodes for the children
// Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack)
if (tok1.type != token_type_invalid) add_child_to_node(node_idx, &tok1);
@ -351,7 +406,7 @@ class parse_ll_t
if (tok3.type != token_type_invalid) add_child_to_node(node_idx, &tok3);
if (tok4.type != token_type_invalid) add_child_to_node(node_idx, &tok4);
if (tok5.type != token_type_invalid) add_child_to_node(node_idx, &tok5);
// The above set the node_idx. Now replace the top of the stack.
symbol_stack.pop_back();
if (tok5.type != token_type_invalid) symbol_stack.push_back(tok5);
@ -360,31 +415,69 @@ class parse_ll_t
if (tok2.type != token_type_invalid) symbol_stack.push_back(tok2);
if (tok1.type != token_type_invalid) symbol_stack.push_back(tok1);
}
template<typename T>
inline void symbol_stack_pop_push2()
inline void symbol_stack_pop_push2(typename T::magic_seq_type_t x = 0)
{
symbol_stack_pop_push_int(T::t0::get_token(), T::t1::get_token(), T::t2::get_token(), T::t3::get_token(), T::t4::get_token());
}
template<typename T>
inline void symbol_stack_pop_push_production(int which)
inline void symbol_stack_pop_push2(typename T::magic_symbol_type_t x = 0)
{
symbol_stack_pop_push_int(T::get_token());
}
// Singular. Sole productions are always of type Seq.
template<typename T>
inline void symbol_stack_produce(parse_token_t tok, typename T::sole_production::magic_seq_type_t magic=0)
{
typedef typename T::sole_production seq;
symbol_stack_pop_push_int(seq::t0::get_token(), seq::t1::get_token(), seq::t2::get_token(), seq::t3::get_token(), seq::t4::get_token());
}
// Plural productions, of type Or.
template<typename T>
inline void symbol_stack_produce(parse_token_t tok, typename T::productions::magic_or_type_t magic=0)
{
typedef typename T::productions ors;
int which = T::production(tok.type, tok.keyword);
switch (which)
{
case 0: symbol_stack_pop_push2<typename T::p0>(); break;
case 1: symbol_stack_pop_push2<typename T::p1>(); break;
case 2: symbol_stack_pop_push2<typename T::p2>(); break;
case 3: symbol_stack_pop_push2<typename T::p3>(); break;
case 4: symbol_stack_pop_push2<typename T::p4>(); break;
case 0:
symbol_stack_pop_push2<typename ors::p0>();
break;
case 1:
symbol_stack_pop_push2<typename ors::p1>();
break;
case 2:
symbol_stack_pop_push2<typename ors::p2>();
break;
case 3:
symbol_stack_pop_push2<typename ors::p3>();
break;
case 4:
symbol_stack_pop_push2<typename ors::p4>();
break;
case NO_PRODUCTION:
parse_error(tok, L"Failed to produce with stack top '%ls' for token '%ls'\n", symbol_stack.back().describe().c_str(), tok.describe().c_str());
break;
default:
parse_error(tok, L"Unexpected production %d for token %ls\n", which, tok.describe().c_str());
break;
}
}
// Non-sequence basic productions
template<typename T>
inline void symbol_stack_produce(parse_token_t tok)
inline void symbol_stack_produce(parse_token_t tok, typename T::sole_production::magic_symbol_type_t magic=0)
{
symbol_stack_pop_push_production<T>(T::production(tok.type, tok.keyword));
symbol_stack_pop_push_int(T::sole_production::get_token());
}
};
void parse_ll_t::dump_stack(void) const
@ -410,7 +503,7 @@ void parse_ll_t::dump_stack(void) const
}
}
}
fprintf(stderr, "Stack dump (%lu elements):\n", symbol_stack.size());
for (size_t idx = 0; idx < lines.size(); idx++)
{
@ -422,9 +515,31 @@ void parse_ll_t::token_unhandled(parse_token_t token, const char *function)
{
fprintf(stderr, "Unhandled token with type %ls in function %s\n", token_type_description(token.type).c_str(), function);
this->dump_stack();
PARSER_DIE();
parse_error_t err;
err.text = format_string(L"Unhandled token with type %ls in function %s", token_type_description(token.type).c_str(), function);
err.source_start = token.source_start;
err.source_length = token.source_length;
this->errors.push_back(err);
this->fatal_errored = true;
}
void parse_ll_t::parse_error(parse_token_t token, const wchar_t *fmt, ...)
{
this->dump_stack();
parse_error_t err;
va_list va;
va_start(va, fmt);
err.text = vformat_string(fmt, va);
va_end(va);
err.source_start = token.source_start;
err.source_length = token.source_length;
this->errors.push_back(err);
this->fatal_errored = true;
}
void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token)
{
wcstring desc = token_type_description(token.type);
@ -436,172 +551,6 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token)
fatal_errored = true;
}
void parse_ll_t::accept_token_job_list(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_job_list);
switch (token.type)
{
case parse_token_type_string:
// 'end' is special
switch (token.keyword)
{
case parse_keyword_end:
case parse_keyword_else:
// End this job list
symbol_stack_pop_push_production<job_list>(0);
break;
default:
// Normal string
symbol_stack_pop_push_production<job_list>(1);
break;
}
break;
case parse_token_type_pipe:
case parse_token_type_redirection:
case parse_token_background:
symbol_stack_pop_push_production<job_list>(1);
break;
case parse_token_type_end:
// Empty line
symbol_stack_pop_push_production<job_list>(2);
break;
case parse_token_type_terminate:
// no more commands, just transition to empty
symbol_stack_pop_push_production<job_list>(0);
break;
default:
token_unhandled(token, __FUNCTION__);
break;
}
}
void parse_ll_t::accept_token_job_continuation(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_job_continuation);
switch (token.type)
{
case parse_token_type_pipe:
// Pipe, continuation
symbol_stack_pop_push_production<job_continuation>(1);
break;
default:
// Not a pipe, no job continuation
symbol_stack_pop_push_production<job_continuation>(0);
break;
}
}
void parse_ll_t::accept_token_statement(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_statement);
switch (token.type)
{
case parse_token_type_string:
switch (token.keyword)
{
case parse_keyword_and:
case parse_keyword_or:
case parse_keyword_not:
symbol_stack_pop_push_production<statement>(0);
break;
case parse_keyword_for:
case parse_keyword_while:
case parse_keyword_function:
case parse_keyword_begin:
symbol_stack_pop_push_production<statement>(1);
break;
case parse_keyword_if:
symbol_stack_pop_push_production<statement>(2);
break;
case parse_keyword_else:
symbol_stack_pop();
break;
case parse_keyword_switch:
symbol_stack_pop_push_production<statement>(3);
break;
case parse_keyword_end:
PARSER_DIE(); //todo
break;
// 'in' is only special within a for_header
case parse_keyword_in:
case parse_keyword_none:
case parse_keyword_command:
case parse_keyword_builtin:
case parse_keyword_case:
symbol_stack_pop_push_production<statement>(4);
break;
}
break;
case parse_token_type_pipe:
case parse_token_type_redirection:
case parse_token_background:
case parse_token_type_terminate:
parse_error(L"statement", token);
break;
default:
token_unhandled(token, __FUNCTION__);
break;
}
}
void parse_ll_t::accept_token_block_header(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_block_header);
switch (token.type)
{
case parse_token_type_string:
switch (token.keyword)
{
case parse_keyword_else:
PARSER_DIE(); //todo
break;
case parse_keyword_for:
symbol_stack_pop_push_production<block_header>(0);
break;
case parse_keyword_while:
symbol_stack_pop_push_production<block_header>(1);
break;
case parse_keyword_function:
symbol_stack_pop_push_production<block_header>(2);
break;
case parse_keyword_begin:
symbol_stack_pop_push_production<block_header>(3);
break;
default:
token_unhandled(token, __FUNCTION__);
break;
}
break;
default:
token_unhandled(token, __FUNCTION__);
break;
}
}
void parse_ll_t::accept_token_else_clause(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_else_clause);
@ -614,25 +563,6 @@ void parse_ll_t::accept_token_else_continuation(parse_token_t token)
symbol_stack_produce<else_continuation>(token);
}
void parse_ll_t::accept_token_boolean_statement(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_boolean_statement);
top_node_set_tag(token.keyword);
symbol_stack_produce<boolean_statement>(token);
}
void parse_ll_t::accept_token_decorated_statement(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_decorated_statement);
top_node_set_tag(token.keyword);
symbol_stack_produce<case_item_list>(token);
}
void parse_ll_t::accept_token_plain_statement(parse_token_t token)
{
PARSE_ASSERT(stack_top_type() == symbol_plain_statement);
symbol_stack_produce<case_item_list>(token);
}
void parse_ll_t::accept_token_argument_list(parse_token_t token)
{
@ -664,7 +594,7 @@ bool parse_ll_t::accept_token_string(parse_token_t token)
symbol_stack_pop();
result = true;
break;
default:
token_unhandled(token, __FUNCTION__);
break;
@ -687,7 +617,7 @@ bool parse_ll_t::top_node_match_token(parse_token_t token)
parse_node_t &node = node_for_top_symbol();
node.source_start = token.source_start;
node.source_length = token.source_length;
// We consumed this symbol
symbol_stack.pop_back();
result = true;
@ -704,7 +634,7 @@ bool parse_ll_t::top_node_match_token(parse_token_t token)
void parse_ll_t::accept_token(parse_token_t token, const wcstring &src)
{
bool logit = true;
bool logit = false;
if (logit)
{
const wcstring txt = wcstring(src, token.source_start, token.source_length);
@ -724,107 +654,113 @@ void parse_ll_t::accept_token(parse_token_t token, const wcstring &src)
consumed = true;
break;
}
switch (stack_top_type())
{
/* Symbols */
/* Symbols */
case symbol_job_list:
accept_token_job_list(token);
symbol_stack_produce<job_list>(token);
break;
case symbol_job:
symbol_stack_pop_push2<parse_symbols::job>();
symbol_stack_produce<job>(token);
break;
case symbol_job_continuation:
accept_token_job_continuation(token);
symbol_stack_produce<job_continuation>(token);
break;
case symbol_statement:
accept_token_statement(token);
symbol_stack_produce<statement>(token);
break;
case symbol_if_statement:
symbol_stack_produce<if_statement>(token);
break;
case symbol_if_clause:
symbol_stack_produce<if_clause>(token);
break;
case symbol_else_clause:
accept_token_else_clause(token);
break;
case symbol_else_continuation:
accept_token_else_continuation(token);
break;
case symbol_block_statement:
symbol_stack_produce<block_statement>(token);
break;
case symbol_block_header:
symbol_stack_produce<block_header>(token);
break;
case symbol_for_header:
symbol_stack_produce<for_header>(token);
break;
case symbol_while_header:
symbol_stack_produce<while_header>(token);
break;
case symbol_begin_header:
symbol_stack_produce<begin_header>(token);
break;
case symbol_function_header:
symbol_stack_produce<function_header>(token);
break;
case symbol_switch_statement:
symbol_stack_produce<switch_statement>(token);
break;
case symbol_case_item_list:
symbol_stack_produce<case_item_list>(token);
break;
case symbol_case_item:
symbol_stack_produce<case_item>(token);
break;
case symbol_boolean_statement:
accept_token_boolean_statement(token);
top_node_set_tag(token.keyword);
symbol_stack_produce<boolean_statement>(token);
break;
case symbol_decorated_statement:
accept_token_decorated_statement(token);
top_node_set_tag(token.keyword);
symbol_stack_produce<decorated_statement>(token);
break;
case symbol_plain_statement:
accept_token_plain_statement(token);
symbol_stack_produce<plain_statement>(token);
break;
case symbol_argument_list_nonempty:
symbol_stack_produce<argument_list_nonempty>(token);
break;
case symbol_argument_list:
accept_token_argument_list(token);
break;
case symbol_arguments_or_redirections_list:
accept_token_arguments_or_redirections_list(token);
break;
case symbol_argument_or_redirection:
accept_token_argument_or_redirection(token);
break;
/* Tokens */
case symbol_optional_background:
symbol_stack_produce<optional_background>(token);
break;
/* Tokens */
case parse_token_type_string:
consumed = accept_token_string(token);
break;
@ -846,11 +782,13 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
parse_keyword_t result = parse_keyword_none;
if (tok == TOK_STRING)
{
const struct {
const struct
{
const wchar_t *txt;
parse_keyword_t keyword;
} keywords[] = {
} keywords[] =
{
{L"if", parse_keyword_if},
{L"else", parse_keyword_else},
{L"for", parse_keyword_for},
@ -867,7 +805,7 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
{L"command", parse_keyword_command},
{L"builtin", parse_keyword_builtin}
};
for (size_t i=0; i < sizeof keywords / sizeof *keywords; i++)
{
if (! wcscmp(keywords[i].txt, tok_txt))
@ -888,36 +826,40 @@ bool parse_t::parse(const wcstring &str, parse_node_tree_t *output, parse_error_
token_type tok_type = static_cast<token_type>(tok_last_type(&tok));
const wchar_t *tok_txt = tok_last(&tok);
int tok_start = tok_get_pos(&tok);
size_t tok_extent = tok_get_extent(&tok);
if (tok_type == TOK_ERROR)
{
fprintf(stderr, "Tokenizer error\n");
break;
}
parse_token_t token = parse_token_from_tokenizer_token(tok_type);
token.tokenizer_type = tok_type;
token.source_start = (size_t)tok_start;
token.source_length = wcslen(tok_txt);
token.source_length = tok_extent;
token.keyword = keyword_for_token(tok_type, tok_txt);
this->parser->accept_token(token, str);
if (this->parser->fatal_errored)
break;
}
wcstring result = dump_tree(this->parser->nodes, str);
wcstring result = L"";//dump_tree(this->parser->nodes, str);
fprintf(stderr, "Tree (%ld nodes):\n%ls", this->parser->nodes.size(), result.c_str());
fprintf(stderr, "%lu nodes, node size %lu, %lu bytes\n", this->parser->nodes.size(), sizeof(parse_node_t), this->parser->nodes.size() * sizeof(parse_node_t));
if (output != NULL)
{
output->swap(this->parser->nodes);
this->parser->nodes.clear();
}
if (errors != NULL)
{
errors->swap(this->parser->errors);
this->parser->errors.clear();
}
return ! this->parser->fatal_errored;
}

414
parse_tree.h
View file

@ -26,11 +26,11 @@ struct parse_error_t
{
/** Text of the error */
wcstring text;
/** Offset and length of the token in the source code that triggered this error */
size_t source_start;
size_t source_length;
/** Return a string describing the error, suitable for presentation to the user */
wcstring describe(const wcstring &src) const;
};
@ -40,8 +40,8 @@ class parse_ll_t;
class parse_t
{
parse_ll_t * const parser;
public:
public:
parse_t();
bool parse(const wcstring &str, parse_node_tree_t *output, parse_error_list_t *errors);
};
@ -49,7 +49,7 @@ class parse_t
enum parse_token_type_t
{
token_type_invalid,
// Non-terminal tokens
symbol_job_list,
symbol_job,
@ -61,33 +61,35 @@ enum parse_token_type_t
symbol_while_header,
symbol_begin_header,
symbol_function_header,
symbol_if_statement,
symbol_if_clause,
symbol_else_clause,
symbol_else_continuation,
symbol_switch_statement,
symbol_case_item_list,
symbol_case_item,
symbol_boolean_statement,
symbol_decorated_statement,
symbol_plain_statement,
symbol_arguments_or_redirections_list,
symbol_argument_or_redirection,
symbol_argument_list_nonempty,
symbol_argument_list,
symbol_optional_background,
// Terminal types
parse_token_type_string,
parse_token_type_pipe,
parse_token_type_redirection,
parse_token_background,
parse_token_type_background,
parse_token_type_end,
parse_token_type_terminate,
FIRST_PARSE_TOKEN_TYPE = parse_token_type_string
};
@ -117,32 +119,32 @@ wcstring keyword_description(parse_keyword_t type);
/** Base class for nodes of a parse tree */
class parse_node_t
{
public:
public:
/* Type of the node */
enum parse_token_type_t type;
/* Start in the source code */
size_t source_start;
/* Length of our range in the source code */
size_t source_length;
/* Children */
node_offset_t child_start;
node_offset_t child_count;
/* Type-dependent data */
uint32_t tag;
/* Description */
wcstring describe(void) const;
/* Constructor */
explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0), child_start(0), child_count(0), tag(0)
{
}
node_offset_t child_offset(node_offset_t which) const
{
PARSE_ASSERT(which < child_count);
@ -154,360 +156,6 @@ class parse_node_tree_t : public std::vector<parse_node_t>
{
};
namespace parse_symbols
{
#define SYMBOL(x) static inline parse_token_type_t get_token() { return x; }
#define PRODUCE(X) static int production(parse_token_type_t tok, parse_keyword_t key) { return X; }
#define NO_PRODUCTION (-1)
template<parse_token_type_t WHICH>
struct Token
{
SYMBOL(WHICH);
typedef Token<WHICH> t0;
typedef Token<token_type_invalid> t1;
typedef Token<token_type_invalid> t2;
typedef Token<token_type_invalid> t3;
typedef Token<token_type_invalid> t4;
typedef Token<token_type_invalid> t5;
};
/* Placeholder */
typedef Token<token_type_invalid> none;
struct EMPTY
{
typedef none t0;
typedef none t1;
typedef none t2;
typedef none t3;
typedef none t4;
typedef none t5;
};
template<typename T0, typename T1, typename T2 = none, typename T3 = none, typename T4 = none, typename T5 = none>
struct Seq
{
typedef T0 t0;
typedef T1 t1;
typedef T2 t2;
typedef T3 t3;
typedef T4 t4;
typedef T5 t5;
};
template<typename P0, typename P1, typename P2 = none, typename P3 = none, typename P4 = none, typename P5 = none>
struct OR
{
typedef P0 p0;
typedef P1 p1;
typedef P2 p2;
typedef P3 p3;
typedef P4 p4;
typedef P5 p5;
};
template<parse_keyword_t WHICH>
struct Keyword
{
static inline parse_keyword_t get_token() { return WHICH; }
};
struct job;
struct statement;
struct job_continuation;
struct boolean_statement;
struct block_statement;
struct if_statement;
struct if_clause;
struct else_clause;
struct else_continuation;
struct switch_statement;
struct decorated_statement;
struct switch_statement;
struct case_item_list;
struct case_item;
struct argument_list_nonempty;
struct argument_list;
struct block_statement;
struct block_header;
struct for_header;
struct while_header;
struct begin_header;
struct function_header;
struct boolean_statement;
struct decorated_statement;
struct plain_statement;
struct arguments_or_redirections_list;
struct argument_or_redirection;
struct redirection;
struct statement_terminator;
/* A job_list is a list of jobs, separated by semicolons or newlines */
struct job_list : OR<
EMPTY,
Seq<job, job_list>,
Seq<Token<parse_token_type_end>, job_list>
>
{
SYMBOL(symbol_job_list)
};
/* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */
struct job : Seq<statement, job_continuation>
{
SYMBOL(symbol_job);
};
struct job_continuation : OR<
EMPTY,
Seq<Token<parse_token_type_pipe>, statement, job_continuation>
>
{
SYMBOL(symbol_job_continuation);
};
/* A statement is a normal command, or an if / while / and etc */
struct statement : OR<
boolean_statement,
block_statement,
if_statement,
switch_statement,
decorated_statement
>
{
SYMBOL(symbol_statement);
};
struct if_statement : Seq<if_clause, else_clause, Keyword<parse_keyword_end> >
{
SYMBOL(symbol_if_statement);
PRODUCE(0)
};
struct if_clause : Seq<Keyword<parse_keyword_if>, job, statement_terminator, job_list>
{
SYMBOL(symbol_if_clause);
PRODUCE(0)
};
struct else_clause : OR<
EMPTY,
Keyword<parse_keyword_else>, else_continuation
>
{
SYMBOL(symbol_else_clause);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_else: return 1;
default: return 0;
}
}
};
struct else_continuation : OR<
Seq<if_clause, else_clause>,
Seq<statement_terminator, job_list>
>
{
SYMBOL(symbol_else_continuation);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_if: return 0;
default: return 1;
}
}
};
struct switch_statement : Seq<Keyword<parse_keyword_switch>, Token<parse_token_type_string>, statement_terminator, case_item_list, Keyword<parse_keyword_end>
>
{
SYMBOL(symbol_switch_statement);
};
struct case_item_list : OR
<
EMPTY,
case_item, case_item_list
>
{
SYMBOL(symbol_case_item_list);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_case: return 1;
default: return 0;
}
}
};
struct case_item : Seq<Keyword<parse_keyword_case>, argument_list, statement_terminator, job_list>
{
SYMBOL(symbol_case_item);
};
struct argument_list_nonempty : Seq<Token<parse_token_type_string>, argument_list>
{
SYMBOL(symbol_argument_list_nonempty);
};
struct argument_list : OR<EMPTY, argument_list_nonempty>
{
SYMBOL(symbol_argument_list);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (tok)
{
case parse_token_type_string: return 1;
default: return 0;
}
}
};
struct block_statement : Seq<block_header, statement_terminator, job_list, Keyword<parse_keyword_end>, arguments_or_redirections_list>
{
SYMBOL(symbol_block_statement);
PRODUCE(0)
};
struct block_header : OR<for_header, while_header, function_header, begin_header>
{
SYMBOL(symbol_block_header);
};
struct for_header : Seq<Keyword<parse_keyword_for>, Token<parse_token_type_string>, Keyword<parse_keyword_in>, arguments_or_redirections_list>
{
SYMBOL(symbol_for_header);
};
struct while_header : Seq<Keyword<parse_keyword_while>, statement>
{
SYMBOL(symbol_while_header);
};
struct begin_header : Keyword<parse_keyword_begin>
{
SYMBOL(symbol_begin_header);
};
struct function_header : Keyword<parse_keyword_function>
{
SYMBOL(symbol_function_header);
};
/* A boolean statement is AND or OR or NOT */
struct boolean_statement : OR<
Seq<Keyword<parse_keyword_and>, statement>,
Seq<Keyword<parse_keyword_or>, statement>,
Seq<Keyword<parse_keyword_not>, statement>
>
{
SYMBOL(symbol_boolean_statement);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_and: return 0;
case parse_keyword_or: return 1;
case parse_keyword_not: return 2;
default: return NO_PRODUCTION;
}
}
};
/* A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" */
struct decorated_statement : OR<
Seq<Keyword<parse_keyword_command>, plain_statement>,
Seq<Keyword<parse_keyword_builtin>, plain_statement>,
plain_statement
>
{
SYMBOL(symbol_decorated_statement);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_command: return 0;
case parse_keyword_builtin: return 1;
default: return 2;
}
}
};
struct plain_statement : Seq<Token<parse_token_type_string>, arguments_or_redirections_list>
{
SYMBOL(symbol_plain_statement);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
return 0;
}
};
struct arguments_or_redirections_list : OR<
EMPTY,
Seq<argument_or_redirection, arguments_or_redirections_list> >
{
SYMBOL(symbol_arguments_or_redirections_list);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (tok)
{
case parse_token_type_string:
case parse_token_type_redirection:
return 1;
default:
return 0;
}
}
};
struct argument_or_redirection : OR<
Token<parse_token_type_string>,
redirection
>
{
SYMBOL(symbol_argument_or_redirection);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (tok)
{
case parse_token_type_string: return 0;
case parse_token_type_redirection: return 1;
default: return NO_PRODUCTION;
}
}
};
struct redirection : Token<parse_token_type_redirection>
{
SYMBOL(parse_token_type_redirection);
};
struct statement_terminator : Token<parse_token_type_end>
{
SYMBOL(parse_token_type_end);
};
}
/* Fish grammar:
@ -520,45 +168,45 @@ namespace parse_symbols
# A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation
job = statement job_continuation
job_continuation = <empty> |
job_continuation = <empty> |
<TOK_PIPE> statement job_continuation
# A statement is a normal command, or an if / while / and etc
statement = boolean_statement | block_statement | if_statement | switch_statement | decorated_statement
# A block is a conditional, loop, or begin/end
if_statement = if_clause else_clause <END>
if_statement = if_clause else_clause <END> arguments_or_redirections_list
if_clause = <IF> job STATEMENT_TERMINATOR job_list
else_clause = <empty> |
<ELSE> else_continuation
else_continuation = if_clause else_clause |
STATEMENT_TERMINATOR job_list
switch_statement = SWITCH <TOK_STRING> STATEMENT_TERMINATOR case_item_list <END>
case_item_list = <empty> |
case_item case_item_list
case_item = CASE argument_list STATEMENT_TERMINATOR job_list
argument_list_nonempty = <TOK_STRING> argument_list
argument_list = <empty> | argument_list_nonempty
block_statement = block_header STATEMENT_TERMINATOR job_list <END> arguments_or_redirections_list
block_statement = block_header <TOK_END> job_list <END> arguments_or_redirections_list
block_header = for_header | while_header | function_header | begin_header
for_header = FOR var_name IN arguments_or_redirections_list
while_header = WHILE statement
begin_header = BEGIN
function_header = FUNCTION function_name argument_list
# A boolean statement is AND or OR or NOT
boolean_statement = AND statement | OR statement | NOT statement
# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command"
decorated_statement = COMMAND plain_statement | BUILTIN plain_statement | plain_statement
plain_statement = COMMAND arguments_or_redirections_list
plain_statement = COMMAND arguments_or_redirections_list optional_background
arguments_or_redirections_list = <empty> |
argument_or_redirection arguments_or_redirections_list
@ -567,6 +215,8 @@ namespace parse_symbols
terminator = <TOK_END> | <TOK_BACKGROUND>
optional_background = <empty> | <TOK_BACKGROUND>
*/
#endif

586
parse_tree_construction.h Normal file
View file

@ -0,0 +1,586 @@
/**\file parse_tree.h
Programmatic representation of fish code.
*/
#ifndef FISH_PARSE_TREE_CONSTRUCTION_H
#define FISH_PARSE_TREE_CONSTRUCTION_H
#include "parse_tree.h"
/* Terrifying template black magic. */
namespace parse_symbols
{
#define SYMBOL(x) static inline parse_token_type_t get_token() { return x; }
#define PRODUCE(X) static int production(parse_token_type_t tok, parse_keyword_t key) { return X; }
#define NO_PRODUCTION (-1)
struct Symbol
{
typedef int magic_symbol_type_t;
};
template<parse_token_type_t WHICH>
struct Token : public Symbol
{
SYMBOL(WHICH);
};
/* Placeholder */
typedef Token<token_type_invalid> none;
typedef Token<token_type_invalid> EMPTY;
template<typename T0, typename T1, typename T2 = none, typename T3 = none, typename T4 = none, typename T5 = none>
struct Seq
{
typedef T0 t0;
typedef T1 t1;
typedef T2 t2;
typedef T3 t3;
typedef T4 t4;
typedef T5 t5;
typedef int magic_seq_type_t;
};
template<typename P0, typename P1, typename P2 = none, typename P3 = none, typename P4 = none, typename P5 = none>
struct OR
{
typedef P0 p0;
typedef P1 p1;
typedef P2 p2;
typedef P3 p3;
typedef P4 p4;
typedef P5 p5;
typedef int magic_or_type_t;
};
template<parse_keyword_t WHICH>
struct Keyword : public Symbol
{
static inline parse_keyword_t get_token()
{
return WHICH;
}
};
struct job;
struct statement;
struct job_continuation;
struct boolean_statement;
struct block_statement;
struct if_statement;
struct if_clause;
struct else_clause;
struct else_continuation;
struct switch_statement;
struct decorated_statement;
struct switch_statement;
struct case_item_list;
struct case_item;
struct argument_list_nonempty;
struct argument_list;
struct block_statement;
struct block_header;
struct for_header;
struct while_header;
struct begin_header;
struct function_header;
struct boolean_statement;
struct decorated_statement;
struct plain_statement;
struct arguments_or_redirections_list;
struct argument_or_redirection;
struct redirection;
struct statement_terminator;
struct optional_background;
/* A job_list is a list of jobs, separated by semicolons or newlines */
struct job_list : public Symbol
{
typedef OR<
EMPTY,
Seq<job, job_list>,
Seq<Token<parse_token_type_end>, job_list>
> productions;
SYMBOL(symbol_job_list)
static int production(parse_token_type_t token_type, parse_keyword_t token_keyword)
{
switch (token_type)
{
case parse_token_type_string:
// 'end' is special
switch (token_keyword)
{
case parse_keyword_end:
case parse_keyword_else:
// End this job list
return 0;
default:
// Normal string
return 1;
}
case parse_token_type_pipe:
case parse_token_type_redirection:
case parse_token_type_background:
return 1;
case parse_token_type_end:
// Empty line
return 2;
case parse_token_type_terminate:
// no more commands, just transition to empty
return 0;
break;
default:
return NO_PRODUCTION;
}
}
};
/* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */
struct job : public Symbol
{
typedef Seq<statement, job_continuation> sole_production;
SYMBOL(symbol_job);
};
struct job_continuation : public Symbol
{
typedef OR<
EMPTY,
Seq<Token<parse_token_type_pipe>, statement, job_continuation>
> productions;
SYMBOL(symbol_job_continuation);
static int production(parse_token_type_t token_type, parse_keyword_t token_keyword)
{
switch (token_type)
{
case parse_token_type_pipe:
// Pipe, continuation
return 1;
default:
// Not a pipe, no job continuation
return 0;
}
}
};
/* A statement is a normal command, or an if / while / and etc */
struct statement : public Symbol
{
typedef OR<
boolean_statement,
block_statement,
if_statement,
switch_statement,
decorated_statement
> productions;
SYMBOL(symbol_statement);
static int production(parse_token_type_t token_type, parse_keyword_t token_keyword)
{
switch (token_type)
{
case parse_token_type_string:
switch (token_keyword)
{
case parse_keyword_and:
case parse_keyword_or:
case parse_keyword_not:
return 0;
case parse_keyword_for:
case parse_keyword_while:
case parse_keyword_function:
case parse_keyword_begin:
return 1;
case parse_keyword_if:
return 2;
case parse_keyword_else:
//symbol_stack_pop();
return NO_PRODUCTION;
case parse_keyword_switch:
return 3;
case parse_keyword_end:
PARSER_DIE(); //todo
return NO_PRODUCTION;
// 'in' is only special within a for_header
case parse_keyword_in:
case parse_keyword_none:
case parse_keyword_command:
case parse_keyword_builtin:
case parse_keyword_case:
return 4;
}
break;
case parse_token_type_pipe:
case parse_token_type_redirection:
case parse_token_type_background:
case parse_token_type_terminate:
return NO_PRODUCTION;
//parse_error(L"statement", token);
default:
return NO_PRODUCTION;
}
}
};
struct if_statement : public Symbol
{
typedef Seq<if_clause, else_clause, Keyword<parse_keyword_end>, arguments_or_redirections_list> sole_production;
SYMBOL(symbol_if_statement);
};
struct if_clause : public Symbol
{
typedef Seq<Keyword<parse_keyword_if>, job, statement_terminator, job_list> sole_production;
SYMBOL(symbol_if_clause);
};
struct else_clause : public Symbol
{
typedef OR<
EMPTY,
Seq<Keyword<parse_keyword_else>, else_continuation>
> productions;
SYMBOL(symbol_else_clause);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_else:
return 1;
default:
return 0;
}
}
};
struct else_continuation : public Symbol
{
typedef OR<
Seq<if_clause, else_clause>,
Seq<statement_terminator, job_list>
> productions;
SYMBOL(symbol_else_continuation);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_if:
return 0;
default:
return 1;
}
}
};
struct switch_statement : public Symbol
{
typedef Seq<Keyword<parse_keyword_switch>,
Token<parse_token_type_string>,
statement_terminator,
case_item_list,
Keyword<parse_keyword_end>
> sole_production;
SYMBOL(symbol_switch_statement);
};
struct case_item_list : public Symbol
{
typedef OR
<
EMPTY,
Seq<case_item, case_item_list>,
Seq<Token<parse_token_type_end>, case_item_list>
> productions;
SYMBOL(symbol_case_item_list);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_case: return 1;
default:
if (tok == parse_token_type_end)
{
/* empty line */
return 2;
}
else
{
return 0;
}
}
}
};
struct case_item : public Symbol
{
typedef Seq<Keyword<parse_keyword_case>, argument_list, statement_terminator, job_list> sole_production;
SYMBOL(symbol_case_item);
};
struct argument_list_nonempty : public Symbol
{
typedef Seq<Token<parse_token_type_string>, argument_list> sole_production;
SYMBOL(symbol_argument_list_nonempty);
};
struct argument_list : public Symbol
{
typedef OR<EMPTY, argument_list_nonempty> productions;
SYMBOL(symbol_argument_list);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (tok)
{
case parse_token_type_string:
return 1;
default:
return 0;
}
}
};
struct block_statement : public Symbol
{
typedef Seq<block_header, statement_terminator, job_list, Keyword<parse_keyword_end>, arguments_or_redirections_list> sole_production;
SYMBOL(symbol_block_statement);
};
struct block_header : public Symbol
{
typedef OR<for_header, while_header, function_header, begin_header> productions;
SYMBOL(symbol_block_header);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
// todo
case parse_keyword_else:
return NO_PRODUCTION;
case parse_keyword_for:
return 0;
case parse_keyword_while:
return 1;
case parse_keyword_function:
return 2;
case parse_keyword_begin:
return 3;
default:
return NO_PRODUCTION;
}
}
};
struct for_header : public Symbol
{
typedef Seq<Keyword<parse_keyword_for>, Token<parse_token_type_string>, Keyword<parse_keyword_in>, arguments_or_redirections_list> sole_production;
SYMBOL(symbol_for_header);
};
struct while_header : public Symbol
{
typedef Seq<Keyword<parse_keyword_while>, statement> sole_production;
SYMBOL(symbol_while_header);
};
struct begin_header : public Symbol
{
typedef Keyword<parse_keyword_begin> sole_production;
SYMBOL(symbol_begin_header);
};
struct function_header : public Symbol
{
typedef Seq< Keyword<parse_keyword_function>, Token<parse_token_type_string>, argument_list> sole_production;
SYMBOL(symbol_function_header);
};
/* A boolean statement is AND or OR or NOT */
struct boolean_statement : public Symbol
{
typedef OR<
Seq<Keyword<parse_keyword_and>, statement>,
Seq<Keyword<parse_keyword_or>, statement>,
Seq<Keyword<parse_keyword_not>, statement>
> productions;
SYMBOL(symbol_boolean_statement);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_and:
return 0;
case parse_keyword_or:
return 1;
case parse_keyword_not:
return 2;
default:
return NO_PRODUCTION;
}
}
};
/* A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" */
struct decorated_statement : public Symbol
{
typedef OR<
Seq<Keyword<parse_keyword_command>, plain_statement>,
Seq<Keyword<parse_keyword_builtin>, plain_statement>,
plain_statement
> productions;
SYMBOL(symbol_decorated_statement);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (key)
{
case parse_keyword_command:
return 0;
case parse_keyword_builtin:
return 1;
default:
return 2;
}
}
};
struct plain_statement : public Symbol
{
typedef Seq<Token<parse_token_type_string>, arguments_or_redirections_list, optional_background> sole_production;
SYMBOL(symbol_plain_statement);
};
struct arguments_or_redirections_list : public Symbol
{
typedef OR<
EMPTY,
Seq<argument_or_redirection, arguments_or_redirections_list> >
productions;
SYMBOL(symbol_arguments_or_redirections_list);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (tok)
{
case parse_token_type_string:
case parse_token_type_redirection:
return 1;
default:
return 0;
}
}
};
struct argument_or_redirection : public Symbol
{
typedef OR<
Token<parse_token_type_string>,
redirection
> productions;
SYMBOL(symbol_argument_or_redirection);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (tok)
{
case parse_token_type_string:
return 0;
case parse_token_type_redirection:
return 1;
default:
return NO_PRODUCTION;
}
}
};
struct redirection : public Symbol
{
typedef Token<parse_token_type_redirection> production;
SYMBOL(parse_token_type_redirection);
};
struct statement_terminator : public Symbol
{
typedef Token<parse_token_type_end> production;
SYMBOL(parse_token_type_end);
};
struct optional_background : public Symbol
{
typedef OR<
EMPTY,
Token<parse_token_type_background>
> productions;
SYMBOL(symbol_optional_background);
static int production(parse_token_type_t tok, parse_keyword_t key)
{
switch (tok)
{
case parse_token_type_background:
return 1;
default:
return 0;
}
}
};
}
#endif

10
parse_util.cpp
View file

@ -245,10 +245,10 @@ void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wc
const wchar_t * const cursor = buff + cursor_pos;
CHECK(buff,);
const size_t bufflen = wcslen(buff);
assert(cursor_pos <= bufflen);
/* ap and bp are the beginning and end of the tightest command substitition found so far */
const wchar_t *ap = buff, *bp = buff + bufflen;
const wchar_t *pos = buff;
@ -260,13 +260,13 @@ void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wc
/* No subshell found, all done */
break;
}
/* Intrepret NULL to mean the end */
if (end == NULL)
{
end = const_cast<wchar_t *>(buff) + bufflen;
}
if (begin < cursor && end >= cursor)
{
/* This command substitution surrounds the cursor, so it's a tighter fit */
@ -288,7 +288,7 @@ void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wc
assert(pos <= buff + bufflen);
}
}
if (a != NULL) *a = ap;
if (b != NULL) *b = bp;
}

10
tokenizer.cpp
View file

@ -651,13 +651,19 @@ wcstring tok_first(const wchar_t *str)
return result;
}
int tok_get_pos(tokenizer_t *tok)
int tok_get_pos(const tokenizer_t *tok)
{
CHECK(tok, 0);
return (int)tok->last_pos;
}
size_t tok_get_extent(const tokenizer_t *tok)
{
CHECK(tok, 0);
size_t current_pos = tok->buff - tok->orig_buff;
return current_pos > tok->last_pos ? current_pos - tok->last_pos : 0;
}
void tok_set_pos(tokenizer_t *tok, int pos)
{

5
tokenizer.h
View file

@ -142,7 +142,10 @@ int tok_has_next(tokenizer_t *tok);
/**
Returns the position of the beginning of the current token in the original string
*/
int tok_get_pos(tokenizer_t *tok);
int tok_get_pos(const tokenizer_t *tok);
/** Returns the extent of the current token */
size_t tok_get_extent(const tokenizer_t *tok);
/**
Returns the original string to tokenizer