Mirrors/fish-shell
2
0
Fork 0
mirror of https://github.com/fish-shell/fish-shell synced 2025-01-01 07:38:46 +00:00
Code Issues Projects Releases Packages Wiki Activity
fish-shell/src/highlight.cpp
ridiculousfish 8d54d2b60e Skip long arguments in syntax highlighting path detection 
When fish performs syntax highlighting, it attempts to determine which
arguments are valid paths and underline them. Skip paths whose length
exceeds PATH_MAX. This is an optimization: such strings are almost
certainly not valid paths and checking them may be expensive.

Relevant is #7837
2021-03-21 16:59:22 -07:00

1362 lines
55 KiB
C++
Raw Blame History

// Functions for syntax highlighting.
#include "config.h" // IWYU pragma: keep
// IWYU pragma: no_include <cstddef>
#include <dirent.h>
#include <errno.h>
#include <sys/stat.h>
#include <unistd.h>
#include <algorithm>
#include <cwchar>
#include <memory>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include "ast.h"
#include "builtin.h"
#include "color.h"
#include "common.h"
#include "env.h"
#include "expand.h"
#include "fallback.h" // IWYU pragma: keep
#include "function.h"
#include "future_feature_flags.h"
#include "highlight.h"
#include "history.h"
#include "output.h"
#include "parse_constants.h"
#include "parse_util.h"
#include "parser.h"
#include "path.h"
#include "tokenizer.h"
#include "wcstringutil.h"
#include "wildcard.h"
#include "wutil.h" // IWYU pragma: keep
#define CURSOR_POSITION_INVALID static_cast<size_t>(-1)
static const wchar_t *get_highlight_var_name(highlight_role_t role) {
switch (role) {
case highlight_role_t::normal:
return L"fish_color_normal";
case highlight_role_t::error:
return L"fish_color_error";
case highlight_role_t::command:
return L"fish_color_command";
case highlight_role_t::keyword:
return L"fish_color_keyword";
case highlight_role_t::statement_terminator:
return L"fish_color_end";
case highlight_role_t::param:
return L"fish_color_param";
case highlight_role_t::comment:
return L"fish_color_comment";
case highlight_role_t::search_match:
return L"fish_color_search_match";
case highlight_role_t::operat:
return L"fish_color_operator";
case highlight_role_t::escape:
return L"fish_color_escape";
case highlight_role_t::quote:
return L"fish_color_quote";
case highlight_role_t::redirection:
return L"fish_color_redirection";
case highlight_role_t::autosuggestion:
return L"fish_color_autosuggestion";
case highlight_role_t::selection:
return L"fish_color_selection";
case highlight_role_t::pager_progress:
return L"fish_pager_color_progress";
case highlight_role_t::pager_background:
return L"fish_pager_color_background";
case highlight_role_t::pager_prefix:
return L"fish_pager_color_prefix";
case highlight_role_t::pager_completion:
return L"fish_pager_color_completion";
case highlight_role_t::pager_description:
return L"fish_pager_color_description";
case highlight_role_t::pager_secondary_background:
return L"fish_pager_color_secondary_background";
case highlight_role_t::pager_secondary_prefix:
return L"fish_pager_color_secondary_prefix";
case highlight_role_t::pager_secondary_completion:
return L"fish_pager_color_secondary_completion";
case highlight_role_t::pager_secondary_description:
return L"fish_pager_color_secondary_description";
case highlight_role_t::pager_selected_background:
return L"fish_pager_color_selected_background";
case highlight_role_t::pager_selected_prefix:
return L"fish_pager_color_selected_prefix";
case highlight_role_t::pager_selected_completion:
return L"fish_pager_color_selected_completion";
case highlight_role_t::pager_selected_description:
return L"fish_pager_color_selected_description";
}
DIE("invalid highlight role");
}
// Table used to fetch fallback highlights in case the specified one
// wasn't set.
static highlight_role_t get_fallback(highlight_role_t role) {
switch (role) {
case highlight_role_t::normal:
return highlight_role_t::normal;
case highlight_role_t::error:
return highlight_role_t::normal;
case highlight_role_t::command:
return highlight_role_t::normal;
case highlight_role_t::keyword:
return highlight_role_t::command;
case highlight_role_t::statement_terminator:
return highlight_role_t::normal;
case highlight_role_t::param:
return highlight_role_t::normal;
case highlight_role_t::comment:
return highlight_role_t::normal;
case highlight_role_t::search_match:
return highlight_role_t::normal;
case highlight_role_t::operat:
return highlight_role_t::normal;
case highlight_role_t::escape:
return highlight_role_t::normal;
case highlight_role_t::quote:
return highlight_role_t::normal;
case highlight_role_t::redirection:
return highlight_role_t::normal;
case highlight_role_t::autosuggestion:
return highlight_role_t::normal;
case highlight_role_t::selection:
return highlight_role_t::normal;
case highlight_role_t::pager_progress:
return highlight_role_t::normal;
case highlight_role_t::pager_background:
return highlight_role_t::normal;
case highlight_role_t::pager_prefix:
return highlight_role_t::normal;
case highlight_role_t::pager_completion:
return highlight_role_t::normal;
case highlight_role_t::pager_description:
return highlight_role_t::normal;
case highlight_role_t::pager_secondary_background:
return highlight_role_t::pager_background;
case highlight_role_t::pager_secondary_prefix:
return highlight_role_t::pager_prefix;
case highlight_role_t::pager_secondary_completion:
return highlight_role_t::pager_completion;
case highlight_role_t::pager_secondary_description:
return highlight_role_t::pager_description;
case highlight_role_t::pager_selected_background:
return highlight_role_t::search_match;
case highlight_role_t::pager_selected_prefix:
return highlight_role_t::pager_prefix;
case highlight_role_t::pager_selected_completion:
return highlight_role_t::pager_completion;
case highlight_role_t::pager_selected_description:
return highlight_role_t::pager_description;
}
DIE("invalid highlight role");
}
/// Determine if the filesystem containing the given fd is case insensitive for lookups regardless
/// of whether it preserves the case when saving a pathname.
///
/// Returns:
/// false: the filesystem is not case insensitive
/// true: the file system is case insensitive
using case_sensitivity_cache_t = std::unordered_map<wcstring, bool>;
static bool fs_is_case_insensitive(const wcstring &path, int fd,
case_sensitivity_cache_t &case_sensitivity_cache) {
bool result = false;
#ifdef _PC_CASE_SENSITIVE
// Try the cache first.
case_sensitivity_cache_t::iterator cache = case_sensitivity_cache.find(path);
if (cache != case_sensitivity_cache.end()) {
/* Use the cached value */
result = cache->second;
} else {
// Ask the system. A -1 value means error (so assume case sensitive), a 1 value means case
// sensitive, and a 0 value means case insensitive.
long ret = fpathconf(fd, _PC_CASE_SENSITIVE);
result = (ret == 0);
case_sensitivity_cache[path] = result;
}
#else
// Silence lint tools about the unused parameters.
UNUSED(path);
UNUSED(fd);
UNUSED(case_sensitivity_cache);
#endif
return result;
}
/// Tests whether the specified string cpath is the prefix of anything we could cd to. directories
/// is a list of possible parent directories (typically either the working directory, or the
/// cdpath). This does I/O!
///
/// Hack: if out_suggested_cdpath is not NULL, it returns the autosuggestion for cd. This descends
/// the deepest unique directory hierarchy.
///
/// We expect the path to already be unescaped.
bool is_potential_path(const wcstring &potential_path_fragment, const wcstring_list_t &directories,
const operation_context_t &ctx, path_flags_t flags) {
ASSERT_IS_BACKGROUND_THREAD();
if (ctx.check_cancel()) return false;
const bool require_dir = static_cast<bool>(flags & PATH_REQUIRE_DIR);
wcstring clean_potential_path_fragment;
bool has_magic = false;
wcstring path_with_magic(potential_path_fragment);
if (flags & PATH_EXPAND_TILDE) expand_tilde(path_with_magic, ctx.vars);
for (auto c : path_with_magic) {
switch (c) {
case PROCESS_EXPAND_SELF:
case VARIABLE_EXPAND:
case VARIABLE_EXPAND_SINGLE:
case BRACE_BEGIN:
case BRACE_END:
case BRACE_SEP:
case ANY_CHAR:
case ANY_STRING:
case ANY_STRING_RECURSIVE: {
has_magic = true;
break;
}
case INTERNAL_SEPARATOR: {
break;
}
default: {
clean_potential_path_fragment.push_back(c);
break;
}
}
}
if (has_magic || clean_potential_path_fragment.empty()) {
return false;
}
// Don't test the same path multiple times, which can happen if the path is absolute and the
// CDPATH contains multiple entries.
std::unordered_set<wcstring> checked_paths;
// Keep a cache of which paths / filesystems are case sensitive.
case_sensitivity_cache_t case_sensitivity_cache;
for (const wcstring &wd : directories) {
if (ctx.check_cancel()) return false;
wcstring abs_path = path_apply_working_directory(clean_potential_path_fragment, wd);
if (flags & PATH_FOR_CD) {
abs_path = normalize_path(abs_path);
}
// Skip this if it's empty or we've already checked it.
if (abs_path.empty() || checked_paths.count(abs_path)) continue;
checked_paths.insert(abs_path);
// If we end with a slash, then it must be a directory.
bool must_be_full_dir = abs_path.at(abs_path.size() - 1) == L'/';
if (must_be_full_dir) {
struct stat buf;
if (0 == wstat(abs_path, &buf) && S_ISDIR(buf.st_mode)) {
return true;
}
} else {
// We do not end with a slash; it does not have to be a directory.
DIR *dir = nullptr;
const wcstring dir_name = wdirname(abs_path);
const wcstring filename_fragment = wbasename(abs_path);
if (dir_name == L"/" && filename_fragment == L"/") {
// cd ///.... No autosuggestion.
return true;
} else if ((dir = wopendir(dir_name))) {
cleanup_t cleanup_dir([&] { closedir(dir); });
// Check if we're case insensitive.
const bool do_case_insensitive =
fs_is_case_insensitive(dir_name, dirfd(dir), case_sensitivity_cache);
// We opened the dir_name; look for a string where the base name prefixes it Don't
// ask for the is_dir value unless we care, because it can cause extra filesystem
// access.
wcstring ent;
bool is_dir = false;
while (wreaddir_resolving(dir, dir_name, ent, require_dir ? &is_dir : nullptr)) {
if (ctx.check_cancel()) return false;
// Maybe skip directories.
if (require_dir && !is_dir) {
continue;
}
if (string_prefixes_string(filename_fragment, ent) ||
(do_case_insensitive &&
string_prefixes_string_case_insensitive(filename_fragment, ent))) {
return true;
}
}
}
}
}
return false;
}
// Given a string, return whether it prefixes a path that we could cd into. Return that path in
// out_path. Expects path to be unescaped.
static bool is_potential_cd_path(const wcstring &path, const wcstring &working_directory,
const operation_context_t &ctx, path_flags_t flags) {
wcstring_list_t directories;
if (string_prefixes_string(L"./", path)) {
// Ignore the CDPATH in this case; just use the working directory.
directories.push_back(working_directory);
} else {
// Get the CDPATH.
auto cdpath = ctx.vars.get(L"CDPATH");
wcstring_list_t pathsv =
cdpath.missing_or_empty() ? wcstring_list_t{L"."} : cdpath->as_list();
for (auto next_path : pathsv) {
if (next_path.empty()) next_path = L".";
// Ensure that we use the working directory for relative cdpaths like ".".
directories.push_back(path_apply_working_directory(next_path, working_directory));
}
}
// Call is_potential_path with all of these directories.
return is_potential_path(path, directories, ctx, flags | PATH_REQUIRE_DIR | PATH_FOR_CD);
}
// Given a plain statement node in a parse tree, get the command and return it, expanded
// appropriately for commands. If we succeed, return true.
static bool statement_get_expanded_command(const wcstring &src,
const ast::decorated_statement_t &stmt,
const operation_context_t &ctx, wcstring *out_cmd) {
// Get the command. Try expanding it. If we cannot, it's an error.
maybe_t<wcstring> cmd = stmt.command.source(src);
if (!cmd) return false;
expand_result_t err = expand_to_command_and_args(*cmd, ctx, out_cmd, nullptr);
return err == expand_result_t::ok;
}
rgb_color_t highlight_color_resolver_t::resolve_spec_uncached(const highlight_spec_t &highlight,
bool is_background,
const environment_t &vars) const {
rgb_color_t result = rgb_color_t::normal();
highlight_role_t role = is_background ? highlight.background : highlight.foreground;
auto var = vars.get(get_highlight_var_name(role));
if (!var) var = vars.get(get_highlight_var_name(get_fallback(role)));
if (!var) var = vars.get(get_highlight_var_name(highlight_role_t::normal));
if (var) result = parse_color(*var, is_background);
// Handle modifiers.
if (!is_background && highlight.valid_path) {
auto var2 = vars.get(L"fish_color_valid_path");
if (var2) {
rgb_color_t result2 = parse_color(*var2, is_background);
if (result.is_normal())
result = result2;
else {
if (result2.is_bold()) result.set_bold(true);
if (result2.is_underline()) result.set_underline(true);
if (result2.is_italics()) result.set_italics(true);
if (result2.is_dim()) result.set_dim(true);
if (result2.is_reverse()) result.set_reverse(true);
}
}
}
if (!is_background && highlight.force_underline) {
result.set_underline(true);
}
return result;
}
rgb_color_t highlight_color_resolver_t::resolve_spec(const highlight_spec_t &highlight,
bool is_background,
const environment_t &vars) {
auto &cache = is_background ? bg_cache_ : fg_cache_;
auto p = cache.insert(std::make_pair(highlight, rgb_color_t{}));
auto iter = p.first;
bool did_insert = p.second;
if (did_insert) {
// Insertion happened, meaning the cache needs to be populated.
iter->second = resolve_spec_uncached(highlight, is_background, vars);
}
return iter->second;
}
static bool command_is_valid(const wcstring &cmd, enum statement_decoration_t decoration,
const wcstring &working_directory, const environment_t &vars);
static bool has_expand_reserved(const wcstring &str) {
bool result = false;
for (auto wc : str) {
if (wc >= EXPAND_RESERVED_BASE && wc <= EXPAND_RESERVED_END) {
result = true;
break;
}
}
return result;
}
// Parse a command line. Return by reference the first command, and the first argument to that
// command (as a string), if any. This is used to validate autosuggestions.
static void autosuggest_parse_command(const wcstring &buff, const operation_context_t &ctx,
wcstring *out_expanded_command, wcstring *out_arg) {
auto ast = ast::ast_t::parse(
buff, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens);
// Find the first statement.
const ast::decorated_statement_t *first_statement = nullptr;
if (const ast::job_conjunction_t *jc = ast.top()->as<ast::job_list_t>()->at(0)) {
first_statement = jc->job.statement.contents->try_as<ast::decorated_statement_t>();
}
if (first_statement &&
statement_get_expanded_command(buff, *first_statement, ctx, out_expanded_command)) {
// Check if the first argument or redirection is, in fact, an argument.
if (const auto *arg_or_redir = first_statement->args_or_redirs.at(0)) {
if (arg_or_redir && arg_or_redir->is_argument()) {
*out_arg = arg_or_redir->argument().source(buff);
}
}
}
}
bool autosuggest_validate_from_history(const history_item_t &item,
const wcstring &working_directory,
const operation_context_t &ctx) {
ASSERT_IS_BACKGROUND_THREAD();
// Parse the string.
wcstring parsed_command;
wcstring cd_dir;
autosuggest_parse_command(item.str(), ctx, &parsed_command, &cd_dir);
// This is for autosuggestions which are not decorated commands, e.g. function declarations.
if (parsed_command.empty()) {
return true;
}
// We handle cd specially.
if (parsed_command == L"cd" && !cd_dir.empty()) {
if (expand_one(cd_dir, expand_flag::skip_cmdsubst, ctx)) {
if (string_prefixes_string(cd_dir, L"--help") ||
string_prefixes_string(cd_dir, L"-h")) {
// cd --help is always valid.
return true;
} else {
// Check the directory target, respecting CDPATH.
// Permit the autosuggestion if the path is valid and not our directory.
auto path = path_get_cdpath(cd_dir, working_directory, ctx.vars);
return path && !paths_are_same_file(working_directory, *path);
}
}
}
// Not handled specially. Is the command valid?
bool cmd_ok = builtin_exists(parsed_command) || function_exists_no_autoload(parsed_command) ||
path_get_path(parsed_command, nullptr, ctx.vars);
if (!cmd_ok) {
return false;
}
// Did the historical command have arguments that look like paths, which aren't paths now?
if (!all_paths_are_valid(item.get_required_paths(), ctx)) {
return false;
}
return true;
}
// Highlights the variable starting with 'in', setting colors within the 'colors' array. Returns the
// number of characters consumed.
static size_t color_variable(const wchar_t *in, size_t in_len,
std::vector<highlight_spec_t>::iterator colors) {
assert(in_len > 0);
assert(in[0] == L'$');
// Handle an initial run of $s.
size_t idx = 0;
size_t dollar_count = 0;
while (in[idx] == '$') {
// Our color depends on the next char.
wchar_t next = in[idx + 1];
if (next == L'$' || valid_var_name_char(next)) {
colors[idx] = highlight_role_t::operat;
} else {
colors[idx] = highlight_role_t::error;
}
idx++;
dollar_count++;
}
// Handle a sequence of variable characters.
while (valid_var_name_char(in[idx])) {
colors[idx++] = highlight_role_t::operat;
}
// Handle a slice, up to dollar_count of them. Note that we currently don't do any validation of
// the slice's contents, e.g. $foo[blah] will not show an error even though it's invalid.
for (size_t slice_count = 0; slice_count < dollar_count && in[idx] == L'['; slice_count++) {
wchar_t *slice_begin = nullptr, *slice_end = nullptr;
int located = parse_util_locate_slice(in + idx, &slice_begin, &slice_end, false);
if (located == 1) {
size_t slice_begin_idx = slice_begin - in, slice_end_idx = slice_end - in;
assert(slice_end_idx > slice_begin_idx);
colors[slice_begin_idx] = highlight_role_t::operat;
colors[slice_end_idx] = highlight_role_t::operat;
idx = slice_end_idx + 1;
} else if (located == 0) {
// not a slice
break;
} else {
assert(located < 0);
// Syntax error. Normally the entire token is colored red for us, but inside a
// double-quoted string that doesn't happen. As such, color the variable + the slice
// start red. Coloring any more than that looks bad, unless we're willing to try and
// detect where the double-quoted string ends, and I'd rather not do that.
std::fill(colors, colors + idx + 1, highlight_role_t::error);
break;
}
}
return idx;
}
/// This function is a disaster badly in need of refactoring. It colors an argument or command,
/// without regard to command substitutions.
static void color_string_internal(const wcstring &buffstr, highlight_spec_t base_color,
std::vector<highlight_spec_t>::iterator colors) {
// Clarify what we expect.
assert((base_color == highlight_role_t::param || base_color == highlight_role_t::command) &&
"Unexpected base color");
const size_t buff_len = buffstr.size();
std::fill(colors, colors + buff_len, base_color);
// Hacky support for %self which must be an unquoted literal argument.
if (buffstr == PROCESS_EXPAND_SELF_STR) {
std::fill_n(colors, std::wcslen(PROCESS_EXPAND_SELF_STR), highlight_role_t::operat);
return;
}
enum { e_unquoted, e_single_quoted, e_double_quoted } mode = e_unquoted;
maybe_t<size_t> unclosed_quote_offset;
int bracket_count = 0;
for (size_t in_pos = 0; in_pos < buff_len; in_pos++) {
const wchar_t c = buffstr.at(in_pos);
switch (mode) {
case e_unquoted: {
if (c == L'\\') {
auto fill_color = highlight_role_t::escape; // may be set to highlight_error
const size_t backslash_pos = in_pos;
size_t fill_end = backslash_pos;
// Move to the escaped character.
in_pos++;
const wchar_t escaped_char = (in_pos < buff_len ? buffstr.at(in_pos) : L'\0');
if (escaped_char == L'\0') {
fill_end = in_pos;
fill_color = highlight_role_t::error;
} else if (std::wcschr(L"~%", escaped_char)) {
if (in_pos == 1) {
fill_end = in_pos + 1;
}
} else if (escaped_char == L',') {
if (bracket_count) {
fill_end = in_pos + 1;
}
} else if (std::wcschr(L"abefnrtv*?$(){}[]'\"<>^ \\#;|&", escaped_char)) {
fill_end = in_pos + 1;
} else if (std::wcschr(L"c", escaped_char)) {
// Like \ci. So highlight three characters.
fill_end = in_pos + 1;
} else if (std::wcschr(L"uUxX01234567", escaped_char)) {
long long res = 0;
int chars = 2;
int base = 16;
wchar_t max_val = ASCII_MAX;
switch (escaped_char) {
case L'u': {
chars = 4;
max_val = UCS2_MAX;
in_pos++;
break;
}
case L'U': {
chars = 8;
max_val = WCHAR_MAX;
in_pos++;
break;
}
case L'x': {
in_pos++;
break;
}
case L'X': {
max_val = BYTE_MAX;
in_pos++;
break;
}
default: {
// a digit like \12
base = 8;
chars = 3;
break;
}
}
// Consume
for (int i = 0; i < chars && in_pos < buff_len; i++) {
long d = convert_digit(buffstr.at(in_pos), base);
if (d < 0) break;
res = (res * base) + d;
in_pos++;
}
// in_pos is now at the first character that could not be converted (or
// buff_len).
assert(in_pos >= backslash_pos && in_pos <= buff_len);
fill_end = in_pos;
// It's an error if we exceeded the max value.
if (res > max_val) fill_color = highlight_role_t::error;
// Subtract one from in_pos, so that the increment in the loop will move to
// the next character.
in_pos--;
}
assert(fill_end >= backslash_pos);
std::fill(colors + backslash_pos, colors + fill_end, fill_color);
} else {
// Not a backslash.
switch (c) {
case L'~': {
if (in_pos == 0) {
colors[in_pos] = highlight_role_t::operat;
}
break;
}
case L'$': {
assert(in_pos < buff_len);
in_pos += color_variable(buffstr.c_str() + in_pos, buff_len - in_pos,
colors + in_pos);
// Subtract one to account for the upcoming loop increment.
in_pos -= 1;
break;
}
case L'?': {
if (!feature_test(features_t::qmark_noglob)) {
colors[in_pos] = highlight_role_t::operat;
}
break;
}
case L'*':
case L'(':
case L')': {
colors[in_pos] = highlight_role_t::operat;
break;
}
case L'{': {
colors[in_pos] = highlight_role_t::operat;
bracket_count++;
break;
}
case L'}': {
colors[in_pos] = highlight_role_t::operat;
bracket_count--;
break;
}
case L',': {
if (bracket_count > 0) {
colors[in_pos] = highlight_role_t::operat;
}
break;
}
case L'\'': {
colors[in_pos] = highlight_role_t::quote;
unclosed_quote_offset = in_pos;
mode = e_single_quoted;
break;
}
case L'\"': {
colors[in_pos] = highlight_role_t::quote;
unclosed_quote_offset = in_pos;
mode = e_double_quoted;
break;
}
default: {
break; // we ignore all other characters
}
}
}
break;
}
// Mode 1 means single quoted string, i.e 'foo'.
case e_single_quoted: {
colors[in_pos] = highlight_role_t::quote;
if (c == L'\\') {
// backslash
if (in_pos + 1 < buff_len) {
const wchar_t escaped_char = buffstr.at(in_pos + 1);
if (escaped_char == L'\\' || escaped_char == L'\'') {
colors[in_pos] = highlight_role_t::escape; // backslash
colors[in_pos + 1] = highlight_role_t::escape; // escaped char
in_pos += 1; // skip over backslash
}
}
} else if (c == L'\'') {
unclosed_quote_offset = none();
mode = e_unquoted;
}
break;
}
// Mode 2 means double quoted string, i.e. "foo".
case e_double_quoted: {
// Slices are colored in advance, past `in_pos`, and we don't want to overwrite
// that.
if (colors[in_pos] == base_color) {
colors[in_pos] = highlight_role_t::quote;
}
switch (c) {
case L'"': {
unclosed_quote_offset = none();
mode = e_unquoted;
break;
}
case L'\\': {
// Backslash
if (in_pos + 1 < buff_len) {
const wchar_t escaped_char = buffstr.at(in_pos + 1);
if (std::wcschr(L"\\\"\n$", escaped_char)) {
colors[in_pos] = highlight_role_t::escape; // backslash
colors[in_pos + 1] = highlight_role_t::escape; // escaped char
in_pos += 1; // skip over backslash
}
}
break;
}
case L'$': {
in_pos += color_variable(buffstr.c_str() + in_pos, buff_len - in_pos,
colors + in_pos);
// Subtract one to account for the upcoming increment in the loop.
in_pos -= 1;
break;
}
default: {
break; // we ignore all other characters
}
}
break;
}
}
}
// Error on unclosed quotes.
if (unclosed_quote_offset) {
colors[*unclosed_quote_offset] = highlight_role_t::error;
}
}
/// Syntax highlighter helper.
class highlighter_t {
// The string we're highlighting. Note this is a reference memmber variable (to avoid copying)!
// We must not outlive this!
const wcstring &buff;
// The operation context. Again, a reference member variable!
const operation_context_t &ctx;
// Whether it's OK to do I/O.
const bool io_ok;
// Working directory.
const wcstring working_directory;
// The ast we produced.
ast::ast_t ast;
// The resulting colors.
using color_array_t = std::vector<highlight_spec_t>;
color_array_t color_array;
// A stack of variables that the current commandline probably defines. We mark redirections
// as valid if they use one of these variables, to avoid marking valid targets as error.
std::vector<wcstring> pending_variables;
// Flags we use for AST parsing.
static constexpr parse_tree_flags_t ast_flags =
parse_flag_continue_after_error | parse_flag_include_comments |
parse_flag_accept_incomplete_tokens | parse_flag_leave_unterminated |
parse_flag_show_extra_semis;
// Color a command.
void color_command(const ast::string_t &node);
// Color a node as if it were an argument.
void color_as_argument(const ast::node_t &node);
// Colors the source range of a node with a given color.
void color_node(const ast::node_t &node, highlight_spec_t color);
// Colors a range with a given color.
void color_range(source_range_t range, highlight_spec_t color);
// return whether a plain statement is 'cd'.
bool is_cd(const ast::decorated_statement_t &stmt) const;
/// \return a substring of our buffer.
wcstring get_source(source_range_t r) const;
public:
// Visit the children of a node.
void visit_children(const ast::node_t &node) {
ast::node_visitor(*this).accept_children_of(&node);
}
// AST visitor implementations.
void visit(const ast::keyword_base_t &kw);
void visit(const ast::token_base_t &tok);
void visit(const ast::redirection_t &redir);
void visit(const ast::variable_assignment_t &varas);
void visit(const ast::semi_nl_t &semi_nl);
void visit(const ast::decorated_statement_t &stmt);
void visit(const ast::block_statement_t &header);
// Visit an argument, perhaps knowing that our command is cd.
void visit(const ast::argument_t &arg, bool cmd_is_cd = false);
// Default implementation is to just visit children.
void visit(const ast::node_t &node) { visit_children(node); }
// Constructor
highlighter_t(const wcstring &str, const operation_context_t &ctx, wcstring wd, bool can_do_io)
: buff(str),
ctx(ctx),
io_ok(can_do_io),
working_directory(std::move(wd)),
ast(ast::ast_t::parse(buff, ast_flags)) {}
// Perform highlighting, returning an array of colors.
color_array_t highlight();
};
wcstring highlighter_t::get_source(source_range_t r) const {
assert(r.start + r.length >= r.start && "Overflow");
assert(r.start + r.length <= this->buff.size() && "Out of range");
return this->buff.substr(r.start, r.length);
}
void highlighter_t::color_node(const ast::node_t &node, highlight_spec_t color) {
color_range(node.source_range(), color);
}
void highlighter_t::color_range(source_range_t range, highlight_spec_t color) {
assert(range.start + range.length <= this->color_array.size() && "Range out of bounds");
std::fill_n(this->color_array.begin() + range.start, range.length, color);
}
void highlighter_t::color_command(const ast::string_t &node) {
source_range_t source_range = node.source_range();
const wcstring cmd_str = get_source(source_range);
// Get an iterator to the colors associated with the argument.
const size_t arg_start = source_range.start;
const color_array_t::iterator colors = color_array.begin() + arg_start;
color_string_internal(cmd_str, highlight_role_t::command, colors);
}
// node does not necessarily have type symbol_argument here.
void highlighter_t::color_as_argument(const ast::node_t &node) {
auto source_range = node.source_range();
const wcstring arg_str = get_source(source_range);
// Get an iterator to the colors associated with the argument.
const size_t arg_start = source_range.start;
const color_array_t::iterator arg_colors = color_array.begin() + arg_start;
// Color this argument without concern for command substitutions.
color_string_internal(arg_str, highlight_role_t::param, arg_colors);
// Now do command substitutions.
size_t cmdsub_cursor = 0, cmdsub_start = 0, cmdsub_end = 0;
wcstring cmdsub_contents;
while (parse_util_locate_cmdsubst_range(arg_str, &cmdsub_cursor, &cmdsub_contents,
&cmdsub_start, &cmdsub_end,
true /* accept incomplete */) > 0) {
// The cmdsub_start is the open paren. cmdsub_end is either the close paren or the end of
// the string. cmdsub_contents extends from one past cmdsub_start to cmdsub_end.
assert(cmdsub_end > cmdsub_start);
assert(cmdsub_end - cmdsub_start - 1 == cmdsub_contents.size());
// Found a command substitution. Compute the position of the start and end of the cmdsub
// contents, within our overall src.
const size_t arg_subcmd_start = arg_start + cmdsub_start,
arg_subcmd_end = arg_start + cmdsub_end;
// Highlight the parens. The open paren must exist; the closed paren may not if it was
// incomplete.
assert(cmdsub_start < arg_str.size());
this->color_array.at(arg_subcmd_start) = highlight_role_t::operat;
if (arg_subcmd_end < this->buff.size())
this->color_array.at(arg_subcmd_end) = highlight_role_t::operat;
// Highlight it recursively.
highlighter_t cmdsub_highlighter(cmdsub_contents, this->ctx, this->working_directory,
this->io_ok);
const color_array_t &subcolors = cmdsub_highlighter.highlight();
// Copy out the subcolors back into our array.
assert(subcolors.size() == cmdsub_contents.size());
std::copy(subcolors.begin(), subcolors.end(),
this->color_array.begin() + arg_subcmd_start + 1);
}
}
/// Indicates whether the source range of the given node forms a valid path in the given
/// working_directory.
static bool range_is_potential_path(const wcstring &src, const source_range_t &range,
const operation_context_t &ctx,
const wcstring &working_directory) {
// Skip strings exceeding PATH_MAX. See #7837.
// Note some paths may exceed PATH_MAX, but this is just for highlighting.
if (range.length > PATH_MAX) {
return false;
}
// Get the node source, unescape it, and then pass it to is_potential_path along with the
// working directory (as a one element list).
bool result = false;
wcstring token = src.substr(range.start, range.length);
if (unescape_string_in_place(&token, UNESCAPE_SPECIAL)) {
// Big hack: is_potential_path expects a tilde, but unescape_string gives us HOME_DIRECTORY.
// Put it back.
if (!token.empty() && token.at(0) == HOME_DIRECTORY) token.at(0) = L'~';
const wcstring_list_t working_directory_list(1, working_directory);
result = is_potential_path(token, working_directory_list, ctx, PATH_EXPAND_TILDE);
}
return result;
}
bool highlighter_t::is_cd(const ast::decorated_statement_t &stmt) const {
wcstring cmd_str;
if (this->io_ok && statement_get_expanded_command(this->buff, stmt, ctx, &cmd_str)) {
return cmd_str == L"cd";
}
return false;
}
void highlighter_t::visit(const ast::keyword_base_t &kw) {
highlight_role_t role = highlight_role_t::normal;
switch (kw.kw) {
case parse_keyword_t::kw_begin:
case parse_keyword_t::kw_builtin:
case parse_keyword_t::kw_case:
case parse_keyword_t::kw_command:
case parse_keyword_t::kw_else:
case parse_keyword_t::kw_end:
case parse_keyword_t::kw_exec:
case parse_keyword_t::kw_for:
case parse_keyword_t::kw_function:
case parse_keyword_t::kw_if:
case parse_keyword_t::kw_in:
case parse_keyword_t::kw_switch:
case parse_keyword_t::kw_while:
role = highlight_role_t::keyword;
break;
case parse_keyword_t::kw_and:
case parse_keyword_t::kw_or:
case parse_keyword_t::kw_not:
case parse_keyword_t::kw_exclam:
case parse_keyword_t::kw_time:
role = highlight_role_t::operat;
break;
case parse_keyword_t::none:
break;
}
color_node(kw, role);
}
void highlighter_t::visit(const ast::token_base_t &tok) {
maybe_t<highlight_role_t> role = highlight_role_t::normal;
switch (tok.type) {
case parse_token_type_t::end:
case parse_token_type_t::pipe:
case parse_token_type_t::background:
role = highlight_role_t::statement_terminator;
break;
case parse_token_type_t::andand:
case parse_token_type_t::oror:
role = highlight_role_t::operat;
break;
case parse_token_type_t::string:
// Assume all strings are params. This handles e.g. the variables a for header or
// function header. Other strings (like arguments to commands) need more complex
// handling, which occurs in their respective overrides of visit().
role = highlight_role_t::param;
default:
break;
}
if (role) color_node(tok, *role);
}
void highlighter_t::visit(const ast::semi_nl_t &semi_nl) {
color_node(semi_nl, highlight_role_t::statement_terminator);
}
void highlighter_t::visit(const ast::argument_t &arg, bool cmd_is_cd) {
color_as_argument(arg);
if (cmd_is_cd && io_ok) {
// Mark this as an error if it's not 'help' and not a valid cd path.
wcstring param = arg.source(this->buff);
if (expand_one(param, expand_flag::skip_cmdsubst, ctx)) {
bool is_help =
string_prefixes_string(param, L"--help") || string_prefixes_string(param, L"-h");
if (!is_help && this->io_ok &&
!is_potential_cd_path(param, working_directory, ctx, PATH_EXPAND_TILDE)) {
this->color_node(arg, highlight_role_t::error);
}
}
}
}
void highlighter_t::visit(const ast::variable_assignment_t &varas) {
color_as_argument(varas);
// Highlight the '=' in variable assignments as an operator.
if (auto where = variable_assignment_equals_pos(varas.source(this->buff))) {
size_t equals_loc = varas.source_range().start + *where;
this->color_array.at(equals_loc) = highlight_role_t::operat;
auto var_name = varas.source(this->buff).substr(0, *where);
this->pending_variables.push_back(std::move(var_name));
}
}
void highlighter_t::visit(const ast::decorated_statement_t &stmt) {
// Color any decoration.
if (stmt.opt_decoration) this->visit(*stmt.opt_decoration);
// Color the command's source code.
// If we get no source back, there's nothing to color.
maybe_t<wcstring> cmd = stmt.command.try_source(this->buff);
if (!cmd.has_value()) return;
wcstring expanded_cmd;
bool is_valid_cmd = false;
if (!this->io_ok) {
// We cannot check if the command is invalid, so just assume it's valid.
is_valid_cmd = true;
} else if (variable_assignment_equals_pos(*cmd)) {
is_valid_cmd = true;
} else {
// Check to see if the command is valid.
// Try expanding it. If we cannot, it's an error.
bool expanded = statement_get_expanded_command(buff, stmt, ctx, &expanded_cmd);
if (expanded && !has_expand_reserved(expanded_cmd)) {
is_valid_cmd =
command_is_valid(expanded_cmd, stmt.decoration(), working_directory, ctx.vars);
}
}
// Color our statement.
if (is_valid_cmd) {
this->color_command(stmt.command);
} else {
this->color_node(stmt.command, highlight_role_t::error);
}
// Color arguments and redirections.
// Except if our command is 'cd' we have special logic for how arguments are colored.
bool is_cd = (expanded_cmd == L"cd");
bool is_set = (expanded_cmd == L"set");
for (const ast::argument_or_redirection_t &v : stmt.args_or_redirs) {
if (v.is_argument()) {
if (is_set) {
auto arg = v.argument().source(this->buff);
if (valid_var_name(arg)) {
this->pending_variables.push_back(std::move(arg));
is_set = false;
}
}
this->visit(v.argument(), is_cd);
} else {
this->visit(v.redirection());
}
}
}
void highlighter_t::visit(const ast::block_statement_t &block) {
this->visit(*block.header.contents.get());
this->visit(block.args_or_redirs);
const ast::node_t &bh = *block.header.contents;
size_t pending_variables_count = this->pending_variables.size();
if (const auto *fh = bh.try_as<ast::for_header_t>()) {
auto var_name = fh->var_name.source(this->buff);
pending_variables.push_back(std::move(var_name));
}
this->visit(block.jobs);
this->visit(block.end);
pending_variables.resize(pending_variables_count);
}
/// \return whether a string contains a command substitution.
static bool has_cmdsub(const wcstring &src) {
size_t cursor = 0;
size_t start = 0;
size_t end = 0;
return parse_util_locate_cmdsubst_range(src, &cursor, nullptr, &start, &end, true) != 0;
}
static bool contains_pending_variable(const std::vector<wcstring> &pending_variables,
const wcstring &haystack) {
for (const auto &var_name : pending_variables) {
size_t pos = -1;
while ((pos = haystack.find(var_name, pos + 1)) != wcstring::npos) {
if (pos == 0 || haystack.at(pos - 1) != L'$') continue;
size_t end = pos + var_name.size();
if (end < haystack.size() && valid_var_name_char(haystack.at(end))) continue;
return true;
}
}
return false;
}
void highlighter_t::visit(const ast::redirection_t &redir) {
maybe_t<pipe_or_redir_t> oper =
pipe_or_redir_t::from_string(redir.oper.source(this->buff)); // like 2>
wcstring target = redir.target.source(this->buff); // like &1 or file path
assert(oper.has_value() &&
"Should have successfully parsed a pipe_or_redir_t since it was in our ast");
// Color the > part.
// It may have parsed successfully yet still be invalid (e.g. 9999999999999>&1)
// If so, color the whole thing invalid and stop.
if (!oper->is_valid()) {
this->color_node(redir, highlight_role_t::error);
return;
}
// Color the operator part like 2>.
this->color_node(redir.oper, highlight_role_t::redirection);
// Color the target part.
// Check if the argument contains a command substitution. If so, highlight it as a param
// even though it's a command redirection, and don't try to do any other validation.
if (has_cmdsub(target)) {
this->color_as_argument(redir.target);
} else {
// No command substitution, so we can highlight the target file or fd. For example,
// disallow redirections into a non-existent directory.
bool target_is_valid = true;
if (!this->io_ok) {
// I/O is disallowed, so we don't have much hope of catching anything but gross
// errors. Assume it's valid.
target_is_valid = true;
} else if (contains_pending_variable(this->pending_variables, target)) {
target_is_valid = true;
} else if (!expand_one(target, expand_flag::skip_cmdsubst, ctx)) {
// Could not be expanded.
target_is_valid = false;
} else {
// Ok, we successfully expanded our target. Now verify that it works with this
// redirection. We will probably need it as a path (but not in the case of fd
// redirections). Note that the target is now unescaped.
const wcstring target_path =
path_apply_working_directory(target, this->working_directory);
switch (oper->mode) {
case redirection_mode_t::fd: {
if (target == L"-") {
target_is_valid = true;
} else {
int fd = fish_wcstoi(target.c_str());
target_is_valid = !errno && fd >= 0;
}
break;
}
case redirection_mode_t::input: {
// Input redirections must have a readable non-directory.
struct stat buf = {};
target_is_valid = !waccess(target_path, R_OK) && !wstat(target_path, &buf) &&
!S_ISDIR(buf.st_mode);
break;
}
case redirection_mode_t::overwrite:
case redirection_mode_t::append:
case redirection_mode_t::noclob: {
// Test whether the file exists, and whether it's writable (possibly after
// creating it). access() returns failure if the file does not exist.
bool file_exists = false, file_is_writable = false;
int err = 0;
struct stat buf = {};
if (wstat(target_path, &buf) < 0) {
err = errno;
}
if (string_suffixes_string(L"/", target)) {
// Redirections to things that are directories is definitely not
// allowed.
file_exists = false;
file_is_writable = false;
} else if (err == 0) {
// No err. We can write to it if it's not a directory and we have
// permission.
file_exists = true;
file_is_writable = !S_ISDIR(buf.st_mode) && !waccess(target_path, W_OK);
} else if (err == ENOENT) {
// File does not exist. Check if its parent directory is writable.
wcstring parent = wdirname(target_path);
// Ensure that the parent ends with the path separator. This will ensure
// that we get an error if the parent directory is not really a
// directory.
if (!string_suffixes_string(L"/", parent)) parent.push_back(L'/');
// Now the file is considered writable if the parent directory is
// writable.
file_exists = false;
file_is_writable = (0 == waccess(parent, W_OK));
} else {
// Other errors we treat as not writable. This includes things like
// ENOTDIR.
file_exists = false;
file_is_writable = false;
}
// NOCLOB means that we must not overwrite files that exist.
target_is_valid = file_is_writable &&
!(file_exists && oper->mode == redirection_mode_t::noclob);
break;
}
}
}
this->color_node(redir.target,
target_is_valid ? highlight_role_t::redirection : highlight_role_t::error);
}
}
/// Determine if a command is valid.
static bool command_is_valid(const wcstring &cmd, enum statement_decoration_t decoration,
const wcstring &working_directory, const environment_t &vars) {
// Determine which types we check, based on the decoration.
bool builtin_ok = true, function_ok = true, abbreviation_ok = true, command_ok = true,
implicit_cd_ok = true;
if (decoration == statement_decoration_t::command ||
decoration == statement_decoration_t::exec) {
builtin_ok = false;
function_ok = false;
abbreviation_ok = false;
command_ok = true;
implicit_cd_ok = false;
} else if (decoration == statement_decoration_t::builtin) {
builtin_ok = true;
function_ok = false;
abbreviation_ok = false;
command_ok = false;
implicit_cd_ok = false;
}
// Check them.
bool is_valid = false;
// Builtins
if (!is_valid && builtin_ok) is_valid = builtin_exists(cmd);
// Functions
if (!is_valid && function_ok) is_valid = function_exists_no_autoload(cmd);
// Abbreviations
if (!is_valid && abbreviation_ok) is_valid = expand_abbreviation(cmd, vars).has_value();
// Regular commands
if (!is_valid && command_ok) is_valid = path_get_path(cmd, nullptr, vars);
// Implicit cd
if (!is_valid && implicit_cd_ok) {
is_valid = path_as_implicit_cd(cmd, working_directory, vars).has_value();
}
// Return what we got.
return is_valid;
}
highlighter_t::color_array_t highlighter_t::highlight() {
// If we are doing I/O, we must be in a background thread.
if (io_ok) {
ASSERT_IS_BACKGROUND_THREAD();
}
this->color_array.resize(this->buff.size());
std::fill(this->color_array.begin(), this->color_array.end(), highlight_spec_t{});
this->visit_children(*ast.top());
if (ctx.check_cancel()) return std::move(color_array);
// Color every comment.
const auto &extras = ast.extras();
for (const source_range_t &r : extras.comments) {
this->color_range(r, highlight_role_t::comment);
}
// Color every extra semi.
for (const source_range_t &r : extras.semis) {
this->color_range(r, highlight_role_t::statement_terminator);
}
// Color every error range.
for (const source_range_t &r : extras.errors) {
this->color_range(r, highlight_role_t::error);
}
// Underline every valid path.
if (io_ok) {
for (const ast::node_t &node : ast) {
const ast::argument_t *arg = node.try_as<ast::argument_t>();
if (!arg || arg->unsourced) continue;
if (ctx.check_cancel()) break;
if (range_is_potential_path(buff, arg->range, ctx, working_directory)) {
// Don't color highlight_role_t::error because it looks dorky. For example,
// trying to cd into a non-directory would show an underline and also red.
for (size_t i = arg->range.start, end = arg->range.start + arg->range.length;
i < end; i++) {
if (this->color_array.at(i).foreground != highlight_role_t::error) {
this->color_array.at(i).valid_path = true;
}
}
}
}
}
return std::move(color_array);
}
std::string colorize(const wcstring &text, const std::vector<highlight_spec_t> &colors,
const environment_t &vars) {
assert(colors.size() == text.size());
highlight_color_resolver_t rv;
outputter_t outp;
highlight_spec_t last_color = highlight_role_t::normal;
for (size_t i = 0; i < text.size(); i++) {
highlight_spec_t color = colors.at(i);
if (color != last_color) {
outp.set_color(rv.resolve_spec(color, false, vars), rgb_color_t::normal());
last_color = color;
}
outp.writech(text.at(i));
}
outp.set_color(rgb_color_t::normal(), rgb_color_t::normal());
return outp.contents();
}
void highlight_shell(const wcstring &buff, std::vector<highlight_spec_t> &color,
const operation_context_t &ctx, bool io_ok) {
const wcstring working_directory = ctx.vars.get_pwd_slash();
highlighter_t highlighter(buff, ctx, working_directory, io_ok);
color = highlighter.highlight();
}
Reference in a new issue View git blame Copy permalink