Support for fuzzy completions

https://github.com/fish-shell/fish-shell/issues/568
https://github.com/fish-shell/fish-shell/issues/528

common.cpp

@@ -1764,6 +1764,120 @@ bool string_suffixes_string(const wchar_t *proposed_suffix, const wcstring &valu
     return suffix_size <= value.size() && value.compare(value.size() - suffix_size, suffix_size, proposed_suffix) == 0;
 }
 
+// Returns true if seq, represented as a subsequence, is contained within string
+static bool subsequence_in_string(const wcstring &seq, const wcstring &str)
+{
+    /* Impossible if seq is larger than string */
+    if (seq.size() > str.size())
+    {
+        return false;
+    }
+    
+    /* Empty strings are considered to be subsequences of everything */
+    if (seq.empty())
+    {
+        return true;
+    }
+    
+    size_t str_idx, seq_idx;
+    for (seq_idx = str_idx = 0; seq_idx < seq.size() && str_idx < str.size(); seq_idx++)
+    {
+        wchar_t c = seq.at(seq_idx);
+        size_t char_loc = str.find(c, str_idx);
+        if (char_loc == wcstring::npos)
+        {
+            /* Didn't find this character */
+            break;
+        }
+        else
+        {
+            /* We found it. Continue the search just after it. */
+            str_idx = char_loc + 1;
+        }
+    }
+    
+    /* We succeeded if we exhausted our sequence */
+    assert(seq_idx <= seq.size());
+    return seq_idx == seq.size();
+}
+
+string_fuzzy_match_t::string_fuzzy_match_t(enum fuzzy_match_type_t t, size_t distance_first, size_t distance_second) :
+        type(t),
+        match_distance_first(distance_first),
+        match_distance_second(distance_second)
+{
+}
+
+
+string_fuzzy_match_t string_fuzzy_match_string(const wcstring &string, const wcstring &match_against, fuzzy_match_type_t limit_type)
+{
+    // Distances are generally the amount of text not matched
+    string_fuzzy_match_t result(fuzzy_match_none, 0, 0);
+    size_t location;
+    if (limit_type >= fuzzy_match_exact && string == match_against)
+    {
+        result.type = fuzzy_match_exact;
+    }
+    else if (limit_type >= fuzzy_match_prefix && string_prefixes_string(string, match_against))
+    {
+        result.type = fuzzy_match_prefix;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+    }
+    else if (limit_type >= fuzzy_match_case_insensitive && wcscasecmp(string.c_str(), match_against.c_str()) == 0)
+    {
+        result.type = fuzzy_match_case_insensitive;
+    }
+    else if (limit_type >= fuzzy_match_prefix_case_insensitive && string_prefixes_string_case_insensitive(string, match_against))
+    {
+        result.type = fuzzy_match_prefix_case_insensitive;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+    }
+    else if (limit_type >= fuzzy_match_substring && (location = match_against.find(string)) != wcstring::npos)
+    {
+        // string is contained within match against
+        result.type = fuzzy_match_substring;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+        result.match_distance_second = location; //prefer earlier matches
+    }
+    else if (limit_type >= fuzzy_match_subsequence_insertions_only && subsequence_in_string(string, match_against))
+    {
+        result.type = fuzzy_match_subsequence_insertions_only;
+        assert(match_against.size() >= string.size());
+        result.match_distance_first = match_against.size() - string.size();
+        // it would be nice to prefer matches with greater matching runs here
+    }
+    return result;
+}
+
+template<typename T>
+static inline int compare_ints(T a, T b)
+{
+    if (a < b) return -1;
+    if (a == b) return 0;
+    return 1;
+}
+
+// Compare types; if the types match, compare distances
+int string_fuzzy_match_t::compare(const string_fuzzy_match_t &rhs) const
+{
+    if (this->type != rhs.type)
+    {
+        return compare_ints(this->type, rhs.type);
+    }
+    else if (this->match_distance_first != rhs.match_distance_first)
+    {
+        return compare_ints(this->match_distance_first, rhs.match_distance_first);
+    }
+    else if (this->match_distance_second != rhs.match_distance_second)
+    {
+        return compare_ints(this->match_distance_second, rhs.match_distance_second);
+    }
+    return 0; //equal
+}
+
 bool list_contains_string(const wcstring_list_t &list, const wcstring &str)
 {
     return std::find(list.begin(), list.end(), str) != list.end();

common.h

@@ -243,10 +243,81 @@ bool string_prefixes_string(const wchar_t *proposed_prefix, const wcstring &valu
 bool string_suffixes_string(const wcstring &proposed_suffix, const wcstring &value);
 bool string_suffixes_string(const wchar_t *proposed_suffix, const wcstring &value);
 
-
 /** Test if a string prefixes another without regard to case. Returns true if a is a prefix of b */
 bool string_prefixes_string_case_insensitive(const wcstring &proposed_prefix, const wcstring &value);
 
+enum fuzzy_match_type_t
+{
+    /* We match the string exactly: FOOBAR matches FOOBAR */
+    fuzzy_match_exact = 0,
+    
+    /* We match a prefix of the string: FO matches FOOBAR */
+    fuzzy_match_prefix,
+    
+    /* We match the string exactly, but in a case insensitive way: foobar matches FOOBAR */
+    fuzzy_match_case_insensitive,
+    
+    /* We match a prefix of the string, in a case insensitive way: foo matches FOOBAR */
+    fuzzy_match_prefix_case_insensitive,
+    
+    /* We match a substring of the string: OOBA matches FOOBAR */
+    fuzzy_match_substring,
+    
+    /* A subsequence match with insertions only: FBR matches FOOBAR */
+    fuzzy_match_subsequence_insertions_only,
+    
+    /* We don't match the string */
+    fuzzy_match_none
+};
+
+/* Indicates where a match type requires replacing the entire token */
+static inline bool match_type_requires_full_replacement(fuzzy_match_type_t t)
+{
+    switch (t)
+    {
+        case fuzzy_match_exact:
+        case fuzzy_match_prefix:
+            return false;
+        default:
+            return true;
+    }
+}
+
+/* Indicates where a match shares a prefix with the string it matches */
+static inline bool match_type_shares_prefix(fuzzy_match_type_t t)
+{
+    switch (t)
+    {
+        case fuzzy_match_exact:
+        case fuzzy_match_prefix:
+        case fuzzy_match_case_insensitive:
+        case fuzzy_match_prefix_case_insensitive:
+            return true;
+        default:
+            return false;
+    }
+}
+
+/** Test if string is a fuzzy match to another */
+struct string_fuzzy_match_t
+{
+    enum fuzzy_match_type_t type;
+    
+    /* Strength of the match. The value depends on the type. Lower is stronger. */
+    size_t match_distance_first;
+    size_t match_distance_second;
+    
+    /* Constructor */
+    string_fuzzy_match_t(enum fuzzy_match_type_t t, size_t distance_first = 0, size_t distance_second = 0);
+    
+    /* Return -1, 0, 1 if this match is (respectively) better than, equal to, or worse than rhs */
+    int compare(const string_fuzzy_match_t &rhs) const;
+};
+
+/* Compute a fuzzy match for a string. If maximum_match is not fuzzy_match_none, limit the type to matches at or below that type. */
+string_fuzzy_match_t string_fuzzy_match_string(const wcstring &string, const wcstring &match_against, fuzzy_match_type_t limit_type = fuzzy_match_none);
+
+
 /** Test if a list contains a string using a linear search. */
 bool list_contains_string(const wcstring_list_t &list, const wcstring &str);
 

complete.cpp

@@ -280,7 +280,11 @@ completion_t::~completion_t()
 }
 
 /* completion_t functions */
-completion_t::completion_t(const wcstring &comp, const wcstring &desc, int flags_val) : completion(comp), description(desc), flags(flags_val)
+completion_t::completion_t(const wcstring &comp, const wcstring &desc, string_fuzzy_match_t mat, int flags_val) :
+    completion(comp),
+    description(desc),
+    match(mat),
+    flags(flags_val)
 {
     if (flags & COMPLETE_AUTO_SPACE)
     {
@@ -292,7 +296,7 @@ completion_t::completion_t(const wcstring &comp, const wcstring &desc, int flags
 
 }
 
-completion_t::completion_t(const completion_t &him) : completion(him.completion), description(him.description), flags(him.flags)
+completion_t::completion_t(const completion_t &him) : completion(him.completion), description(him.description), match(him.match), flags(him.flags)
 {
 }
 
@@ -302,6 +306,7 @@ completion_t &completion_t::operator=(const completion_t &him)
     {
         this->completion = him.completion;
         this->description = him.description;
+        this->match = him.match;
         this->flags = him.flags;
     }
     return *this;
@@ -371,6 +376,12 @@ class completer_t
         return !!(flags & COMPLETION_REQUEST_FUZZY_MATCH);
     }
     
+    fuzzy_match_type_t max_fuzzy_match_type() const
+    {
+        /* If we are doing fuzzy matching, request all types; if not request only prefix matching */
+        return (flags & COMPLETION_REQUEST_FUZZY_MATCH) ? fuzzy_match_none : fuzzy_match_prefix_case_insensitive;
+    }
+
 
 public:
     completer_t(const wcstring &c, completion_request_flags_t f) :
@@ -428,6 +439,11 @@ public:
         expand_flags_t result = 0;
         if (this->type() == COMPLETE_AUTOSUGGEST)
             result |= EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_JOBS;
+        
+        /* Allow fuzzy matching */
+        if (this->fuzzy())
+            result |= EXPAND_FUZZY_MATCH;
+        
         return result;
     }
 
@@ -467,9 +483,9 @@ void completion_autoload_t::command_removed(const wcstring &cmd)
 
 
 /** Create a new completion entry */
-void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc, complete_flags_t flags)
+void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc, complete_flags_t flags, string_fuzzy_match_t match)
 {
-    completions.push_back(completion_t(comp, desc, flags));
+    completions.push_back(completion_t(comp, desc, match, flags));
 }
 
 /**
@@ -969,7 +985,7 @@ void completer_t::complete_strings(const wcstring &wc_escaped,
 
         if (next_str)
         {
-            wildcard_complete(next_str, wc, desc, desc_func, this->completions, flags);
+            wildcard_complete(next_str, wc, desc, desc_func, this->completions, this->expand_flags(), flags);
         }
     }
 
@@ -1192,10 +1208,6 @@ void completer_t::complete_cmd(const wcstring &str_cmd, bool use_function, bool
             }
         }
         
-        /*
-          These return the original strings - don't free them
-        */
-
         if (use_function)
         {
             //function_get_names( &possible_comp, cmd[0] == L'_' );
@@ -1555,7 +1567,7 @@ bool completer_t::complete_param(const wcstring &scmd_orig, const wcstring &spop
                             }
                             else
                             {
-                                flags = COMPLETE_REPLACES_TOKEN | COMPLETE_CASE_INSENSITIVE;
+                                flags = COMPLETE_REPLACES_TOKEN;
                             }
 
                             has_arg = ! o->comp.empty();
@@ -1645,57 +1657,50 @@ bool completer_t::complete_variable(const wcstring &str, size_t start_offset)
     const wchar_t * const whole_var = str.c_str();
     const wchar_t *var = &whole_var[start_offset];
     size_t varlen = wcslen(var);
-    int res = 0;
+    bool res = false;
     
     const wcstring_list_t names = complete_get_variable_names();
     for (size_t i=0; i<names.size(); i++)
     {
         const wcstring & env_name = names.at(i);
-        size_t namelen = env_name.size();
-        int match=0, match_no_case=0;
         
-        if (varlen > namelen)
+        string_fuzzy_match_t match = string_fuzzy_match_string(var, env_name, this->max_fuzzy_match_type());
+        if (match.type == fuzzy_match_none)
+        {
+            // No match
             continue;
-
-        match = string_prefixes_string(var, env_name);
-
-        if (!match)
-        {
-            match_no_case = (wcsncasecmp(var, env_name.c_str(), varlen) == 0);
         }
         
-        if (match || match_no_case)
+        wcstring comp;
+        int flags = 0;
+        
+        if (! match_type_requires_full_replacement(match.type))
         {
-            wcstring comp;
+            // Take only the suffix
-            int flags = 0;
+            comp.append(env_name.c_str() + varlen);
-
-            if (match)
-            {
-                comp.append(env_name.c_str() + varlen);
-            }
-            else
-            {
-                comp.append(whole_var, start_offset);
-                comp.append(env_name);
-                flags = COMPLETE_CASE_INSENSITIVE | COMPLETE_REPLACES_TOKEN | COMPLETE_DONT_ESCAPE;
-            }
-
-            wcstring desc;
-            if (this->wants_descriptions())
-            {
-                env_var_t value_unescaped = env_get_string(env_name);
-                if (value_unescaped.missing())
-                    continue;
-
-                wcstring value = expand_escape_variable(value_unescaped);
-                if (this->type() != COMPLETE_AUTOSUGGEST)
-                    desc = format_string(COMPLETE_VAR_DESC_VAL, value.c_str());
-            }
-
-            append_completion(this->completions,  comp.c_str(), desc.c_str(), flags);
-            res =1;
-
         }
+        else
+        {
+            comp.append(whole_var, start_offset);
+            comp.append(env_name);
+            flags = COMPLETE_REPLACES_TOKEN | COMPLETE_DONT_ESCAPE;
+        }
+        
+        wcstring desc;
+        if (this->wants_descriptions())
+        {
+            env_var_t value_unescaped = env_get_string(env_name);
+            if (value_unescaped.missing())
+                continue;
+            
+            wcstring value = expand_escape_variable(value_unescaped);
+            if (this->type() != COMPLETE_AUTOSUGGEST)
+                desc = format_string(COMPLETE_VAR_DESC_VAL, value.c_str());
+        }
+        
+        append_completion(this->completions,  comp.c_str(), desc.c_str(), flags, match);
+        
+        res = true;
     }
 
     return res;
@@ -1781,7 +1786,7 @@ bool completer_t::try_complete_user(const wcstring &str)
                         append_completion(this->completions,
                                           name,
                                           desc,
-                                          COMPLETE_CASE_INSENSITIVE | COMPLETE_REPLACES_TOKEN | COMPLETE_DONT_ESCAPE | COMPLETE_NO_SPACE);
+                                          COMPLETE_REPLACES_TOKEN | COMPLETE_DONT_ESCAPE | COMPLETE_NO_SPACE);
                         res=1;
                     }
                 }
@@ -1836,7 +1841,7 @@ void complete(const wcstring &cmd, std::vector<completion_t> &comps, completion_
     {
         pos = cursor_pos-(cmdsubst_begin-cmd_cstr);
 
-        wcstring buff = wcstring(cmdsubst_begin, cmdsubst_end-cmdsubst_begin);
+        const wcstring buff = wcstring(cmdsubst_begin, cmdsubst_end-cmdsubst_begin);
 
         int had_cmd=0;
         int end_loop=0;

complete.h

@@ -75,16 +75,11 @@ enum
     */
     COMPLETE_NO_SPACE = 1 << 0,
 
-    /** This completion is case insensitive. */
-    COMPLETE_CASE_INSENSITIVE = 1 << 1,
-
     /** This is not the suffix of a token, but replaces it entirely */
     COMPLETE_REPLACES_TOKEN = 1 << 2,
 
-    /**
+    /** This completion may or may not want a space at the end - guess by
-       This completion may or may not want a space at the end - guess by
+       checking the last character of the completion. */
-       checking the last character of the completion.
-    */
     COMPLETE_AUTO_SPACE = 1 << 3,
 
     /** This completion should be inserted as-is, without escaping. */
@@ -107,16 +102,15 @@ public:
     /* Destructor. Not inlining it saves code size. */
     ~completion_t();
 
-    /**
+    /** The completion string */
-       The completion string
-    */
     wcstring completion;
 
-    /**
+    /** The description for this completion */
-       The description for this completion
-    */
     wcstring description;
 
+    /** The type of fuzzy match */
+    string_fuzzy_match_t match;
+
     /**
        Flags determining the completion behaviour.
 
@@ -129,13 +123,8 @@ public:
     */
     int flags;
     
-    bool is_case_insensitive() const
-    {
-        return !!(flags & COMPLETE_CASE_INSENSITIVE);
-    }
-
     /* Construction. Note: defining these so that they are not inlined reduces the executable size. */
-    completion_t(const wcstring &comp, const wcstring &desc = L"", int flags_val = 0);
+    completion_t(const wcstring &comp, const wcstring &desc = L"", string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact), int flags_val = 0);
     completion_t(const completion_t &);
     completion_t &operator=(const completion_t &);
 
@@ -286,7 +275,7 @@ void complete_load(const wcstring &cmd, bool reload);
    \param flags completion flags
 
 */
-void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc = L"", int flags = 0);
+void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc = L"", int flags = 0, string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact));
 
 /* Function used for testing */
 void complete_set_variable_names(const wcstring_list_t *names);

expand.h

@@ -56,7 +56,10 @@ enum
     EXPAND_SKIP_JOBS = 1 << 8,
 
     /** Don't expand home directories */
-    EXPAND_SKIP_HOME_DIRECTORIES = 1 << 9
+    EXPAND_SKIP_HOME_DIRECTORIES = 1 << 9,
+    
+    /** Allow fuzzy matching */
+    EXPAND_FUZZY_MATCH = 1 << 10
 };
 typedef int expand_flags_t;
 

fish_tests.cpp

@@ -661,7 +661,21 @@ static void test_expand()
         err(L"Expansion not correctly handling literal path components in dotfiles");
     }
 
-    //system("rm -Rf /tmp/fish_expand_test");
+    system("rm -Rf /tmp/fish_expand_test");
+}
+
+static void test_fuzzy_match(void)
+{
+    say(L"Testing fuzzy string matching");
+
+    if (string_fuzzy_match_string(L"", L"").type != fuzzy_match_exact) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"alpha", L"alpha").type != fuzzy_match_exact) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"alp", L"alpha").type != fuzzy_match_prefix) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"ALPHA!", L"alPhA!").type != fuzzy_match_case_insensitive) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"alPh", L"ALPHA!").type != fuzzy_match_prefix_case_insensitive) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"LPH", L"ALPHA!").type != fuzzy_match_substring) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"AA", L"ALPHA!").type != fuzzy_match_subsequence_insertions_only) err(L"test_fuzzy_match failed on line %ld", __LINE__);
+    if (string_fuzzy_match_string(L"BB", L"ALPHA!").type != fuzzy_match_none) err(L"test_fuzzy_match failed on line %ld", __LINE__);
 }
 
 /** Test path functions */
@@ -975,6 +989,17 @@ static void test_complete(void)
     assert(completions.at(1).completion == L"oo2");
     assert(completions.at(2).completion == L"oo3");
     
+    completions.clear();
+    complete(L"$1", completions, COMPLETION_REQUEST_DEFAULT);
+    assert(completions.empty());
+    
+    completions.clear();
+    complete(L"$1", completions, COMPLETION_REQUEST_DEFAULT | COMPLETION_REQUEST_FUZZY_MATCH);
+    assert(completions.size() == 2);
+    assert(completions.at(0).completion == L"$Foo1");
+    assert(completions.at(1).completion == L"$Bar1");
+
+    
     complete_set_variable_names(NULL);
 }
 
@@ -1025,8 +1050,8 @@ static void test_completion_insertions()
     TEST_1_COMPLETION(L"'foo\\'^", L"bar", COMPLETE_NO_SPACE, false, L"'foo\\'bar^");
     TEST_1_COMPLETION(L"foo\\'^", L"bar", COMPLETE_NO_SPACE, false, L"foo\\'bar^");
 
-    TEST_1_COMPLETION(L"foo^", L"bar", COMPLETE_CASE_INSENSITIVE | COMPLETE_REPLACES_TOKEN, false, L"bar ^");
+    TEST_1_COMPLETION(L"foo^", L"bar", COMPLETE_REPLACES_TOKEN, false, L"bar ^");
-    TEST_1_COMPLETION(L"'foo^", L"bar", COMPLETE_CASE_INSENSITIVE | COMPLETE_REPLACES_TOKEN, false, L"bar ^");
+    TEST_1_COMPLETION(L"'foo^", L"bar", COMPLETE_REPLACES_TOKEN, false, L"bar ^");
 }
 
 static void perform_one_autosuggestion_test(const wcstring &command, const wcstring &wd, const wcstring &expected, long line)
@@ -1724,6 +1749,7 @@ int main(int argc, char **argv)
     test_parser();
     test_lru();
     test_expand();
+    test_fuzzy_match();
     test_test();
     test_path();
     test_word_motion();

reader.cpp

@@ -947,29 +947,6 @@ static int insert_char(wchar_t c)
 }
 
 
-/**
-   Calculate the length of the common prefix substring of two strings.
-*/
-static size_t comp_len(const wchar_t *a, const wchar_t *b)
-{
-    size_t i;
-    for (i=0; a[i] != L'\0' && b[i] != L'\0' && a[i]==b[i]; i++)
-        ;
-    return i;
-}
-
-/**
-   Calculate the case insensitive length of the common prefix substring of two strings.
-*/
-static size_t comp_ilen(const wchar_t *a, const wchar_t *b)
-{
-    size_t i;
-    for (i=0; a[i] != L'\0' && b[i] != L'\0' && towlower(a[i])==towlower(b[i]); i++)
-        ;
-    return i;
-}
-
-
 /**
    Insert the string in the given command line at the given cursor
    position. The function checks if the string is quoted or not and
@@ -1163,34 +1140,18 @@ static void run_pager(const wcstring &prefix, int is_quoted, const std::vector<c
 
     escaped_separator = escape(COMPLETE_SEP_STR, 1);
     
-    bool has_case_sensitive = false;
     for (size_t i=0; i< comp.size(); i++)
     {
-        const completion_t &el = comp.at(i);
-        if (!(el.flags & COMPLETE_CASE_INSENSITIVE))
-        {
-            has_case_sensitive = true;
-            break;
-        }
-    }
-
-    for (size_t i=0; i< comp.size(); i++)
-    {
-
         long base_len=-1;
         const completion_t &el = comp.at(i);
         
         wcstring completion_text;
         wcstring description_text;
 
-        if (has_case_sensitive && (el.flags & COMPLETE_CASE_INSENSITIVE))
-        {
-            continue;
-        }
-
         // Note that an empty completion is perfectly sensible here, e.g. tab-completing 'foo' with a file called 'foo' and another called 'foobar'
-        if (el.flags & COMPLETE_REPLACES_TOKEN)
+        if ((el.flags & COMPLETE_REPLACES_TOKEN) && match_type_shares_prefix(el.match.type))
         {
+            // Compute base_len if we have not yet
             if (base_len == -1)
             {
                 const wchar_t *begin, *buff = data->command_line.c_str();
@@ -1511,14 +1472,16 @@ static bool reader_can_replace(const wcstring &in, int flags)
     return true;
 }
 
-/* Compare two completions, except make the case insensitive comes larger than everyone (so they come last) */
+/* Compare two completions, ordering completions with better match types first */
-bool case_sensitive_completion_compare(const completion_t &a, const completion_t &b)
+bool compare_completions_by_match_type(const completion_t &a, const completion_t &b)
 {
-    if (a.is_case_insensitive() != b.is_case_insensitive())
+    /* Compare match types */
+    int match_compare = a.match.compare(b.match);
+    if (match_compare != 0)
     {
-        /* Case insensitive ones come last. Exactly one of a, b is case insensitive. If it's a, return false, i.e. not less than, to make it appear at the end. */
+        return match_compare < 0;
-        return ! a.is_case_insensitive();
     }
+
     /* Compare using file comparison */
     return wcsfilecmp(a.completion.c_str(), b.completion.c_str()) < 0;
 }
@@ -1526,7 +1489,28 @@ bool case_sensitive_completion_compare(const completion_t &a, const completion_t
 /* Order completions such that case insensitive completions come first. */
 static void prioritize_completions(std::vector<completion_t> &comp)
 {
-    sort(comp.begin(), comp.end(), case_sensitive_completion_compare);
+    /* Determine the best match type */
+    size_t i;
+    fuzzy_match_type_t best_type = fuzzy_match_none;
+    for (i=0; i < comp.size(); i++)
+    {
+        const completion_t &el = comp.at(i);
+        if (el.match.type < best_type)
+            best_type = el.match.type;
+    }
+    
+    /* Throw out completions whose match types are not the best. */
+    i = comp.size();
+    while (i--)
+    {
+        if (comp.at(i).match.type != best_type)
+        {
+            comp.erase(comp.begin() + i);
+        }
+    }
+    
+    /* Sort the remainder */
+    sort(comp.begin(), comp.end(), compare_completions_by_match_type);
 }
 
 /* Given a list of completions, get the completion at an index past *inout_idx, and then increment it. inout_idx should be initialized to (size_t)(-1) for the first call. */
@@ -1552,7 +1536,7 @@ static const completion_t *cycle_competions(const std::vector<completion_t> &com
         const completion_t &c = comp.at(idx);
 
         /* Try this completion */
-        if (! c.is_case_insensitive() || reader_can_replace(command_line, c.flags))
+        if (! (c.flags & COMPLETE_REPLACES_TOKEN) || reader_can_replace(command_line, c.flags))
         {
             /* Success */
             result = &c;
@@ -1586,12 +1570,8 @@ static const completion_t *cycle_competions(const std::vector<completion_t> &com
 
 static bool handle_completions(const std::vector<completion_t> &comp)
 {
-    wchar_t *base = NULL;
-    size_t len = 0;
     bool done = false;
     bool success = false;
-    int count = 0;
-    int flags=0;
     const wchar_t *begin, *end, *buff = data->command_line.c_str();
 
     parse_util_token_extent(buff, data->buff_pos, &begin, 0, 0, 0);
@@ -1625,7 +1605,7 @@ static bool handle_completions(const std::vector<completion_t> &comp)
               the token doesn't contain evil operators
               like {}
              */
-            if (! c.is_case_insensitive() || reader_can_replace(tok, c.flags))
+            if (! (c.flags & COMPLETE_REPLACES_TOKEN) || reader_can_replace(tok, c.flags))
             {
                 completion_insert(c.completion.c_str(), c.flags);
             }
@@ -1638,141 +1618,146 @@ static bool handle_completions(const std::vector<completion_t> &comp)
 
     if (!done)
     {
-        /* Try to find something to insert with the correct case */
-        for (size_t i=0; i< comp.size() ; i++)
-        {
-            const completion_t &c =  comp.at(i);
     
-            /* Ignore case insensitive completions for now */
+        /* Determine the type of the best match(es) */
-            if (c.is_case_insensitive())
+        fuzzy_match_type_t best_match_type = fuzzy_match_none;
+        for (size_t i=0; i < comp.size(); i++)
+        {
+            const completion_t &el = comp.at(i);
+            if (el.match.type < best_match_type)
+            {
+                best_match_type = el.match.type;
+            }
+        }
+        
+        /* Determine whether we are going to replace the token or not. If any commands of the best type do not require replacement, then ignore all those that want to use replacement */
+        bool will_replace_token = true;
+        for (size_t i=0; i< comp.size(); i++)
+        {
+            const completion_t &el = comp.at(i);
+            if (el.match.type == best_match_type && ! (el.flags & COMPLETE_REPLACES_TOKEN))
+            {
+                will_replace_token = false;
+                break;
+            }
+        }
+        
+        /* Decide which completions survived. There may be a lot of them; it would be nice if we could figure out how to avoid copying them here */
+        std::vector<completion_t> surviving_completions;
+        for (size_t i=0; i < comp.size(); i++)
+        {
+            const completion_t &el = comp.at(i);
+            /* Only use completions with the best match type */
+            if (el.match.type != best_match_type)
                 continue;
             
-            count++;
+            /* Only use completions that match replace_token */
+            bool completion_replace_token = !! (el.flags & COMPLETE_REPLACES_TOKEN);
+            if (completion_replace_token != will_replace_token)
+                continue;
             
-            if (base)
+            /* Don't use completions that want to replace, if we cannot replace them */
+            if (completion_replace_token && ! reader_can_replace(tok, el.flags))
+                continue;
+            
+            /* This completion survived */
+            surviving_completions.push_back(el);
+        }
+        
+        
+        /* Try to find a common prefix to insert among the surviving completions */
+        wcstring common_prefix;
+        complete_flags_t flags = 0;
+        bool prefix_is_partial_completion = false;
+        for (size_t i=0; i < surviving_completions.size(); i++)
+        {
+            const completion_t &el = surviving_completions.at(i);
+            if (i == 0)
             {
-                size_t new_len = comp_len(base, c.completion.c_str());
+                /* First entry, use the whole string */
-                len = mini(new_len, len);
+                common_prefix = el.completion;
+                flags = el.flags;
             }
             else
             {
-                base = wcsdup(c.completion.c_str());
+                /* Determine the shared prefix length. */
-                len = wcslen(base);
+                size_t idx, max = mini(common_prefix.size(), el.completion.size());
-                flags = c.flags;
+                for (idx=0; idx < max; idx++) {
-            }
+                    wchar_t ac = common_prefix.at(idx), bc = el.completion.at(idx);
-        }
+                    bool matches = (ac == bc);
+                    /* If we are replacing the token, allow case to vary */
+                    if (will_replace_token && ! matches)
+                    {
+                        /* Hackish way to compare two strings in a case insensitive way, hopefully better than towlower(). */
+                        matches = (wcsncasecmp(&ac, &bc, 1) == 0);
+                    }
+                    if (! matches)
+                        break;
+                }
                 
-        /* If we found something to insert, do it. */
+                /* idx is now the length of the new common prefix */
-        if (len > 0)
+                common_prefix.resize(idx);
-        {
+                prefix_is_partial_completion = true;
-            if (count > 1)
-                flags = flags | COMPLETE_NO_SPACE;
                 
-            base[len]=L'\0';
+                /* Early out if we decide there's no common prefix */
-            completion_insert(base, flags);
+                if (idx == 0)
-            done = true;
-            success = true;
-        }
-    }
-
-
-
-    if (!done && base == NULL)
-    {
-        /* Try to find something to insert ignoring case */
-        if (begin)
-        {
-            size_t offset = tok.size();
-
-            count = 0;
-
-            for (size_t i=0; i< comp.size(); i++)
-            {
-                const completion_t &c = comp.at(i);
-
-                if (! c.is_case_insensitive())
-                    continue;
-
-                if (!reader_can_replace(tok, c.flags))
-                {
-                    len=0;
                     break;
-                }
-
-                count++;
-
-                if (base)
-                {
-                    size_t new_len = offset + comp_ilen(base+offset, c.completion.c_str()+offset);
-                    len = new_len < len ? new_len: len;
-                }
-                else
-                {
-                    base = wcsdup(c.completion.c_str());
-                    len = wcslen(base);
-                    flags = c.flags;
-
-                }
             }
-
-            if (len > offset)
-            {
-                if (count > 1)
-                    flags = flags | COMPLETE_NO_SPACE;
-
-                base[len]=L'\0';
-                completion_insert(base, flags);
-                done = 1;
-                success = true;
-            }
-
         }
-    }
         
-    free(base);
+        if (! common_prefix.empty())
-
-    if (!done)
-    {
-        /*
-          There is no common prefix in the completions, and show_list
-          is true, so we print the list
-        */
-        size_t len, prefix_start = 0;
-        wcstring prefix;
-        parse_util_get_parameter_info(data->command_line, data->buff_pos, NULL, &prefix_start, NULL);
-
-        assert(data->buff_pos >= prefix_start);
-        len = data->buff_pos - prefix_start;
-
-        if (len <= PREFIX_MAX_LEN)
         {
-            prefix.append(data->command_line, prefix_start, len);
+            /* We got something. If more than one completion contributed, then it means we have a prefix; don't insert a space after it */
+            if (prefix_is_partial_completion)
+                flags |= COMPLETE_NO_SPACE;
+            completion_insert(common_prefix.c_str(), flags);
+            success = true;
         }
         else
         {
-            // append just the end of the string
+            /* We didn't get a common prefix. Print the list. */
-            prefix = wcstring(&ellipsis_char, 1);
+            size_t len, prefix_start = 0;
-            prefix.append(data->command_line, prefix_start + len - PREFIX_MAX_LEN, PREFIX_MAX_LEN);
+            wcstring prefix;
+            parse_util_get_parameter_info(data->command_line, data->buff_pos, NULL, &prefix_start, NULL);
+
+            assert(data->buff_pos >= prefix_start);
+            len = data->buff_pos - prefix_start;
+            
+            if (match_type_requires_full_replacement(best_match_type))
+            {
+                // No prefix
+                prefix.clear();
+            }
+            else if (len <= PREFIX_MAX_LEN)
+            {
+                prefix.append(data->command_line, prefix_start, len);
+            }
+            else
+            {
+                // append just the end of the string
+                prefix = wcstring(&ellipsis_char, 1);
+                prefix.append(data->command_line, prefix_start + len - PREFIX_MAX_LEN, PREFIX_MAX_LEN);
+            }
+
+            {
+                int is_quoted;
+
+                wchar_t quote;
+                parse_util_get_parameter_info(data->command_line, data->buff_pos, &quote, NULL, NULL);
+                is_quoted = (quote != L'\0');
+
+                /* Clear the autosuggestion from the old commandline before abandoning it (see #561) */
+                if (! data->autosuggestion.empty())
+                    reader_repaint_without_autosuggestion();
+
+                write_loop(1, "\n", 1);
+                
+                run_pager(prefix, is_quoted, surviving_completions);
+            }
+            s_reset(&data->screen, screen_reset_abandon_line);
+            reader_repaint();
+            success = false;
         }
-
-        {
-            int is_quoted;
-
-            wchar_t quote;
-            parse_util_get_parameter_info(data->command_line, data->buff_pos, &quote, NULL, NULL);
-            is_quoted = (quote != L'\0');
-
-            /* Clear the autosuggestion from the old commandline before abandoning it (see #561) */
-            if (! data->autosuggestion.empty())
-                reader_repaint_without_autosuggestion();
-
-            write_loop(1, "\n", 1);
-
-            run_pager(prefix, is_quoted, comp);
-        }
-        s_reset(&data->screen, screen_reset_abandon_line);
-        reader_repaint();
-        success = false;
     }
     return success;
 }
@@ -3078,7 +3063,7 @@ const wchar_t *reader_readline(void)
                     const wcstring buffcpy = wcstring(cmdsub_begin, token_end);
 
                     //fprintf(stderr, "Complete (%ls)\n", buffcpy.c_str());
-                    data->complete_func(buffcpy, comp, COMPLETION_REQUEST_DEFAULT | COMPLETION_REQUEST_DESCRIPTIONS, NULL);
+                    data->complete_func(buffcpy, comp, COMPLETION_REQUEST_DEFAULT | COMPLETION_REQUEST_DESCRIPTIONS | COMPLETION_REQUEST_FUZZY_MATCH, NULL);
 
                     /* Munge our completions */
                     sort_and_make_unique(comp);

wildcard.cpp

@@ -106,12 +106,12 @@ wildcards using **.
 static std::map<wcstring, wcstring> suffix_map;
 
 
-int wildcard_has(const wchar_t *str, int internal)
+bool wildcard_has(const wchar_t *str, bool internal)
 {
     if (!str)
     {
         debug(2, L"Got null string on line %d of file %s", __LINE__, __FILE__);
-        return 0;
+        return false;
     }
 
     if (internal)
@@ -119,7 +119,7 @@ int wildcard_has(const wchar_t *str, int internal)
         for (; *str; str++)
         {
             if ((*str == ANY_CHAR) || (*str == ANY_STRING) || (*str == ANY_STRING_RECURSIVE))
-                return 1;
+                return true;
         }
     }
     else
@@ -128,12 +128,12 @@ int wildcard_has(const wchar_t *str, int internal)
         for (; *str; str++)
         {
             if (((*str == L'*') || (*str == L'?')) && (prev != L'\\'))
-                return 1;
+                return true;
             prev = *str;
         }
     }
 
-    return 0;
+    return false;
 }
 
 /**
@@ -209,7 +209,8 @@ static bool wildcard_complete_internal(const wcstring &orig,
                                        const wchar_t *desc,
                                        wcstring(*desc_func)(const wcstring &),
                                        std::vector<completion_t> &out,
-                                       int flags)
+                                       expand_flags_t expand_flags,
+                                       complete_flags_t flags)
 {
     if (!wc || ! str || orig.empty())
     {
@@ -217,20 +218,59 @@ static bool wildcard_complete_internal(const wcstring &orig,
         return 0;
     }
     
-    if (*wc == 0 &&
+    bool has_match = false;
-            ((str[0] != L'.') || (!is_first)))
+    string_fuzzy_match_t fuzzy_match(fuzzy_match_exact);
-    {
+    const bool at_end_of_wildcard = (*wc == L'\0');
-        wcstring out_completion;
+    const wchar_t *completion_string = NULL;
-        wcstring out_desc = (desc ? desc : L"");
     
-        if (flags & COMPLETE_REPLACES_TOKEN)
+    // Hack hack hack
+    // Implement EXPAND_FUZZY_MATCH by short-circuiting everything if there are no remaining wildcards
+    if ((expand_flags & EXPAND_FUZZY_MATCH) && ! at_end_of_wildcard && ! wildcard_has(wc, true))
+    {
+        fuzzy_match = string_fuzzy_match_string(wc, str);
+        if (fuzzy_match.type != fuzzy_match_none)
         {
-            out_completion = orig;
+            has_match = true;
+            
+            /* If we're not a prefix or exact match, then we need to replace the token. Note that in this case we're not going to call ourselves recursively, so these modified flags won't "leak" except into the completion. */
+            if (match_type_requires_full_replacement(fuzzy_match.type))
+            {
+                flags |= COMPLETE_REPLACES_TOKEN;
+                completion_string = orig.c_str();
+            }
+            else
+            {
+                /* Since we are not replacing the token, be careful to only store the part of the string after the wildcard */
+                size_t wc_len = wcslen(wc);
+                assert(wcslen(str) >= wc_len);
+                completion_string = str + wcslen(wc);
+            }
         }
-        else
+    }
+    
+    /* Maybe we satisfied the wildcard normally */
+    if (! has_match) {
+        bool file_has_leading_dot = (is_first && str[0] == L'.');
+        if (at_end_of_wildcard && ! file_has_leading_dot)
         {
-            out_completion = str;
+            has_match = true;
+            if (flags & COMPLETE_REPLACES_TOKEN)
+            {
+                completion_string = orig.c_str();
+            }
+            else
+            {
+                completion_string = str;
+            }
         }
+    }
+    
+    if (has_match)
+    {
+        /* Wildcard complete */
+        assert(completion_string != NULL);
+        wcstring out_completion = completion_string;
+        wcstring out_desc = (desc ? desc : L"");
 
         size_t complete_sep_loc = out_completion.find(PROG_COMPLETE_SEP);
         if (complete_sep_loc != wcstring::npos)
@@ -241,7 +281,7 @@ static bool wildcard_complete_internal(const wcstring &orig,
         }
         else
         {
-            if (desc_func)
+            if (desc_func && ! (expand_flags & EXPAND_NO_DESCRIPTIONS))
             {
                 /*
                   A description generating function is specified, call
@@ -256,11 +296,10 @@ static bool wildcard_complete_internal(const wcstring &orig,
         }
 
         /* Note: out_completion may be empty if the completion really is empty, e.g. tab-completing 'foo' when a file 'foo' exists. */
-        append_completion(out, out_completion, out_desc, flags);
+        append_completion(out, out_completion, out_desc, flags, fuzzy_match);
         return true;
     }
     
-
     if (*wc == ANY_STRING)
     {
         bool res=false;
@@ -272,7 +311,7 @@ static bool wildcard_complete_internal(const wcstring &orig,
         /* Try all submatches */
         do
         {
-            res = wildcard_complete_internal(orig, str, wc+1, 0, desc, desc_func, out, flags);
+            res = wildcard_complete_internal(orig, str, wc+1, 0, desc, desc_func, out, expand_flags, flags);
             if (res)
                 break;
         }
@@ -280,17 +319,13 @@ static bool wildcard_complete_internal(const wcstring &orig,
         return res;
 
     }
-    else if (*wc == ANY_CHAR)
+    else if (*wc == ANY_CHAR || *wc == *str)
     {
-        return wildcard_complete_internal(orig, str+1, wc+1, 0, desc, desc_func, out, flags);
+        return wildcard_complete_internal(orig, str+1, wc+1, 0, desc, desc_func, out, expand_flags, flags);
-    }
-    else if (*wc == *str)
-    {
-        return wildcard_complete_internal(orig, str+1, wc+1, 0, desc, desc_func, out, flags);
     }
     else if (towlower(*wc) == towlower(*str))
     {
-        return wildcard_complete_internal(orig, str+1, wc+1, 0, desc, desc_func, out, flags | COMPLETE_CASE_INSENSITIVE | COMPLETE_REPLACES_TOKEN);
+        return wildcard_complete_internal(orig, str+1, wc+1, 0, desc, desc_func, out, expand_flags, flags | COMPLETE_REPLACES_TOKEN);
     }
     return false;
 }
@@ -300,10 +335,11 @@ bool wildcard_complete(const wcstring &str,
                        const wchar_t *desc,
                        wcstring(*desc_func)(const wcstring &),
                        std::vector<completion_t> &out,
-                       int flags)
+                       expand_flags_t expand_flags,
+                       complete_flags_t flags)
 {
     bool res;
-    res =  wildcard_complete_internal(str, str.c_str(), wc, true, desc, desc_func, out, flags);
+    res =  wildcard_complete_internal(str, str.c_str(), wc, true, desc, desc_func, out, expand_flags, flags);
     return res;
 }
 
@@ -622,7 +658,7 @@ static void wildcard_completion_allocate(std::vector<completion_t> &list,
 
     if (sz >= 0 && S_ISDIR(buf.st_mode))
     {
-        flags = flags | COMPLETE_NO_SPACE;
+        flags |= COMPLETE_NO_SPACE;
         munged_completion = completion;
         munged_completion.push_back(L'/');
         if (wants_desc)
@@ -642,7 +678,7 @@ static void wildcard_completion_allocate(std::vector<completion_t> &list,
     }
 
     const wcstring &completion_to_use = munged_completion.empty() ? completion : munged_completion;
-    wildcard_complete(completion_to_use, wc, sb.c_str(), NULL, list, flags);
+    wildcard_complete(completion_to_use, wc, sb.c_str(), NULL, list, expand_flags, flags);
 }
 
 /**
@@ -650,20 +686,19 @@ static void wildcard_completion_allocate(std::vector<completion_t> &list,
   expansion flags specified. flags can be a combination of
   EXECUTABLES_ONLY and DIRECTORIES_ONLY.
 */
-static int test_flags(const wchar_t *filename,
+static bool test_flags(const wchar_t *filename, expand_flags_t flags)
-                      int flags)
 {
     if (flags & DIRECTORIES_ONLY)
     {
         struct stat buf;
         if (wstat(filename, &buf) == -1)
         {
-            return 0;
+            return false;
         }
 
         if (!S_ISDIR(buf.st_mode))
         {
-            return 0;
+            return false;
         }
     }
 
@@ -671,10 +706,10 @@ static int test_flags(const wchar_t *filename,
     if (flags & EXECUTABLES_ONLY)
     {
         if (waccess(filename, X_OK) != 0)
-            return 0;
+            return false;
     }
 
-    return 1;
+    return true;
 }
 
 /** Appends a completion to the completion list, if the string is missing from the set. */
@@ -738,7 +773,7 @@ static int wildcard_expand_internal(const wchar_t *wc,
            Avoid excessive number of returned matches for wc ending with a *
         */
         size_t len = wcslen(wc);
-        if (len && (wc[len-1]==ANY_STRING))
+        if (len > 0 && (wc[len-1]==ANY_STRING))
         {
             wchar_t * foo = wcsdup(wc);
             foo[len-1]=0;
@@ -748,11 +783,9 @@ static int wildcard_expand_internal(const wchar_t *wc,
         }
     }
 
-    /*
+    /* Initialize various variables */
-      Initialize various variables
-    */
 
-    dir_string = base_dir[0]==L'\0'?L".":base_dir;
+    dir_string = (base_dir[0] == L'\0') ? L"." : base_dir;
 
     if (!(dir = wopendir(dir_string)))
     {
@@ -795,11 +828,7 @@ static int wildcard_expand_internal(const wchar_t *wc,
 
                         if (test_flags(long_name.c_str(), flags))
                         {
-                            wildcard_completion_allocate(out,
+                            wildcard_completion_allocate(out, long_name, next, L"", flags);
-                                                         long_name,
-                                                         next,
-                                                         L"",
-                                                         flags);
                         }
                     }
                 }
@@ -812,9 +841,7 @@ static int wildcard_expand_internal(const wchar_t *wc,
         }
         else
         {
-            /*
+            /* This is the last wildcard segment, and it is not empty. Match files/directories. */
-              This is the last wildcard segment, and it is not empty. Match files/directories.
-            */
             wcstring name_str;
             while (wreaddir(dir, name_str))
             {
@@ -823,24 +850,13 @@ static int wildcard_expand_internal(const wchar_t *wc,
 
                     const wcstring long_name = make_path(base_dir, name_str);
 
-                    /*
+                    /* Test for matches before stating file, so as to minimize the number of calls to the much slower stat function. The only expand flag we care about is EXPAND_FUZZY_MATCH; we have no complete flags. */
-                      Test for matches before stating file, so as to minimize the number of calls to the much slower stat function
-                    */
                     std::vector<completion_t> test;
-                    if (wildcard_complete(name_str,
+                    if (wildcard_complete(name_str, wc, L"", NULL, test, flags & EXPAND_FUZZY_MATCH, 0))
-                                          wc,
-                                          L"",
-                                          0,
-                                          test,
-                                          0))
                     {
                         if (test_flags(long_name.c_str(), flags))
                         {
-                            wildcard_completion_allocate(out,
+                            wildcard_completion_allocate(out, long_name, name_str, wc, flags);
-                                                         long_name,
-                                                         name_str,
-                                                         wc,
-                                                         flags);
 
                         }
                     }

wildcard.h

@@ -18,6 +18,7 @@
 #include "util.h"
 #include "common.h"
 #include "expand.h"
+#include "complete.h"
 
 /*
   Use unencoded private-use keycodes for internal characters
@@ -79,10 +80,8 @@ int wildcard_expand_string(const wcstring &wc, const wcstring &base_dir, expand_
 bool wildcard_match(const wcstring &str, const wcstring &wc, bool leading_dots_fail_to_match = false);
 
 
-/**
+/** Check if the specified string contains wildcards */
-   Check if the specified string contains wildcards
+bool wildcard_has(const wchar_t *str, bool internal);
-*/
-int wildcard_has(const wchar_t *str, int internal);
 
 /**
    Test wildcard completion
@@ -92,6 +91,7 @@ bool wildcard_complete(const wcstring &str,
                        const wchar_t *desc,
                        wcstring(*desc_func)(const wcstring &),
                        std::vector<completion_t> &out,
-                       expand_flags_t flags);
+                       expand_flags_t expand_flags,
+                       complete_flags_t flags);
 
 #endif