Support for importing fish 1.x's history and format, and also bash

fish_tests.cpp

@@ -833,6 +833,7 @@ class history_tests_t {
 public:
     static void test_history(void);
     static void test_history_merge(void);
+    static void test_history_formats(void);
 };
 
 static wcstring random_string(void) {
@@ -975,6 +976,129 @@ void history_tests_t::test_history_merge(void) {
     delete everything; //not as scary as it looks
 }
 
+static bool install_sample_history(const wchar_t *name) {
+    char command[512];
+    snprintf(command, sizeof command, "cp tests/%ls ~/.config/fish/%ls_history", name, name);    
+    if (system(command)) {
+        err(L"Failed to copy sample history");
+        return false;
+    }
+    return true;
+}
+
+/* Indicates whether the history is equal to the given null-terminated array of strings. */
+static bool history_equals(history_t &hist, const wchar_t * const *strings) {
+    /* Count our expected items */
+    size_t expected_count = 0;
+    while (strings[expected_count]) {
+        expected_count++;
+    }
+    
+    /* Ensure the contents are the same */
+    size_t history_idx = 1;
+    size_t array_idx = 0;
+    for (;;) {
+        const wchar_t *expected = strings[array_idx];
+        history_item_t item = hist.item_at_index(history_idx);
+        if (expected == NULL) {
+            if (! item.empty()) {
+                err(L"Expected empty item at history index %lu", history_idx);
+            }
+            break;
+        } else {
+            if (item.str() != expected) {
+                err(L"Expected '%ls', found '%ls' at index %lu", expected, item.str().c_str(), history_idx);
+            }
+        }
+        history_idx++;
+        array_idx++;
+    }
+    
+    return true;
+}
+
+void history_tests_t::test_history_formats(void) {
+    const wchar_t *name;
+    
+    // Test inferring and reading legacy and bash history formats
+    name = L"history_sample_fish_1_x";
+    say(L"Testing %ls", name);
+    if (! install_sample_history(name)) {
+        err(L"Couldn't open file tests/%ls", name);
+    } else {
+        /* Note: This is backwards from what appears in the file */
+        const wchar_t * const expected[] = {
+            L"#def",
+
+            L"echo #abc",
+            
+            L"function yay\n"
+            "echo hi\n"
+            "end",
+            
+            L"cd foobar",
+            
+            L"ls /",
+                        
+            NULL
+        };
+        
+        history_t &test_history = history_t::history_with_name(name);
+        if (! history_equals(test_history, expected)) {
+            err(L"test_history_formats failed for %ls\n", name);
+        }
+        test_history.clear();
+    }
+    
+    name = L"history_sample_fish_2_0";
+    say(L"Testing %ls", name);
+    if (! install_sample_history(name)) {
+        err(L"Couldn't open file tests/%ls", name);
+    } else {
+        const wchar_t * const expected[] = {
+            L"echo this has\\\nbackslashes",
+            
+            L"function foo\n"
+            "echo bar\n"
+            "end",
+            
+            L"echo alpha",
+            
+            NULL
+        };
+        
+        history_t &test_history = history_t::history_with_name(name);
+        if (! history_equals(test_history, expected)) {
+            err(L"test_history_formats failed for %ls\n", name);
+        }
+        test_history.clear();
+    }
+    
+    say(L"Testing bash import");
+    FILE *f = fopen("tests/history_sample_bash", "r");
+    if (! f) {
+        err(L"Couldn't open file tests/history_sample_bash");
+    } else {
+        // It should skip over the export command since that's a bash-ism
+        const wchar_t *expected[] = {
+            L"echo supsup",
+            
+            L"history --help",
+            
+            L"echo foo",
+            
+            NULL
+        };
+        history_t &test_history = history_t::history_with_name(L"bash_import");
+        test_history.populate_from_bash(f);
+        if (! history_equals(test_history, expected)) {
+            err(L"test_history_formats failed for bash import\n");
+        }
+        test_history.clear();
+        fclose(f);
+    }
+}
+
 
 /**
    Main test 
@@ -1013,6 +1137,7 @@ int main( int argc, char **argv )
     test_autosuggest();
     history_tests_t::test_history();
     history_tests_t::test_history_merge();
+    history_tests_t::test_history_formats();
 	
 	say( L"Encountered %d errors in low-level tests", err_count );
 

history.cpp

@@ -91,7 +91,6 @@ class history_lru_node_t : public lru_node_t {
         required_paths(item.required_paths)
     {}
     
-    bool write_to_file(FILE *f) const;
     bool write_yaml_to_file(FILE *f) const;
 };
 
@@ -264,7 +263,7 @@ static const char *next_line(const char *start, size_t length) {
 // Pass a pointer to a cursor size_t, initially 0
 // If custoff_timestamp is nonzero, skip items created at or after that timestamp
 // Returns (size_t)(-1) when done
-static size_t offset_of_next_item(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp)
+static size_t offset_of_next_item_fish_2_0(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp)
 {
     size_t cursor = *inout_cursor;
     size_t result = (size_t)(-1);
@@ -334,6 +333,78 @@ static size_t offset_of_next_item(const char *begin, size_t mmap_length, size_t
 }
 
 
+// Same as offset_of_next_item_fish_2_0, but for fish 1.x (pre fishfish)
+// Adapted from history_populate_from_mmap in history.c
+static size_t offset_of_next_item_fish_1_x(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp) {
+    if (mmap_length == 0 || *inout_cursor >= mmap_length)
+        return (size_t)(-1);
+    
+	const char *end = begin + mmap_length;    
+	const char *pos;
+
+	bool ignore_newline = false;
+	bool do_push = true;
+    bool all_done = false;
+
+    size_t result = *inout_cursor;
+	for( pos = begin + *inout_cursor; pos < end && ! all_done; pos++ )
+	{
+
+		if( do_push )
+		{
+			ignore_newline = (*pos == '#');
+			do_push = false;
+		}
+
+		switch( *pos )
+		{
+			case '\\':
+			{
+				pos++;
+				break;
+			}
+
+			case '\n':
+			{
+				if( ignore_newline )
+				{
+					ignore_newline = false;
+				}
+				else
+				{
+                    /* Note: pos will be left pointing just after this newline, because of the ++ in the loop */
+                    
+                    all_done = true;
+				}
+				break;
+			}
+		}
+	}
+    *inout_cursor = (pos - begin);
+    return result;
+}
+
+// Returns the offset of the next item based on the given history type, or -1
+static size_t offset_of_next_item(const char *begin, size_t mmap_length, history_file_type_t mmap_type, size_t *inout_cursor, time_t cutoff_timestamp) {
+    size_t result;
+    switch (mmap_type) {
+        case history_type_fish_2_0:
+            result = offset_of_next_item_fish_2_0(begin, mmap_length, inout_cursor, cutoff_timestamp);
+            break;
+        
+        case history_type_fish_1_x:
+            result = offset_of_next_item_fish_1_x(begin, mmap_length, inout_cursor, cutoff_timestamp);
+            break;
+    
+        default:
+        case history_type_unknown:
+            // Oh well
+            result = (size_t)(-1);
+            break;
+    }
+    return result;
+}
+
 history_t & history_t::history_with_name(const wcstring &name) {
     /* Note that histories are currently never deleted, so we can return a reference to them without using something like shared_ptr */
     scoped_lock locker(hist_lock);
@@ -395,15 +466,16 @@ void history_t::add(const wcstring &str, const path_list_t &valid_paths)
 
 void history_t::remove(const wcstring &str)
 {
-    history_item_t item_to_delete(str);
+    /* Add to our list of deleted items */
-    deleted_items.push_back(item_to_delete);
+    deleted_items.insert(str);
     
-    for (std::vector<history_item_t>::iterator iter = new_items.begin(); iter != new_items.end(); ++iter)
+    /* Remove from our list of new items */
+    for (std::vector<history_item_t>::iterator iter = new_items.begin(); iter != new_items.end();)
     {
-        if (iter->match_contents(item_to_delete))
+        if (iter->str() == str) {
-        {
+            iter = new_items.erase(iter);
-            new_items.erase(iter);
+        } else {
-            break;
+            iter++;
         }
     }
 }
@@ -426,7 +498,7 @@ void history_t::get_string_representation(wcstring &result, const wcstring &sepa
     load_old_if_needed();
     for (std::deque<size_t>::const_reverse_iterator iter = old_item_offsets.rbegin(); iter != old_item_offsets.rend(); ++iter) {        
         size_t offset = *iter;
-        const history_item_t item = history_t::decode_item(mmap_start + offset, mmap_length - offset);
+        const history_item_t item = history_t::decode_item(mmap_start + offset, mmap_length - offset, mmap_type);
         if (! first)
             result.append(separator);
         result.append(item.str());
@@ -454,7 +526,7 @@ history_item_t history_t::item_at_index(size_t idx) {
     if (idx < old_item_count) {
         /* idx=0 corresponds to last item in old_item_offsets */
         size_t offset = old_item_offsets.at(old_item_count - idx - 1);
-        return history_t::decode_item(mmap_start + offset, mmap_length - offset);
+        return history_t::decode_item(mmap_start + offset, mmap_length - offset, mmap_type);
     }
     
     /* Index past the valid range, so return an empty history item */
@@ -506,7 +578,8 @@ static bool extract_prefix(std::string &key, std::string &value, const std::stri
     return where != std::string::npos;
 }
 
-history_item_t history_t::decode_item(const char *base, size_t len) {
+/* Decode an item via the fish 2.0 format */
+history_item_t history_t::decode_item_fish_2_0(const char *base, size_t len) {
     wcstring cmd;
     time_t when = 0;
     path_list_t paths;
@@ -562,7 +635,6 @@ history_item_t history_t::decode_item(const char *base, size_t len) {
                 /* We're going to consume this line */
                 cursor += advance;
                 
-
                 /* Skip the leading dash-space and then store this path it */
                 line.erase(0, 2);
                 unescape_yaml(line);
@@ -574,14 +646,151 @@ history_item_t history_t::decode_item(const char *base, size_t len) {
 done:
     paths.reverse();
     return history_item_t(cmd, when, paths);
+}
 
+history_item_t history_t::decode_item(const char *base, size_t len, history_file_type_t type) {
+    switch (type) {
+        case history_type_fish_1_x: return history_t::decode_item_fish_1_x(base, len);
+        case history_type_fish_2_0: return history_t::decode_item_fish_2_0(base, len);
+        default: return history_item_t(L"");
+    }
+}
+
+/**
+   Remove backslashes from all newlines. This makes a string from the
+   history file better formated for on screen display.
+*/
+static wcstring history_unescape_newlines_fish_1_x( const wcstring &in_str )
+{
+    wcstring out;
+	for (const wchar_t *in = in_str.c_str(); *in; in++)
+	{
+		if( *in == L'\\' )
+		{
+			if( *(in+1)!= L'\n')
+			{
+				out.push_back(*in);
+			}
+		}
+		else
+		{
+			out.push_back(*in);
+		}
+	}
+	return out;
+}
+
+
+/* Decode an item via the fish 1.x format. Adapted from fish 1.x's item_get(). */
+history_item_t history_t::decode_item_fish_1_x(const char *begin, size_t length) {
+
+    const char *end = begin + length;
+    const char *pos=begin;
+
+    bool was_backslash = 0;
+    wcstring out;
+    bool first_char = true;
+    bool timestamp_mode = false;
+    time_t timestamp = 0;
+    
+    while( 1 )
+    {
+        wchar_t c;
+        mbstate_t state;
+        size_t res;
+
+        memset( &state, 0, sizeof(state) );
+
+        res = mbrtowc( &c, pos, end-pos, &state );
+
+        if( res == (size_t)-1 )
+        {
+            pos++;
+            continue;
+        }
+        else if( res == (size_t)-2 )
+        {
+            break;
+        }
+        else if( res == (size_t)0 )
+        {
+            pos++;
+            continue;
+        }
+        pos += res;
+
+        if( c == L'\n' )
+        {
+            if( timestamp_mode )
+            {
+                const wchar_t *time_string = out.c_str();
+                while( *time_string && !iswdigit(*time_string))
+                    time_string++;
+                errno=0;
+
+                if( *time_string )
+                {
+                    time_t tm;
+                    wchar_t *end;
+
+                    errno = 0;
+                    tm = (time_t)wcstol( time_string, &end, 10 );
+
+                    if( tm && !errno && !*end )
+                    {
+                        timestamp = tm;
+                    }
+
+                }
+
+                out.clear();
+                timestamp_mode = false;
+                continue;
+            }
+            if( !was_backslash )
+                break;
+        }
+
+        if( first_char )
+        {
+            if( c == L'#' )
+                timestamp_mode = true;
+        }
+
+        first_char = false;
+
+        out.push_back(c);
+
+        was_backslash = ( (c == L'\\') && !was_backslash);
+
+    }
+    
+    out = history_unescape_newlines_fish_1_x(out);
+    return history_item_t(out, timestamp);
+}
+
+
+/* Try to infer the history file type based on inspecting the data */
+static history_file_type_t infer_file_type(const char *data, size_t len) {
+    history_file_type_t result = history_type_unknown;
+    if (len > 0) {
+        /* Old fish started with a # */
+        if (data[0] == '#') {
+            result = history_type_fish_1_x;
+        } else {
+            /* Assume new fish */
+            result = history_type_fish_2_0;
+        }
+    }
+    return result;
 }
 
 void history_t::populate_from_mmap(void)
 {
+    mmap_type = infer_file_type(mmap_start, mmap_length);
     size_t cursor = 0;
     for (;;) {
-        size_t offset = offset_of_next_item(mmap_start, mmap_length, &cursor, birth_timestamp);
+        size_t offset = offset_of_next_item(mmap_start, mmap_length, mmap_type, &cursor, birth_timestamp);
         // If we get back -1, we're done
         if (offset == (size_t)(-1))
             break;
@@ -831,15 +1040,16 @@ void history_t::save_internal()
         const char *local_mmap_start = NULL;
         size_t local_mmap_size = 0;
         if (map_file(name, &local_mmap_start, &local_mmap_size)) {
+            const history_file_type_t local_mmap_type = infer_file_type(local_mmap_start, local_mmap_size);
             size_t cursor = 0;
             for (;;) {
-                size_t offset = offset_of_next_item(local_mmap_start, local_mmap_size, &cursor, 0);
+                size_t offset = offset_of_next_item(local_mmap_start, local_mmap_size, local_mmap_type, &cursor, 0);
                 /* If we get back -1, we're done */
                 if (offset == (size_t)(-1))
                     break;
 
                 /* Try decoding an old item */
-                const history_item_t old_item = history_t::decode_item(local_mmap_start + offset, local_mmap_size - offset);
+                const history_item_t old_item = history_t::decode_item(local_mmap_start + offset, local_mmap_size - offset, local_mmap_type);
                 if (old_item.empty() || is_deleted(old_item))
                 {
 //                    debug(0, L"Item is deleted : %s\n", old_item.str().c_str());
@@ -913,12 +1123,14 @@ void history_t::save_internal()
 	}
 }
 
-void history_t::save(void) {
+void history_t::save(void)
+{
     scoped_lock locker(lock);
     this->save_internal();
 }
 
-void history_t::clear(void) {
+void history_t::clear(void)
+{
     scoped_lock locker(lock);
     new_items.clear();
     deleted_items.clear();
@@ -931,6 +1143,63 @@ void history_t::clear(void) {
     
 }
 
+/* Indicate whether we ought to import the bash history file into fish */
+static bool should_import_bash_history_line(const std::string &line)
+{
+    if (line.empty())
+        return false;
+    
+    /* Very naive tests! Skip export; probably should skip others. */
+    const char * const ignore_prefixes[] = {
+        "export ",
+        "#"
+    };
+    
+    for (size_t i=0; i < sizeof ignore_prefixes / sizeof *ignore_prefixes; i++) {
+        const char *prefix = ignore_prefixes[i];
+        if (! line.compare(0, strlen(prefix), prefix)) {
+            return false;
+        }
+    }
+    printf("Importing %s\n", line.c_str());
+    return true;
+}
+
+void history_t::populate_from_bash(FILE *stream)
+{
+    /* Bash's format is very simple: just lines with #s for comments.
+       Ignore a few commands that are bash-specific. This list ought to be expanded.
+    */
+    std::string line;
+    for (;;) {
+        line.clear();
+        bool success = false, has_newline = false;
+        
+        /* Loop until we've read a line */
+        do {
+            char buff[128];
+            success = !! fgets(buff, sizeof buff, stream);
+            if (success) {
+                /* Skip the newline */
+                char *newline = strchr(buff, '\n');
+                if (newline) *newline = '\0';
+                has_newline = (newline != NULL);
+                
+                /* Append what we've got */
+                line.append(buff);
+            }
+        } while (success && ! has_newline);
+        
+        /* Maybe add this line */
+        if (should_import_bash_history_line(line)) {
+            this->add(str2wcstring(line));
+        }
+        
+        if (line.empty())
+            break;
+    }
+}
+
 void history_init()
 {
 }
@@ -1041,10 +1310,5 @@ void history_t::add_with_file_detection(const wcstring &str)
 
 bool history_t::is_deleted(const history_item_t &item) const
 {
-    for (std::vector<history_item_t>::const_iterator iter = deleted_items.begin(); iter != deleted_items.end(); ++iter)
+    return deleted_items.count(item.str()) > 0;
-    { 
-        if (iter->match_contents(item))  { return true; }
-    }
-
-    return false;
 }

history.h

@@ -57,20 +57,18 @@ class history_item_t {
     
     const path_list_t &get_required_paths() const { return required_paths; }
     
-    bool write_to_file(FILE *f) const;
-    
     bool operator==(const history_item_t &other) const {
         return contents == other.contents &&
                creation_timestamp == other.creation_timestamp &&
                required_paths == other.required_paths;
     }
+};
 
-    bool match_contents(const history_item_t &other) const {
+/* The type of file that we mmap'd */
-        return contents == other.contents;
+enum history_file_type_t {
-    }
+    history_type_unknown,
-    
+    history_type_fish_2_0,
-    /* Functions for testing only */
+    history_type_fish_1_x
-    
 };
 
 class history_t {
@@ -101,8 +99,8 @@ private:
 	/** New items. */
 	std::vector<history_item_t> new_items;
 
-  	/** Deleted items. */
+  	/** Deleted item contents. */
-	std::vector<history_item_t> deleted_items;
+	std::set<wcstring> deleted_items;
 
 	/** How many items we've added without saving */
 	size_t unsaved_item_count;
@@ -110,17 +108,18 @@ private:
 	/** The mmaped region for the history file */
 	const char *mmap_start;
 
-	/** The size of the mmaped region */
+	/** The size of the mmap'd region */
 	size_t mmap_length;
 
+    /** The type of file we mmap'd */
+    history_file_type_t mmap_type;
+
     /** Timestamp of when this history was created */
     const time_t birth_timestamp;
 
 	/** Timestamp of last save */
 	time_t save_timestamp;
     
-    static history_item_t decode_item(const char *ptr, size_t len);
-    
     void populate_from_mmap(void);
     
     /** List of old items, as offsets into out mmap data */
@@ -138,6 +137,11 @@ private:
     /** Saves history */
     void save_internal();
 
+    /* Versioned decoding */
+    static history_item_t decode_item_fish_2_0(const char *base, size_t len);
+    static history_item_t decode_item_fish_1_x(const char *base, size_t len);
+    static history_item_t decode_item(const char *base, size_t len, history_file_type_t type);
+        
 public:
     /** Returns history with the given name, creating it if necessary */
     static history_t & history_with_name(const wcstring &name);
@@ -157,6 +161,9 @@ public:
     /** Irreversibly clears history */
     void clear();
     
+    /** Populates from a bash history file */
+    void populate_from_bash(FILE *f);
+    
     /* Gets all the history into a string with ARRAY_SEP_STR. This is intended for the $history environment variable. This may be long! */
     void get_string_representation(wcstring &str, const wcstring &separator);
     

tests/history_sample_bash

@@ -0,0 +1,7 @@
+echo foo
+history --help
+#1339718290
+export HISTTIMEFORMAT='%F %T '
+#1339718298
+echo supsup
+#abcde

tests/history_sample_fish_1_x

@@ -0,0 +1,12 @@
+# 1339519901
+ls /
+# 1339519903
+cd foobar
+# 1339519906
+function yay\
+echo hi\
+end
+# 1339520882
+echo #abc
+# 1339520884
+#def

tests/history_sample_fish_2_0

@@ -0,0 +1,6 @@
+- cmd: echo alpha
+   when: 1339717374
+- cmd: function foo\necho bar\nend
+   when: 1339717377
+- cmd: echo this has\\\nbackslashes
+   when: 1339717385