From 18f04adccbaff74f12ad12f3f6ceb123e5ccf47f Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Fri, 15 Jun 2012 16:22:37 -0700 Subject: [PATCH] Support for importing fish 1.x's history and format, and also bash --- fish_tests.cpp | 125 ++++++++++++++ history.cpp | 314 +++++++++++++++++++++++++++++++--- history.h | 35 ++-- tests/history_sample_bash | 7 + tests/history_sample_fish_1_x | 12 ++ tests/history_sample_fish_2_0 | 6 + 6 files changed, 460 insertions(+), 39 deletions(-) create mode 100644 tests/history_sample_bash create mode 100644 tests/history_sample_fish_1_x create mode 100644 tests/history_sample_fish_2_0 diff --git a/fish_tests.cpp b/fish_tests.cpp index d99801949..efd699c39 100644 --- a/fish_tests.cpp +++ b/fish_tests.cpp @@ -833,6 +833,7 @@ class history_tests_t { public: static void test_history(void); static void test_history_merge(void); + static void test_history_formats(void); }; static wcstring random_string(void) { @@ -975,6 +976,129 @@ void history_tests_t::test_history_merge(void) { delete everything; //not as scary as it looks } +static bool install_sample_history(const wchar_t *name) { + char command[512]; + snprintf(command, sizeof command, "cp tests/%ls ~/.config/fish/%ls_history", name, name); + if (system(command)) { + err(L"Failed to copy sample history"); + return false; + } + return true; +} + +/* Indicates whether the history is equal to the given null-terminated array of strings. */ +static bool history_equals(history_t &hist, const wchar_t * const *strings) { + /* Count our expected items */ + size_t expected_count = 0; + while (strings[expected_count]) { + expected_count++; + } + + /* Ensure the contents are the same */ + size_t history_idx = 1; + size_t array_idx = 0; + for (;;) { + const wchar_t *expected = strings[array_idx]; + history_item_t item = hist.item_at_index(history_idx); + if (expected == NULL) { + if (! item.empty()) { + err(L"Expected empty item at history index %lu", history_idx); + } + break; + } else { + if (item.str() != expected) { + err(L"Expected '%ls', found '%ls' at index %lu", expected, item.str().c_str(), history_idx); + } + } + history_idx++; + array_idx++; + } + + return true; +} + +void history_tests_t::test_history_formats(void) { + const wchar_t *name; + + // Test inferring and reading legacy and bash history formats + name = L"history_sample_fish_1_x"; + say(L"Testing %ls", name); + if (! install_sample_history(name)) { + err(L"Couldn't open file tests/%ls", name); + } else { + /* Note: This is backwards from what appears in the file */ + const wchar_t * const expected[] = { + L"#def", + + L"echo #abc", + + L"function yay\n" + "echo hi\n" + "end", + + L"cd foobar", + + L"ls /", + + NULL + }; + + history_t &test_history = history_t::history_with_name(name); + if (! history_equals(test_history, expected)) { + err(L"test_history_formats failed for %ls\n", name); + } + test_history.clear(); + } + + name = L"history_sample_fish_2_0"; + say(L"Testing %ls", name); + if (! install_sample_history(name)) { + err(L"Couldn't open file tests/%ls", name); + } else { + const wchar_t * const expected[] = { + L"echo this has\\\nbackslashes", + + L"function foo\n" + "echo bar\n" + "end", + + L"echo alpha", + + NULL + }; + + history_t &test_history = history_t::history_with_name(name); + if (! history_equals(test_history, expected)) { + err(L"test_history_formats failed for %ls\n", name); + } + test_history.clear(); + } + + say(L"Testing bash import"); + FILE *f = fopen("tests/history_sample_bash", "r"); + if (! f) { + err(L"Couldn't open file tests/history_sample_bash"); + } else { + // It should skip over the export command since that's a bash-ism + const wchar_t *expected[] = { + L"echo supsup", + + L"history --help", + + L"echo foo", + + NULL + }; + history_t &test_history = history_t::history_with_name(L"bash_import"); + test_history.populate_from_bash(f); + if (! history_equals(test_history, expected)) { + err(L"test_history_formats failed for bash import\n"); + } + test_history.clear(); + fclose(f); + } +} + /** Main test @@ -1013,6 +1137,7 @@ int main( int argc, char **argv ) test_autosuggest(); history_tests_t::test_history(); history_tests_t::test_history_merge(); + history_tests_t::test_history_formats(); say( L"Encountered %d errors in low-level tests", err_count ); diff --git a/history.cpp b/history.cpp index c271b32e8..65e3273cc 100644 --- a/history.cpp +++ b/history.cpp @@ -91,7 +91,6 @@ class history_lru_node_t : public lru_node_t { required_paths(item.required_paths) {} - bool write_to_file(FILE *f) const; bool write_yaml_to_file(FILE *f) const; }; @@ -264,7 +263,7 @@ static const char *next_line(const char *start, size_t length) { // Pass a pointer to a cursor size_t, initially 0 // If custoff_timestamp is nonzero, skip items created at or after that timestamp // Returns (size_t)(-1) when done -static size_t offset_of_next_item(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp) +static size_t offset_of_next_item_fish_2_0(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp) { size_t cursor = *inout_cursor; size_t result = (size_t)(-1); @@ -334,6 +333,78 @@ static size_t offset_of_next_item(const char *begin, size_t mmap_length, size_t } +// Same as offset_of_next_item_fish_2_0, but for fish 1.x (pre fishfish) +// Adapted from history_populate_from_mmap in history.c +static size_t offset_of_next_item_fish_1_x(const char *begin, size_t mmap_length, size_t *inout_cursor, time_t cutoff_timestamp) { + if (mmap_length == 0 || *inout_cursor >= mmap_length) + return (size_t)(-1); + + const char *end = begin + mmap_length; + const char *pos; + + bool ignore_newline = false; + bool do_push = true; + bool all_done = false; + + size_t result = *inout_cursor; + for( pos = begin + *inout_cursor; pos < end && ! all_done; pos++ ) + { + + if( do_push ) + { + ignore_newline = (*pos == '#'); + do_push = false; + } + + switch( *pos ) + { + case '\\': + { + pos++; + break; + } + + case '\n': + { + if( ignore_newline ) + { + ignore_newline = false; + } + else + { + /* Note: pos will be left pointing just after this newline, because of the ++ in the loop */ + + all_done = true; + } + break; + } + } + } + *inout_cursor = (pos - begin); + return result; +} + +// Returns the offset of the next item based on the given history type, or -1 +static size_t offset_of_next_item(const char *begin, size_t mmap_length, history_file_type_t mmap_type, size_t *inout_cursor, time_t cutoff_timestamp) { + size_t result; + switch (mmap_type) { + case history_type_fish_2_0: + result = offset_of_next_item_fish_2_0(begin, mmap_length, inout_cursor, cutoff_timestamp); + break; + + case history_type_fish_1_x: + result = offset_of_next_item_fish_1_x(begin, mmap_length, inout_cursor, cutoff_timestamp); + break; + + default: + case history_type_unknown: + // Oh well + result = (size_t)(-1); + break; + } + return result; +} + history_t & history_t::history_with_name(const wcstring &name) { /* Note that histories are currently never deleted, so we can return a reference to them without using something like shared_ptr */ scoped_lock locker(hist_lock); @@ -395,15 +466,16 @@ void history_t::add(const wcstring &str, const path_list_t &valid_paths) void history_t::remove(const wcstring &str) { - history_item_t item_to_delete(str); - deleted_items.push_back(item_to_delete); - - for (std::vector::iterator iter = new_items.begin(); iter != new_items.end(); ++iter) + /* Add to our list of deleted items */ + deleted_items.insert(str); + + /* Remove from our list of new items */ + for (std::vector::iterator iter = new_items.begin(); iter != new_items.end();) { - if (iter->match_contents(item_to_delete)) - { - new_items.erase(iter); - break; + if (iter->str() == str) { + iter = new_items.erase(iter); + } else { + iter++; } } } @@ -426,7 +498,7 @@ void history_t::get_string_representation(wcstring &result, const wcstring &sepa load_old_if_needed(); for (std::deque::const_reverse_iterator iter = old_item_offsets.rbegin(); iter != old_item_offsets.rend(); ++iter) { size_t offset = *iter; - const history_item_t item = history_t::decode_item(mmap_start + offset, mmap_length - offset); + const history_item_t item = history_t::decode_item(mmap_start + offset, mmap_length - offset, mmap_type); if (! first) result.append(separator); result.append(item.str()); @@ -454,7 +526,7 @@ history_item_t history_t::item_at_index(size_t idx) { if (idx < old_item_count) { /* idx=0 corresponds to last item in old_item_offsets */ size_t offset = old_item_offsets.at(old_item_count - idx - 1); - return history_t::decode_item(mmap_start + offset, mmap_length - offset); + return history_t::decode_item(mmap_start + offset, mmap_length - offset, mmap_type); } /* Index past the valid range, so return an empty history item */ @@ -506,7 +578,8 @@ static bool extract_prefix(std::string &key, std::string &value, const std::stri return where != std::string::npos; } -history_item_t history_t::decode_item(const char *base, size_t len) { +/* Decode an item via the fish 2.0 format */ +history_item_t history_t::decode_item_fish_2_0(const char *base, size_t len) { wcstring cmd; time_t when = 0; path_list_t paths; @@ -562,7 +635,6 @@ history_item_t history_t::decode_item(const char *base, size_t len) { /* We're going to consume this line */ cursor += advance; - /* Skip the leading dash-space and then store this path it */ line.erase(0, 2); unescape_yaml(line); @@ -574,14 +646,151 @@ history_item_t history_t::decode_item(const char *base, size_t len) { done: paths.reverse(); return history_item_t(cmd, when, paths); +} + +history_item_t history_t::decode_item(const char *base, size_t len, history_file_type_t type) { + switch (type) { + case history_type_fish_1_x: return history_t::decode_item_fish_1_x(base, len); + case history_type_fish_2_0: return history_t::decode_item_fish_2_0(base, len); + default: return history_item_t(L""); + } +} + +/** + Remove backslashes from all newlines. This makes a string from the + history file better formated for on screen display. +*/ +static wcstring history_unescape_newlines_fish_1_x( const wcstring &in_str ) +{ + wcstring out; + for (const wchar_t *in = in_str.c_str(); *in; in++) + { + if( *in == L'\\' ) + { + if( *(in+1)!= L'\n') + { + out.push_back(*in); + } + } + else + { + out.push_back(*in); + } + } + return out; +} + + +/* Decode an item via the fish 1.x format. Adapted from fish 1.x's item_get(). */ +history_item_t history_t::decode_item_fish_1_x(const char *begin, size_t length) { + + const char *end = begin + length; + const char *pos=begin; + + bool was_backslash = 0; + wcstring out; + bool first_char = true; + bool timestamp_mode = false; + time_t timestamp = 0; + while( 1 ) + { + wchar_t c; + mbstate_t state; + size_t res; + + memset( &state, 0, sizeof(state) ); + + res = mbrtowc( &c, pos, end-pos, &state ); + + if( res == (size_t)-1 ) + { + pos++; + continue; + } + else if( res == (size_t)-2 ) + { + break; + } + else if( res == (size_t)0 ) + { + pos++; + continue; + } + pos += res; + + if( c == L'\n' ) + { + if( timestamp_mode ) + { + const wchar_t *time_string = out.c_str(); + while( *time_string && !iswdigit(*time_string)) + time_string++; + errno=0; + + if( *time_string ) + { + time_t tm; + wchar_t *end; + + errno = 0; + tm = (time_t)wcstol( time_string, &end, 10 ); + + if( tm && !errno && !*end ) + { + timestamp = tm; + } + + } + + out.clear(); + timestamp_mode = false; + continue; + } + if( !was_backslash ) + break; + } + + if( first_char ) + { + if( c == L'#' ) + timestamp_mode = true; + } + + first_char = false; + + out.push_back(c); + + was_backslash = ( (c == L'\\') && !was_backslash); + + } + + out = history_unescape_newlines_fish_1_x(out); + return history_item_t(out, timestamp); +} + + +/* Try to infer the history file type based on inspecting the data */ +static history_file_type_t infer_file_type(const char *data, size_t len) { + history_file_type_t result = history_type_unknown; + if (len > 0) { + /* Old fish started with a # */ + if (data[0] == '#') { + result = history_type_fish_1_x; + } else { + /* Assume new fish */ + result = history_type_fish_2_0; + } + } + return result; } void history_t::populate_from_mmap(void) { + mmap_type = infer_file_type(mmap_start, mmap_length); size_t cursor = 0; for (;;) { - size_t offset = offset_of_next_item(mmap_start, mmap_length, &cursor, birth_timestamp); + size_t offset = offset_of_next_item(mmap_start, mmap_length, mmap_type, &cursor, birth_timestamp); // If we get back -1, we're done if (offset == (size_t)(-1)) break; @@ -831,15 +1040,16 @@ void history_t::save_internal() const char *local_mmap_start = NULL; size_t local_mmap_size = 0; if (map_file(name, &local_mmap_start, &local_mmap_size)) { + const history_file_type_t local_mmap_type = infer_file_type(local_mmap_start, local_mmap_size); size_t cursor = 0; for (;;) { - size_t offset = offset_of_next_item(local_mmap_start, local_mmap_size, &cursor, 0); + size_t offset = offset_of_next_item(local_mmap_start, local_mmap_size, local_mmap_type, &cursor, 0); /* If we get back -1, we're done */ if (offset == (size_t)(-1)) break; /* Try decoding an old item */ - const history_item_t old_item = history_t::decode_item(local_mmap_start + offset, local_mmap_size - offset); + const history_item_t old_item = history_t::decode_item(local_mmap_start + offset, local_mmap_size - offset, local_mmap_type); if (old_item.empty() || is_deleted(old_item)) { // debug(0, L"Item is deleted : %s\n", old_item.str().c_str()); @@ -913,12 +1123,14 @@ void history_t::save_internal() } } -void history_t::save(void) { +void history_t::save(void) +{ scoped_lock locker(lock); this->save_internal(); } -void history_t::clear(void) { +void history_t::clear(void) +{ scoped_lock locker(lock); new_items.clear(); deleted_items.clear(); @@ -931,6 +1143,63 @@ void history_t::clear(void) { } +/* Indicate whether we ought to import the bash history file into fish */ +static bool should_import_bash_history_line(const std::string &line) +{ + if (line.empty()) + return false; + + /* Very naive tests! Skip export; probably should skip others. */ + const char * const ignore_prefixes[] = { + "export ", + "#" + }; + + for (size_t i=0; i < sizeof ignore_prefixes / sizeof *ignore_prefixes; i++) { + const char *prefix = ignore_prefixes[i]; + if (! line.compare(0, strlen(prefix), prefix)) { + return false; + } + } + printf("Importing %s\n", line.c_str()); + return true; +} + +void history_t::populate_from_bash(FILE *stream) +{ + /* Bash's format is very simple: just lines with #s for comments. + Ignore a few commands that are bash-specific. This list ought to be expanded. + */ + std::string line; + for (;;) { + line.clear(); + bool success = false, has_newline = false; + + /* Loop until we've read a line */ + do { + char buff[128]; + success = !! fgets(buff, sizeof buff, stream); + if (success) { + /* Skip the newline */ + char *newline = strchr(buff, '\n'); + if (newline) *newline = '\0'; + has_newline = (newline != NULL); + + /* Append what we've got */ + line.append(buff); + } + } while (success && ! has_newline); + + /* Maybe add this line */ + if (should_import_bash_history_line(line)) { + this->add(str2wcstring(line)); + } + + if (line.empty()) + break; + } +} + void history_init() { } @@ -1041,10 +1310,5 @@ void history_t::add_with_file_detection(const wcstring &str) bool history_t::is_deleted(const history_item_t &item) const { - for (std::vector::const_iterator iter = deleted_items.begin(); iter != deleted_items.end(); ++iter) - { - if (iter->match_contents(item)) { return true; } - } - - return false; + return deleted_items.count(item.str()) > 0; } diff --git a/history.h b/history.h index 4b146c1f6..b25d32176 100644 --- a/history.h +++ b/history.h @@ -57,20 +57,18 @@ class history_item_t { const path_list_t &get_required_paths() const { return required_paths; } - bool write_to_file(FILE *f) const; - bool operator==(const history_item_t &other) const { return contents == other.contents && creation_timestamp == other.creation_timestamp && required_paths == other.required_paths; } +}; - bool match_contents(const history_item_t &other) const { - return contents == other.contents; - } - - /* Functions for testing only */ - +/* The type of file that we mmap'd */ +enum history_file_type_t { + history_type_unknown, + history_type_fish_2_0, + history_type_fish_1_x }; class history_t { @@ -101,8 +99,8 @@ private: /** New items. */ std::vector new_items; - /** Deleted items. */ - std::vector deleted_items; + /** Deleted item contents. */ + std::set deleted_items; /** How many items we've added without saving */ size_t unsaved_item_count; @@ -110,17 +108,18 @@ private: /** The mmaped region for the history file */ const char *mmap_start; - /** The size of the mmaped region */ + /** The size of the mmap'd region */ size_t mmap_length; + /** The type of file we mmap'd */ + history_file_type_t mmap_type; + /** Timestamp of when this history was created */ const time_t birth_timestamp; /** Timestamp of last save */ time_t save_timestamp; - static history_item_t decode_item(const char *ptr, size_t len); - void populate_from_mmap(void); /** List of old items, as offsets into out mmap data */ @@ -137,6 +136,11 @@ private: /** Saves history */ void save_internal(); + + /* Versioned decoding */ + static history_item_t decode_item_fish_2_0(const char *base, size_t len); + static history_item_t decode_item_fish_1_x(const char *base, size_t len); + static history_item_t decode_item(const char *base, size_t len, history_file_type_t type); public: /** Returns history with the given name, creating it if necessary */ @@ -157,12 +161,15 @@ public: /** Irreversibly clears history */ void clear(); + /** Populates from a bash history file */ + void populate_from_bash(FILE *f); + /* Gets all the history into a string with ARRAY_SEP_STR. This is intended for the $history environment variable. This may be long! */ void get_string_representation(wcstring &str, const wcstring &separator); /** Return the specified history at the specified index. 0 is the index of the current commandline. (So the most recent item is at index 1.) */ history_item_t item_at_index(size_t idx); - + bool is_deleted(const history_item_t &item) const; }; diff --git a/tests/history_sample_bash b/tests/history_sample_bash new file mode 100644 index 000000000..0be128a00 --- /dev/null +++ b/tests/history_sample_bash @@ -0,0 +1,7 @@ +echo foo +history --help +#1339718290 +export HISTTIMEFORMAT='%F %T ' +#1339718298 +echo supsup +#abcde diff --git a/tests/history_sample_fish_1_x b/tests/history_sample_fish_1_x new file mode 100644 index 000000000..dd09d4cb6 --- /dev/null +++ b/tests/history_sample_fish_1_x @@ -0,0 +1,12 @@ +# 1339519901 +ls / +# 1339519903 +cd foobar +# 1339519906 +function yay\ +echo hi\ +end +# 1339520882 +echo #abc +# 1339520884 +#def diff --git a/tests/history_sample_fish_2_0 b/tests/history_sample_fish_2_0 new file mode 100644 index 000000000..f44bbfc0c --- /dev/null +++ b/tests/history_sample_fish_2_0 @@ -0,0 +1,6 @@ +- cmd: echo alpha + when: 1339717374 +- cmd: function foo\necho bar\nend + when: 1339717377 +- cmd: echo this has\\\nbackslashes + when: 1339717385