Work on new history implementation

builtin.cpp

@@ -2151,7 +2151,7 @@ static int builtin_read( parser_t &parser, wchar_t **argv )
 		reader_set_prompt( prompt );
 		if( shell )
 		{
-			reader_set_complete_function( &complete2 );
+			reader_set_complete_function( &complete );
 			reader_set_highlight_function( &highlight_shell );
 			reader_set_test_function( &reader_shell_test );
 		}

builtin_complete.cpp

@@ -557,7 +557,7 @@ static int builtin_complete( parser_t &parser, wchar_t **argv )
 			
 //				comp = al_halloc( 0 );
 			
-				complete2( do_complete, comp );
+				complete( do_complete, comp );
 			
 				for( size_t i=0; i< comp.size() ; i++ )
 				{

complete.cpp

@@ -1074,7 +1074,7 @@ static void complete_cmd( const wchar_t *cmd,
 		if( use_command )
 		{
 			
-			if( expand_string2(cmd, comp, ACCEPT_INCOMPLETE | EXECUTABLES_ONLY ) != EXPAND_ERROR )
+			if( expand_string(cmd, comp, ACCEPT_INCOMPLETE | EXECUTABLES_ONLY ) != EXPAND_ERROR )
 			{
 				complete_cmd_desc( cmd, comp );
 			}
@@ -1113,7 +1113,7 @@ static void complete_cmd( const wchar_t *cmd,
 					
 					prev_count =  comp.size() ;
 					
-					if( expand_string2(
+					if( expand_string(
 									   nxt_completion,
 									   comp,
 									   ACCEPT_INCOMPLETE |
@@ -1180,7 +1180,7 @@ static void complete_cmd( const wchar_t *cmd,
 					continue;
 				}
 			
-				if( expand_string2(		   nxt_completion,
+				if( expand_string(		   nxt_completion,
 								   comp,
 								   ACCEPT_INCOMPLETE | DIRECTORIES_ONLY ) != EXPAND_ERROR )
 				{
@@ -1563,7 +1563,7 @@ static void complete_param_expand( wchar_t *str,
 		ACCEPT_INCOMPLETE | 
 		(do_file?0:EXPAND_SKIP_WILDCARDS);
 	
-	if( expand_string2( 		wcsdup(comp_str),
+	if( expand_string( 		wcsdup(comp_str),
 					   comp_out,
 					   flags ) == EXPAND_ERROR )
 	{
@@ -1755,7 +1755,7 @@ static int try_complete_user( const wchar_t *cmd,
 	return res;
 }
 
-void complete2( const wchar_t *cmd,
+void complete( const wchar_t *cmd,
 			   std::vector<completion_t> &comp )
 {
 	wchar_t *tok_begin, *tok_end, *cmdsubst_begin, *cmdsubst_end, *prev_begin, *prev_end;

complete.h

@@ -219,18 +219,9 @@ void complete_remove( const wchar_t *cmd,
 		      const wchar_t *long_opt );
 
 /**
-  Find all completions of the command cmd, insert them into out. The
+  Find all completions of the command cmd, insert them into out. 
-  caller must free the variables returned in out.  The results are
-  returned in the array_list_t 'out', in the format of wide character
-  strings, with each element consisting of a suggested completion and
-  a description of what kind of object this completion represents,
-  separated by a separator of type COMPLETE_SEP.
-
-  Values returned by this function should be freed by the caller.
 */
-//void complete( const wchar_t *cmd, array_list_t *out );
+void complete( const wchar_t* cmd, std::vector<completion_t> &out);
-
-void complete2( const wchar_t* cmd, std::vector<completion_t> &out);
 
 /**
    Print a list of all current completions into the string_buffer_t. 

env.cpp

@@ -190,10 +190,10 @@ static buffer_t export_buffer;
 static int has_changed = 1;
 
 /**
-   This stringbuffer is used to store the value of dynamically
+   This string is used to store the value of dynamically
    generated variables, such as history.
 */
-static string_buffer_t dyn_var;
+static wcstring dyn_var;
 
 /**
    Variable used by env_get_names to communicate auxiliary information
@@ -531,7 +531,6 @@ void env_init()
 	wchar_t *uname;
 	wchar_t *version;
 	
-	sb_init( &dyn_var );
 	b_init( &export_buffer );
 	
 	/*
@@ -700,8 +699,6 @@ void env_destroy()
 {
 	env_universal_destroy();
 	
-	sb_destroy( &dyn_var );
-
 	b_destroy( &export_buffer );
 	
 	while( &top->env != global )
@@ -1160,7 +1157,7 @@ env_var_t env_get_string( const wchar_t *key )
 		for( i=add_current;; i++ )
 		{
             // PCA This looks bad!
-			wchar_t *next = history_get( i-add_current );
+			wchar_t *next = NULL;//history_get( i-add_current );
 			if( !next )
 			{
 				break;
@@ -1242,7 +1239,7 @@ env_var_t env_get_string( const wchar_t *key )
     }
 }
 
-wchar_t *env_get( const wchar_t *key )
+const wchar_t *env_get( const wchar_t *key )
 {
     ASSERT_IS_MAIN_THREAD();
 
@@ -1255,58 +1252,52 @@ wchar_t *env_get( const wchar_t *key )
 	if( wcscmp( key, L"history" ) == 0 )
 	{
 		wchar_t *current;
-		int i;		
+		dyn_var.clear();
-		int add_current=0;
-		sb_clear( &dyn_var );						
 		
 		current = reader_get_buffer();
 		if( current && wcslen( current ) )
 		{
-			add_current=1;
+            dyn_var.append(current);
-			sb_append( &dyn_var, current );
+            dyn_var.append(ARRAY_SEP_STR);
-		}
+        }
-		
+        
-		for( i=add_current;; i++ )
+        history_t *history = reader_get_history();
-		{
+        if (history) {
-			wchar_t *next = history_get( i-add_current );
+            for (size_t idx = 1; idx < (size_t)(-1); idx++) {
-			if( !next )
+                history_item_t item = history->item_at_index(idx);
-			{
+                if (item.empty()) break;
-				break;
+                
-			}
+                dyn_var.append(item.str());
-			
+                dyn_var.append(ARRAY_SEP_STR);
-			if( i!=0)
+            }
-			{
+        }
-				sb_append( &dyn_var, ARRAY_SEP_STR );
+        
-			}
+        /* We always have a trailing ARRAY_SEP_STR; get rid of it */
-
+        if (dyn_var.size() >= wcslen(ARRAY_SEP_STR)) {
-			sb_append( &dyn_var, next );
+            dyn_var.resize(dyn_var.size() - wcslen(ARRAY_SEP_STR));
-		}
+        }
-
+        
-		return (wchar_t *)dyn_var.buff;
+		return dyn_var.c_str();
 	}
 	else if( wcscmp( key, L"COLUMNS" )==0 )
 	{
-		sb_clear( &dyn_var );						
+        dyn_var = to_string<int>(common_get_width());
-		sb_printf( &dyn_var, L"%d", common_get_width() );		
+		return dyn_var.c_str();		
-		return (wchar_t *)dyn_var.buff;		
 	}	
 	else if( wcscmp( key, L"LINES" )==0 )
 	{
-		sb_clear( &dyn_var );						
+        dyn_var = to_string<int>(common_get_height());
-		sb_printf( &dyn_var, L"%d", common_get_height() );		
+		return dyn_var.c_str();
-		return (wchar_t *)dyn_var.buff;
 	}
 	else if( wcscmp( key, L"status" )==0 )
 	{
-		sb_clear( &dyn_var );			
+        dyn_var = to_string<int>(proc_get_last_status());
-		sb_printf( &dyn_var, L"%d", proc_get_last_status() );		
+		return dyn_var.c_str();
-		return (wchar_t *)dyn_var.buff;		
 	}
 	else if( wcscmp( key, L"umask" )==0 )
 	{
-		sb_clear( &dyn_var );			
+        dyn_var = format_string(L"0%0.3o", get_umask());
-		sb_printf( &dyn_var, L"0%0.3o", get_umask() );		
+		return dyn_var.c_str();
-		return (wchar_t *)dyn_var.buff;		
 	}
 	
 	while( env != 0 )

env.h

@@ -93,7 +93,7 @@ int env_set( const wchar_t *key,
   valid until the next call to env_get(), env_set(), env_push() or
   env_pop() takes place.
 */
-wchar_t *env_get( const wchar_t *key );
+const wchar_t *env_get( const wchar_t *key );
 
 class env_var_t : public wcstring {
 private:

expand.cpp

@@ -1506,7 +1506,7 @@ static void remove_internal_separator2( wcstring &s, int conv )
 }
 
 
-int expand_string2( const wcstring &input, std::vector<completion_t> &output, int flags )
+int expand_string( const wcstring &input, std::vector<completion_t> &output, int flags )
 {
 	parser_t parser(PARSER_TYPE_ERRORS_ONLY);
    	std::vector<completion_t> list1, list2;
@@ -1725,7 +1725,7 @@ bool expand_one(wcstring &string, int flags) {
         return true;
 	}
 	
-    if (expand_string2(string, completions, flags)) {
+    if (expand_string(string, completions, flags)) {
         if (completions.size() == 1) {
             string = completions.at(0).completion;
             result = true;

expand.h

@@ -138,18 +138,12 @@ class parser_t;
    out. If expansion is performed, the original parameter is freed and
    newly allocated strings are inserted into the list out.
 
-   If \c context is non-null, all the strings contained in the
+   \param input The parameter to expand
-   array_list_t \c out will be registered to be free'd when context is
+   \param output The list to which the result will be appended.
-   free'd.
- 
-   \param context the halloc context to use for automatic deallocation
-   \param in The parameter to expand
    \param flag Specifies if any expansion pass should be skipped. Legal values are any combination of EXPAND_SKIP_CMDSUBST EXPAND_SKIP_VARIABLES and EXPAND_SKIP_WILDCARDS
-   \param out The list to which the result will be appended.
    \return One of EXPAND_OK, EXPAND_ERROR, EXPAND_WILDCARD_MATCH and EXPAND_WILDCARD_NO_MATCH. EXPAND_WILDCARD_NO_MATCH and EXPAND_WILDCARD_MATCH are normal exit conditions used only on strings containing wildcards to tell if the wildcard produced any matches.
 */
-__warn_unused int expand_string( void *context, wchar_t *in, array_list_t *out, int flag );
+__warn_unused int expand_string( const wcstring &input, std::vector<completion_t> &output, int flag );
-__warn_unused int expand_string2( const wcstring &input, std::vector<completion_t> &output, int flag );
 
 
 /**

fish_tests.cpp

@@ -51,6 +51,7 @@
 #include "path.h"
 #include "halloc.h"
 #include "halloc_util.h"
+#include "history.h"
 
 /**
    The number of tests to run
@@ -679,14 +680,13 @@ static void test_lru(void) {
 
 static int expand_test( const wchar_t *in, int flags, ... )
 {
-	array_list_t out;	
+	std::vector<completion_t> output;
 	va_list va;
-	int i=0;
+	size_t i=0;
 	int res=1;
 	wchar_t *arg;
 	
-	al_init( &out );
+	if( expand_string( in, output, flags) )
-	if( expand_string( 0, wcsdup(in), &out, flags) )
 	{
 		
 	}
@@ -696,13 +696,13 @@ static int expand_test( const wchar_t *in, int flags, ... )
 
 	while( (arg=va_arg(va, wchar_t *) )!= 0 ) 
 	{
-		if( al_get_count( &out ) == i )
+		if( output.size() == i )
 		{
 			res=0;
 			break;
 		}
 		
-		if( wcscmp( (wchar_t *)al_get( &out, i ),arg) != 0 )
+        if (output.at(i).completion != arg)
 		{
 			res=0;
 			break;
@@ -712,7 +712,6 @@ static int expand_test( const wchar_t *in, int flags, ... )
 	}
 	va_end( va );
 	
-	al_foreach( &out, &free );
 	return res;
 		
 }
@@ -770,7 +769,7 @@ static void test_path()
 void perf_complete()
 {
 	wchar_t c;
-	array_list_t out;
+	std::vector<completion_t> out;
 	long long t1, t2;
 	int matches=0;
 	double t;
@@ -783,7 +782,6 @@ void perf_complete()
 	
 	
 	say( L"Testing completion performance" );
-	al_init( &out );
 	
 	reader_push(L"");
 	say( L"Here we go" );
@@ -796,12 +794,10 @@ void perf_complete()
 		str[0]=c;
 		reader_set_buffer( str, 0 );
 		
-		complete( str, &out );
+		complete( str, out );
 	
-		matches += al_get_count( &out );
+		matches += out.size();
-		
+        out.clear();
-		al_foreach( &out, &free );
-		al_truncate( &out, 0 );
 	}
 	t2=get_time();
 	
@@ -818,12 +814,10 @@ void perf_complete()
 		
 		reader_set_buffer( str, 0 );
 		
-		complete( str, &out );
+		complete( str, out );
 	
-		matches += al_get_count( &out );
+		matches += out.size();
-		
+        out.clear();
-		al_foreach( &out, &free );
-		al_truncate( &out, 0 );
 	}
 	t2=get_time();
 	
@@ -831,12 +825,42 @@ void perf_complete()
 	
 	say( L"Two letter command completion took %f seconds per completion, %f microseconds/match", t, (double)(t2-t1)/matches );
 	
-	al_destroy( &out );
-
 	reader_pop();
 	
 }
 
+static void test_history_matches(history_search_t &search, size_t matches) {
+    size_t i;
+    for (i=0; i < matches; i++) {
+        assert(search.go_backwards());
+    }
+    assert(! search.go_backwards());
+    
+    for (i=1; i < matches; i++) {
+        assert(search.go_forwards());
+    }
+    assert(! search.go_forwards());
+}
+
+static void test_history(void) {
+    say( L"Testing history");
+    
+    history_t &history = history_t::history_with_name(L"test_history");
+    history.add(L"Alpha");
+    history.add(L"Beta");
+    history.add(L"Gamma");
+
+    /* All three items match "a" */
+    history_search_t search1(history, L"a");
+    test_history_matches(search1, 3);
+    assert(search1.current_item().str() == L"Alpha");
+    
+    /* One item matches "et" */
+    history_search_t search2(history, L"et");
+    test_history_matches(search2, 1);
+    assert(search2.current_item().str() == L"Beta");
+}
+
 
 
 
@@ -869,6 +893,7 @@ int main( int argc, char **argv )
 	test_lru();
 	test_expand();
 	test_path();
+    test_history();
 	
 	say( L"Encountered %d errors in low-level tests", err_count );
 

history.cpp

@@ -28,7 +28,327 @@
 #include "intern.h"
 #include "path.h"
 #include "signal.h"
+#include <map>
 
+static pthread_mutex_t hist_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static std::map<wcstring, history_t *> histories;
+
+static wcstring history_filename(const wcstring &name, const wcstring &suffix);
+
+static hash_table_t *mode_table=0;
+
+/* Unescapes newlines in-place */
+static void unescape_newlines(wcstring &str);
+
+/* Custom deleter for our shared_ptr */
+class history_item_data_deleter_t {
+    private:
+        const bool free_it;
+    public:
+    history_item_data_deleter_t(bool flag) : free_it(flag) { }
+    void operator()(const wchar_t *data) {
+        if (free_it)
+            free((void *)data);
+    }
+};
+
+history_item_t::history_item_t(const wcstring &str) : contents(str), timestamp(time(NULL))
+{
+}
+
+history_item_t::history_item_t(const wcstring &str, time_t when) : contents(str), timestamp(when)
+{
+}
+
+
+
+history_t & history_t::history_with_name(const wcstring &name) {
+    /* Note that histories are currently never deleted, so we can return a reference to them without using something like shared_ptr */
+    scoped_lock locker(hist_lock);
+    history_t *& current = histories[name];
+    if (current == NULL)
+        current = new history_t(name);
+    return *current;
+}
+
+history_t::history_t(const wcstring &pname) :
+    name(pname),
+    mmap_start(NULL),
+    mmap_length(0),
+    save_timestamp(0),
+    loaded_old(false)
+{
+    pthread_mutex_init(&lock, NULL);
+}
+
+history_t::~history_t()
+{
+    pthread_mutex_destroy(&lock);
+}
+
+void history_t::add(const wcstring &str)
+{
+    scoped_lock locker(lock);
+    new_items.push_back(history_item_t(str.c_str(), true));
+}
+
+history_item_t history_t::item_at_index(size_t idx) {
+    scoped_lock locker(lock);
+    
+    /* 0 is considered an invalid index */
+    assert(idx > 0);
+    idx--;
+    
+    /* idx=0 corresponds to last item in new_items */
+    size_t new_item_count = new_items.size();
+    if (idx < new_item_count) {
+        return new_items.at(new_item_count - idx - 1);
+    }
+    
+    /* Now look in our old items */
+    idx -= new_item_count;
+    load_old_if_needed();
+    size_t old_item_count = old_item_offsets.size();
+    if (idx < old_item_count) {
+        /* idx=0 corresponds to last item in old_item_offsets */
+        size_t offset = old_item_offsets.at(old_item_count - idx - 1);
+        return history_t::decode_item(mmap_start + offset, mmap_length - offset);
+    }
+    
+    /* Index past the valid range, so return an empty history item */
+    return history_item_t(wcstring(), 0);
+}
+
+history_item_t history_t::decode_item(const char *begin, size_t len)
+{
+    const char *pos = begin, *end = begin + len;
+    
+    int was_backslash = 0;
+    
+    wcstring output;
+    time_t timestamp = 0;
+    
+    int first_char = 1;	
+    bool timestamp_mode = false;
+    
+    while( 1 )
+    {
+        wchar_t c;
+        mbstate_t state;
+        bzero(&state, sizeof state);
+        
+        size_t res;
+        
+        res = mbrtowc( &c, pos, end-pos, &state );
+        
+        if( res == (size_t)-1 )
+        {
+            pos++;
+            continue;
+        }
+        else if( res == (size_t)-2 )
+        {
+            break;
+        }
+        else if( res == (size_t)0 )
+        {
+            pos++;
+            continue;
+        }
+        pos += res;
+        
+        if( c == L'\n' )
+        {
+            if( timestamp_mode )
+            {
+                const wchar_t *time_string = output.c_str();
+                while( *time_string && !iswdigit(*time_string))
+                    time_string++;
+                errno=0;
+                
+                if( *time_string )
+                {
+                    time_t tm;
+                    wchar_t *end;
+                    
+                    errno = 0;
+                    tm = (time_t)wcstol( time_string, &end, 10 );
+                    
+                    if( tm && !errno && !*end )
+                    {
+                        timestamp = tm;
+                    }
+                    
+                }					
+                
+                output.clear();
+                timestamp_mode = 0;
+                continue;
+            }
+            if( !was_backslash )
+                break;
+        }
+        
+        if( first_char )
+        {
+            if( c == L'#' ) 
+                timestamp_mode = 1;
+        }
+        
+        first_char = 0;
+        
+        output.push_back(c);
+        
+        was_backslash = ( (c == L'\\') && !was_backslash);
+        
+    }
+    
+    unescape_newlines(output);
+    return history_item_t(output, timestamp);
+}
+
+void history_t::populate_from_mmap(void)
+{
+	const char *begin = mmap_start;
+	const char *end = begin + mmap_length;
+	const char *pos;
+	
+	int ignore_newline = 0;
+	int do_push = 1;
+	
+	for( pos = begin; pos <end; pos++ )
+	{
+		
+		if( do_push )
+		{
+			ignore_newline = *pos == '#';
+            /* Need to unique-ize */
+            old_item_offsets.push_back(pos - begin);
+			do_push = 0;
+		}
+		
+		switch( *pos )
+		{
+			case '\\':
+			{
+				pos++;
+				break;
+			}
+			
+			case '\n':
+			{
+				if( ignore_newline )
+				{
+					ignore_newline = 0;
+				}
+				else
+				{
+					do_push = 1;
+				}				
+				break;
+			}
+		}
+	}	
+}
+
+void history_t::load_old_if_needed(void)
+{
+    if (loaded_old) return;
+    loaded_old = true;
+    
+	int fd;
+	int ok=0;
+	
+	signal_block();
+    wcstring filename = history_filename(name, L"");
+	
+	if( ! filename.empty() )
+	{
+		if( ( fd = wopen( filename.c_str(), O_RDONLY ) ) > 0 )
+		{
+			off_t len = lseek( fd, 0, SEEK_END );
+			if( len != (off_t)-1)
+			{
+				mmap_length = (size_t)len;
+				if( lseek( fd, 0, SEEK_SET ) == 0 )
+				{
+					if( (mmap_start = (char *)mmap( 0, mmap_length, PROT_READ, MAP_PRIVATE, fd, 0 )) != MAP_FAILED )
+					{
+						ok = 1;
+						this->populate_from_mmap();
+					}
+				}
+			}
+			close( fd );
+		}
+	}
+	signal_unblock();
+}
+
+bool history_search_t::go_forwards() {
+    /* Forwards means reducing our index. */
+    if (idx == 0)
+        return false;
+    
+    size_t i;
+    for (i=idx-1; i > 0; i--) {
+        const history_item_t item = history->item_at_index(i);
+        /* Skip if it's empty. Empty items only occur at the end of the history. */
+        if (item.empty())
+            continue;
+        
+        /* Look for term in item.data */
+        if (item.matches_search(term)) {
+            idx = i;
+            break;
+        }
+    }
+    return i > 0;
+}
+
+bool history_search_t::go_backwards() {
+    /* Backwards means increasing our index */
+    const size_t max_idx = (size_t)(-1);
+    if (idx == max_idx)
+        return false;
+    
+    size_t i;
+    for (i=idx+1;; i++) {
+        /* We're done if we reach the largest index */
+        if (i == max_idx)
+            return false;
+            
+        const history_item_t item = history->item_at_index(i);
+        /* We're done if it's empty */
+        if (item.empty()) {
+            return false;
+        }
+
+        /* Look for term in item.data */
+        if (item.matches_search(term)) {
+            idx = i;
+            return true;
+        }
+    }
+    return false;
+}
+
+/** Goes to the end (forwards) */
+void history_search_t::go_to_end(void) {
+    idx = 0;
+}
+
+/** Goes to the beginning (backwards) */
+void history_search_t::go_to_beginning(void) {
+    idx = (size_t)(-1);
+}
+
+
+history_item_t history_search_t::current_item() const {
+    assert(idx > 0 && idx < (size_t)(-1));
+    return history->item_at_index(idx);
+}
 
 /**
    Interval in seconds between automatic history save
@@ -125,13 +445,8 @@ typedef struct
 	   Original creation time for the entry
 	*/
 	time_t timestamp;
-}
+} item_t;
-	item_t;
 
-/**
-   Table of all history modes
-*/
-static hash_table_t *mode_table=0;
 
 /**
    The surrent history mode
@@ -214,6 +529,19 @@ static wchar_t *history_escape_newlines( wchar_t *in )
 	return (wchar_t *)out->buff;
 }
 
+static void unescape_newlines(wcstring &str)
+{
+    /* Replace instances of backslash + newline with just the newline */
+    const wchar_t *needle = L"\\\n", *replacement = L"\n";
+    size_t needle_len = wcslen(needle);
+    size_t offset = 0;
+    while((offset = str.find(needle, offset)) != wcstring::npos)
+    {
+        str.replace(offset, needle_len, replacement);
+        offset += needle_len;
+    }
+}
+
 /**
    Remove backslashes from all newlines. This makes a string from the
    history file better formated for on screen display. The memory for
@@ -479,6 +807,20 @@ static wchar_t *history_filename( void *context, const wchar_t *name, const wcha
 	return res;
 }
 
+static wcstring history_filename(const wcstring &name, const wcstring &suffix)
+{
+    wcstring path;
+    if (! path_get_config(path))
+        return L"";
+    
+    wcstring result = path;
+    result.append(L"/");
+    result.append(name);
+    result.append(L"_history");
+    result.append(suffix);
+    return result;
+}
+
 /**
    Go through the mmaped region and insert pointers to suitable loacations into the item list
 */
@@ -607,6 +949,131 @@ static void history_load( history_mode_t *m )
 	signal_unblock();
 }
 
+#if 0
+/**
+   Save the specified mode to file
+*/
+void history_t::save( void *n, history_mode_t *m )
+{
+    scoped_lock locker(lock);
+    
+    /* Nothing to do if there's no new items */
+    if (new_items.empty())
+        return;
+    
+	FILE *out;
+	int i;
+	int has_new=0;
+	wchar_t *tmp_name;
+
+	int ok = 1;
+	
+	signal_block();
+    
+    wcstring tmp_name = history_filename(name, L".tmp");
+	
+	if( ! tmp_name.empty() )
+	{
+		if( (out=wfopen( tmp_name, "w" ) ) )
+		{
+			hash_table_t mine;
+			
+			hash_init( &mine, &hash_item_func, &hash_item_cmp );
+			
+			for( i=0; i<al_get_count(&m->item); i++ )
+			{
+				void *ptr = al_get( &m->item, i );
+				int is_new = item_is_new( m, ptr );
+				if( is_new )
+				{
+					hash_put( &mine, item_get( m, ptr ), L"" );
+				}
+			}
+			
+			/*
+			  Re-save the old history
+			*/
+			for( i=0; ok && (i<al_get_count(&on_disk->item)); i++ )
+			{
+				void *ptr = al_get( &on_disk->item, i );
+				item_t *i = item_get( on_disk, ptr );
+				if( !hash_get( &mine, i ) )
+				{
+					if( item_write( out, on_disk, ptr ) == -1 )
+					{
+						ok = 0;
+						break;
+					}
+				}
+				
+			}
+			
+			hash_destroy( &mine );
+		
+			/*
+			  Add our own items last
+			*/		
+			for( i=0; ok && (i<al_get_count(&m->item)); i++ )
+			{
+				void *ptr = al_get( &m->item, i );
+				int is_new = item_is_new( m, ptr );
+				if( is_new )
+				{
+					if( item_write( out, m, ptr ) == -1 )
+					{						
+						ok = 0;
+					}
+				}
+			}
+			
+			if( fclose( out ) || !ok )
+			{
+				/*
+				  This message does not have high enough priority to
+				  be shown by default.
+				*/
+				debug( 2, L"Error when writing history file" );
+			}
+			else
+			{
+				wrename( tmp_name, history_filename( on_disk, m->name, 0 ) );
+			}
+		}
+		free( tmp_name );
+	}	
+	
+	halloc_free( on_disk);
+
+	if( ok )
+	{
+
+		/*
+		  Reset the history. The item_t entries created in this session
+		  are not lost or dropped, they are stored in the session_item
+		  hash table. On reload, they will be automatically inserted at
+		  the end of the history list.
+		*/
+			
+		if( m->mmap_start && (m->mmap_start != MAP_FAILED ) )
+		{
+			munmap( m->mmap_start, m->mmap_length );
+		}
+		
+		al_truncate( &m->item, 0 );
+		al_truncate( &m->used, 0 );
+		m->pos = 0;
+		m->has_loaded = 0;
+		m->mmap_start=0;
+		m->mmap_length=0;
+	
+		m->save_timestamp=time(0);
+		m->new_count = 0;
+	}
+	
+	signal_unblock();
+}
+#endif
+
 /**
    Save the specified mode to file
 */
@@ -951,23 +1418,6 @@ const wchar_t *history_next_match( const wchar_t *needle)
 }
 
 
-void history_set_mode( const wchar_t *name )
-{
-	if( !mode_table )
-	{
-		mode_table = (hash_table_t *)malloc( sizeof(hash_table_t ));
-		hash_init( mode_table, &hash_wcs_func, &hash_wcs_cmp );		
-	}
-	
-	current_mode = (history_mode_t *)hash_get( mode_table, name );
-	
-	if( !current_mode )
-	{
-		current_mode = history_create_mode( name );
-		hash_put( mode_table, name, current_mode );		
-	}	
-}
-
 void history_init()
 {
 }

history.h

@@ -6,6 +6,137 @@
 #define FISH_HISTORY_H
 
 #include <wchar.h>
+#include "common.h"
+#include "pthread.h"
+#include <vector>
+#include <tr1/memory>
+using std::tr1::shared_ptr;
+
+class history_item_t {
+    friend class history_t;
+
+    private:
+    history_item_t(const wcstring &);
+    history_item_t(const wcstring &, time_t);
+
+	/** The actual contents of the entry */
+	wcstring contents;
+    	
+	/** Original creation time for the entry */
+	time_t timestamp;
+    
+    public:
+    const wcstring &str() const { return contents; }
+    bool empty() const { return contents.empty(); }
+    bool matches_search(const wcstring &val) const { return contents.find(val) != wcstring::npos; }
+};
+
+class history_t {
+private:
+    /** No copying */
+    history_t(const history_t&);
+    history_t &operator=(const history_t&);
+
+    /** Private creator */
+    history_t(const wcstring &pname);
+    
+    /** Destructor */
+    ~history_t();
+
+    /** Lock for thread safety */
+    pthread_mutex_t lock;
+
+	/** The name of this list. Used for picking a suitable filename and for switching modes. */
+	const wcstring name;
+	
+	/** New items. */
+	std::vector<history_item_t> new_items;
+    
+	/** The mmaped region for the history file */
+	const char *mmap_start;
+
+	/** The size of the mmaped region */
+	size_t mmap_length;
+
+	/**
+	   Timestamp of last save
+	*/
+	time_t save_timestamp;
+    
+    static history_item_t decode_item(const char *ptr, size_t len);
+    
+    void populate_from_mmap(void);
+    
+    /** List of old items, as offsets into out mmap data */
+    std::vector<size_t> old_item_offsets;
+    
+    /** Whether we've loaded old items */
+    bool loaded_old;
+    
+    /** Loads old if necessary */
+    void load_old_if_needed(void);
+    
+public:
+    /** Returns history with the given name, creating it if necessary */
+    static history_t & history_with_name(const wcstring &name);
+    
+    /** Add a new history item to the end */
+    void add(const wcstring &str);
+    
+    /** Saves history */
+    void save();
+    
+    /** Return the specified history at the specified index. 0 is the index of the current commandline. */
+    history_item_t item_at_index(size_t idx);
+};
+
+class history_search_t {
+
+    /** The history in which we are searching */
+    history_t * history;
+
+    /** The search term */
+    wcstring term;
+    
+    /** Current index into the history */
+    size_t idx;
+        
+public:
+
+    /** Finds the next search term (forwards in time). Returns true if one was found. */
+    bool go_forwards(void);
+    
+    /** Finds the previous search result (backwards in time). Returns true if one was found. */
+    bool go_backwards(void);
+    
+    /** Goes to the end (forwards) */
+    void go_to_end(void);
+    
+    /** Goes to the beginning (backwards) */
+    void go_to_beginning(void);
+    
+    /** Returns the current search result. asserts if there is no current item. */
+    history_item_t current_item(void) const;
+
+    /** Constructor */
+    history_search_t(history_t &hist, const wcstring &str) :
+        history(&hist),
+        term(str),
+        idx()
+    {
+    }
+    
+    /* Default constructor */
+    history_search_t() :
+        history(),
+        term(),
+        idx()
+    {
+    }
+    
+};
+
+
 
 /**
    Init history library. The history file won't actually be loaded
@@ -14,56 +145,10 @@
 void history_init();
 
 /**
-   Saves the new history to disc and frees all memory used by the history.
+   Saves the new history to disc.
 */
 void history_destroy();
 
-/**
-  Add a new history item to the bottom of the history, containing a
-  copy of str. Remove any duplicates. Moves the current item past the
-  end of the history list.
-*/
-void history_add( const wchar_t *str );
-
-/**
-  Find previous history item starting with str. If this moves before
-  the start of the history, str is returned.
-*/
-const wchar_t *history_prev_match( const wchar_t *str );
-
-/**
-   Return the specified history at the specified index, or 0 if out of bounds. 0 is the index of the current commandline.
-*/
-wchar_t *history_get( int idx );
-
-
-/**
-   Move to first history item
-*/
-void history_first();
-
-/**
-   Make current point to last history item
-*/
-void history_reset();
-
-
-/**
-  Find next history item starting with str. If this moves past
-  the end of the history, str is returned.  
-*/
-const wchar_t *history_next_match( const wchar_t *str);
-
-
-/**
-   Set the current mode name for history. Each application that uses
-   the history has it's own mode. This must be called prior to any use
-   of the history.
-*/
-
-void history_set_mode( const wchar_t *name );
-
-
 /**
    Perform sanity checks
 */

parser.cpp

@@ -806,7 +806,7 @@ int parser_t::eval_args( const wchar_t *line, std::vector<completion_t> &args )
 					DIE_MEM();
 				}
 				
-				if( expand_string2( tmp, args, 0 ) == EXPAND_ERROR )
+				if( expand_string( tmp, args, 0 ) == EXPAND_ERROR )
 				{
 					err_pos=tok_get_pos( &tok );
 					do_loop=0;
@@ -1344,7 +1344,7 @@ void parser_t::parse_job_argument_list( process_t *p,
 						p->count_help_magic = 1;
 					}
 
-					switch( expand_string2( tok_last( tok ), args, 0 ) )
+					switch( expand_string( tok_last( tok ), args, 0 ) )
 					{
 						case EXPAND_ERROR:
 						{

path.cpp

@@ -471,8 +471,47 @@ wchar_t *path_get_config( void *context)
 	{
 		debug( 0, _(L"Unable to create a configuration directory for fish. Your personal settings will not be saved. Please set the $XDG_CONFIG_HOME variable to a directory where the current user has write access." ));
 		return 0;
+	}	
+}
+
+bool path_get_config(wcstring &path)
+{
+	int done = 0;
+	wcstring res;
+	
+	const env_var_t xdg_dir  = env_get_string( L"XDG_CONFIG_HOME" );
+	if( ! xdg_dir.missing() )
+	{
+		res = xdg_dir + L"/fish";
+		if( !create_directory( res.c_str() ) )
+		{
+			done = 1;
+		}
+	}
+	else
+	{		
+		const env_var_t home = env_get_string( L"HOME" );
+		if( ! home.missing() )
+		{
+			res = home + L"/.config/fish";
+			if( !create_directory( res.c_str() ) )
+			{
+				done = 1;
+			}
+		}
 	}
 	
+	if( done )
+	{
+        path = res;
+        return true;
+	}
+	else
+	{
+		debug( 0, _(L"Unable to create a configuration directory for fish. Your personal settings will not be saved. Please set the $XDG_CONFIG_HOME variable to a directory where the current user has write access." ));
+		return false;
+	}	
+
 }
 
 wchar_t *path_make_canonical( void *context, const wchar_t *path )

path.h

@@ -22,6 +22,7 @@
    \return 0 if the no configuration directory can be located or created, the directory path otherwise.
 */
 wchar_t *path_get_config( void *context);
+bool path_get_config(wcstring &path);
 
 /**
    Finds the full path of an executable in a newly allocated string.

reader.cpp

@@ -193,12 +193,18 @@ class reader_data_t
 	   The representation of the current screen contents
 	 */
 	screen_t screen;
+    
+    /** The history */
+    history_t *history;
 
 	/**
 	   String containing the current search item
 	*/
 	wcstring search_buff;
 
+    /* History search */
+    history_search_t history_search;
+
 	/**
 	   Saved position used by token history search
 	*/
@@ -1867,8 +1873,6 @@ static void handle_token_history( int forward, int reset )
 	{
 		if( current_pos == -1 )
 		{
-			const wchar_t *item;
-			
 			/*
 			  Move to previous line
 			*/
@@ -1877,19 +1881,13 @@ static void handle_token_history( int forward, int reset )
 			/*
 			  Search for previous item that contains this substring
 			*/
-			item = history_prev_match(data->search_buff.c_str());
+            if (! data->history_search.go_backwards()) {
-
+                /* No luck */
-			/*
+                data->token_history_buff = wcsdup(L"");
-			  If there is no match, the original string is returned
+            } else {
-
+                history_item_t item = data->history_search.current_item();
-			  If so, we clear the match string to avoid infinite loop
+                data->token_history_buff = wcsdup(item.str().c_str());
-			*/
+            }
-			if( wcscmp( item, data->search_buff.c_str() ) == 0 )
-			{
-				item=L"";
-			}
-
-			data->token_history_buff = wcsdup( item );
 			current_pos = wcslen(data->token_history_buff);
 
 		}
@@ -2120,6 +2118,11 @@ wchar_t *reader_get_buffer(void)
 	return data?data->buff:NULL;
 }
 
+history_t *reader_get_history(void) {
+    ASSERT_IS_MAIN_THREAD();
+	return data ? data->history : NULL;
+}
+
 void reader_set_buffer( const wchar_t *b, int p )
 {
 	int l = wcslen( b );
@@ -2144,7 +2147,7 @@ void reader_set_buffer( const wchar_t *b, int p )
 
 	data->search_mode = NO_SEARCH;
 	data->search_buff.clear();
-	history_reset();
+	data->history_search.go_to_end();
 
 	reader_super_highlight_me_plenty( data->buff_pos,
 					  0 );
@@ -2264,6 +2267,7 @@ void reader_push( const wchar_t *name )
     reader_data_t zerod = {};
     reader_data_t *n = new reader_data_t(zerod);
     
+    n->history = & history_t::history_with_name(name);
 	n->app_name = name;
 	n->next = data;
 	sb_init( &n->kill_item );
@@ -2282,7 +2286,7 @@ void reader_push( const wchar_t *name )
 	reader_set_highlight_function( &highlight_universal );
 	reader_set_test_function( &default_test );
 	reader_set_prompt( L"" );
-	history_set_mode( name );
+	//history_set_mode( name );
 
 	data->token_history_buff=0;
 }
@@ -2320,7 +2324,7 @@ void reader_pop()
 	else
 	{
 		end_loop = 0;
-		history_set_mode( data->app_name.c_str() );
+		//history_set_mode( data->app_name.c_str() );
 		s_reset( &data->screen, 1 );
 	}
 }
@@ -2417,6 +2421,7 @@ static void highlight_search(void) {
 }
 
 static void highlight_complete(void *ctx_ptr, int result) {
+    ASSERT_IS_MAIN_THREAD();
 	background_highlight_context_t *ctx = (background_highlight_context_t *)ctx_ptr;
 	if (ctx->buff == data->buff) {
 		/* The data hasn't changed, so swap in our colors */
@@ -2554,7 +2559,7 @@ static int read_i()
 	event_fire_generic(L"fish_prompt");
 	
 	reader_push(L"fish");
-	reader_set_complete_function( &complete2 );
+	reader_set_complete_function( &complete );
 	reader_set_highlight_function( &highlight_shell );
 	reader_set_test_function( &reader_shell_test );
 	parser_t &parser = parser_t::principal_parser();
@@ -2980,7 +2985,8 @@ wchar_t *reader_readline()
 										
 					if( data->token_history_pos==-1 )
 					{
-						history_reset();
+						//history_reset();
+                        data->history_search.go_to_end();
 						reader_set_buffer( data->search_buff.c_str(), data->search_buff.size() );
 					}
 					else
@@ -3045,7 +3051,9 @@ wchar_t *reader_readline()
 						if( wcslen( data->buff ) )
 						{
 //						wcscpy(data->search_buff,L"");
-							history_add( data->buff );
+							//history_add( data->buff );
+                            if (data->history)
+                                data->history->add(data->buff);
 						}
 						finished=1;
 						data->buff_pos=data->buff_len;
@@ -3101,6 +3109,7 @@ wchar_t *reader_readline()
 					}
 					
                     data->search_buff.append(data->buff);
+                    data->history_search = history_search_t(*data->history, data->search_buff);
 				}
 
 				switch( data->search_mode )
@@ -3108,19 +3117,25 @@ wchar_t *reader_readline()
 
 					case LINE_SEARCH:
 					{
-						const wchar_t *it = 0;
+						bool success;
 						
 						if( ( c == R_HISTORY_SEARCH_BACKWARD ) ||
 							( c == R_HISTORY_TOKEN_SEARCH_BACKWARD ) )
 						{
-							it = history_prev_match(data->search_buff.c_str());
+							success = data->history_search.go_backwards();
 						}
 						else
 						{
-							it = history_next_match(data->search_buff.c_str());
+							success = data->history_search.go_forwards();
 						}
 						
-						handle_history( it );
+                        wcstring new_text;
+                        if (success) {
+                            new_text = data->history_search.current_item().str();
+                        } else {
+                            new_text = data->search_buff;
+                        }
+						handle_history( new_text.c_str() );
 						
 						break;
 					}
@@ -3197,13 +3212,13 @@ wchar_t *reader_readline()
 
 			case R_BEGINNING_OF_HISTORY:
 			{
-				history_first();
+				data->history_search.go_to_beginning();
 				break;
 			}
 
 			case R_END_OF_HISTORY:
 			{
-				history_reset();
+				data->history_search.go_to_end();
 				break;
 			}
 
@@ -3283,7 +3298,7 @@ wchar_t *reader_readline()
 		{
 			data->search_mode = NO_SEARCH;
 			data->search_buff.clear();
-			history_reset();
+			data->history_search.go_to_end();
 			data->token_history_pos=-1;
 		}
 		

reader.h

@@ -17,6 +17,7 @@
 
 class parser_t;
 class completion_t;
+class history_t;
 
 /**
   Read commands from \c fd until encountering EOF
@@ -84,6 +85,9 @@ void reader_run_command( const wchar_t *buff );
 */
 wchar_t *reader_get_buffer();
 
+/** Returns the current reader's history */
+history_t *reader_get_history(void);
+
 /**
    Set the string of characters in the command buffer, as well as the cursor position.