fish-shell/common.cpp

/** \file common.c
	
Various functions, mostly string utilities, that are used by most
parts of fish.
*/

#include "config.h"


#include <unistd.h>

#ifdef HAVE_STROPTS_H
#include <stropts.h>
#endif

#ifdef HAVE_SIGINFO_H
#include <siginfo.h>
#endif

#include <stdlib.h>
#include <termios.h>
#include <wchar.h>
#include <string.h>
#include <stdio.h>
#include <dirent.h>
#include <sys/types.h>

#ifdef HAVE_SYS_TERMIOS_H
#include <sys/termios.h>
#endif

#ifdef HAVE_SYS_IOCTL_H
#include <sys/ioctl.h>
#endif

#include <sys/stat.h>
#include <unistd.h>
#include <wctype.h>
#include <errno.h>
#include <limits.h>
#include <stdarg.h>		
#include <locale.h>
#include <time.h>
#include <sys/time.h>
#include <fcntl.h>
#include <algorithm>

#ifdef HAVE_EXECINFO_H
#include <execinfo.h>
#endif


#if HAVE_NCURSES_H
#include <ncurses.h>
#else
#include <curses.h>
#endif

#if HAVE_TERMIO_H
#include <termio.h>
#endif

#if HAVE_TERM_H
#include <term.h>
#elif HAVE_NCURSES_TERM_H
#include <ncurses/term.h>
#endif

#include "fallback.h"
#include "util.h"

#include "wutil.h"
#include "common.h"
#include "expand.h"
#include "proc.h"
#include "wildcard.h"
#include "parser.h"
#include "complete.h"

#include "util.cpp"
#include "fallback.cpp"



struct termios shell_modes;      

static pthread_t main_thread_id = 0;

wchar_t ellipsis_char;

char *profile=0;

const wchar_t *program_name;

int debug_level=1;

/**
   This struct should be continually updated by signals as the term resizes, and as such always contain the correct current size.
*/
static struct winsize termsize;


void show_stackframe() 
{
	void *trace[32];
	char **messages = (char **)NULL;
	int i, trace_size = 0;

	trace_size = backtrace(trace, 32);
	messages = backtrace_symbols(trace, trace_size);

	if( messages )
	{
		debug( 0, L"Backtrace:" );
		for( i=0; i<trace_size; i++ )
		{
			fwprintf( stderr, L"%s\n", messages[i]);
		}
		free( messages );
	}
}

wcstring_list_t completions_to_wcstring_list( const std::vector<completion_t> &list )
{
    wcstring_list_t strings;
    strings.reserve(list.size());
    for (std::vector<completion_t>::const_iterator iter = list.begin(); iter != list.end(); ++iter) {
        strings.push_back(iter->completion);
    }
    return strings;
}


int fgetws2( wchar_t **b, int *len, FILE *f )
{
	int i=0;
	wint_t c;
	
	wchar_t *buff = *b;

	while( 1 )
	{
		/* Reallocate the buffer if necessary */
		if( i+1 >= *len )
		{
			int new_len = maxi( 128, (*len)*2);
			buff = (wchar_t *)realloc( buff, sizeof(wchar_t)*new_len );
			if( buff == 0 )
			{
				DIE_MEM();
			}
			else
			{
				*len = new_len;
				*b = buff;
			}			
		}
		
		errno=0;		
		
		c = getwc( f );
		
		if( errno == EILSEQ )
		{
			continue;
		}
	
//fwprintf( stderr, L"b\n" );
		
		switch( c )
		{
			/* End of line */ 
			case WEOF:
			case L'\n':
			case L'\0':
				buff[i]=L'\0';
				return i;				
				/* Ignore carriage returns */
			case L'\r':
				break;
				
			default:
				buff[i++]=c;
				break;
		}		


	}

}


static bool string_sort_predicate(const wcstring& d1, const wcstring& d2)
{
    return wcsfilecmp(d1.c_str(), d2.c_str()) < 0;
}

void sort_strings( std::vector<wcstring> &strings)
{
    std::sort(strings.begin(), strings.end(), string_sort_predicate);
}

void sort_completions( std::vector<completion_t> &completions)
{
    std::sort(completions.begin(), completions.end());
}
wchar_t *str2wcs( const char *in )
{
	wchar_t *out;
	size_t len = strlen(in);
	
	out = (wchar_t *)malloc( sizeof(wchar_t)*(len+1) );

	if( !out )
	{
		DIE_MEM();
	}

	return str2wcs_internal( in, out );
}

wcstring str2wcstring( const char *in )
{
    wchar_t *tmp = str2wcs(in);
    wcstring result = tmp;
    free(tmp);
    return result;
}

wcstring str2wcstring( const std::string &in )
{
    wchar_t *tmp = str2wcs(in.c_str());
    wcstring result = tmp;
    free(tmp);
    return result;
}

wchar_t *str2wcs_internal( const char *in, wchar_t *out )
{
	size_t res=0;
	int in_pos=0;
	int out_pos = 0;
	mbstate_t state;
	size_t len;

	CHECK( in, 0 );
	CHECK( out, 0 );
		
	len = strlen(in);

	memset( &state, 0, sizeof(state) );

	while( in[in_pos] )
	{
		res = mbrtowc( &out[out_pos], &in[in_pos], len-in_pos, &state );

		if( ( ( out[out_pos] >= ENCODE_DIRECT_BASE) &&
		      ( out[out_pos] < ENCODE_DIRECT_BASE+256)) ||
		    ( out[out_pos] == INTERNAL_SEPARATOR ) )
		{
			out[out_pos] = ENCODE_DIRECT_BASE + (unsigned char)in[in_pos];
			in_pos++;
			memset( &state, 0, sizeof(state) );
			out_pos++;
		}
		else
		{
			
			switch( res )
			{
				case (size_t)(-2):
				case (size_t)(-1):
				{
					out[out_pos] = ENCODE_DIRECT_BASE + (unsigned char)in[in_pos];
					in_pos++;
					memset( &state, 0, sizeof(state) );
					break;
				}
				
				case 0:
				{
					return out;
				}
		
				default:
				{
					in_pos += res;
					break;
				}
			}
			out_pos++;
		}
		
	}
	out[out_pos] = 0;
	
	return out;	
}

char *wcs2str( const wchar_t *in )
{
	char *out;	
	
	out = (char *)malloc( MAX_UTF8_BYTES*wcslen(in)+1 );

	if( !out )
	{
		DIE_MEM();
	}

	return wcs2str_internal( in, out );
}

std::string wcs2string(const wcstring &input)
{
    char *tmp = wcs2str(input.c_str());
    std::string result = tmp;
    free(tmp);
    return result;
}

char *wcs2str_internal( const wchar_t *in, char *out )
{
	size_t res=0;
	int in_pos=0;
	int out_pos = 0;
	mbstate_t state;

	CHECK( in, 0 );
	CHECK( out, 0 );
	
	memset( &state, 0, sizeof(state) );
	
	while( in[in_pos] )
	{
		if( in[in_pos] == INTERNAL_SEPARATOR )
		{
		}
		else if( ( in[in_pos] >= ENCODE_DIRECT_BASE) &&
			 ( in[in_pos] < ENCODE_DIRECT_BASE+256) )
		{
			out[out_pos++] = in[in_pos]- ENCODE_DIRECT_BASE;
		}
		else
		{
			res = wcrtomb( &out[out_pos], in[in_pos], &state );
			
			if( res == (size_t)(-1) )
			{
				debug( 1, L"Wide character %d has no narrow representation", in[in_pos] );
				memset( &state, 0, sizeof(state) );
			}
			else
			{
				out_pos += res;
			}
		}
		in_pos++;
	}
	out[out_pos] = 0;
	
	return out;	
}

char **wcsv2strv( const wchar_t * const *in )
{
	int count =0;
	int i;

	while( in[count] != 0 )
		count++;
	char **res = (char **)malloc( sizeof( char *)*(count+1));
	if( res == 0 )
	{
		DIE_MEM();		
	}

	for( i=0; i<count; i++ )
	{
		res[i]=wcs2str(in[i]);
	}
	res[count]=0;
	return res;

}

wchar_t *wcsdupcat_internal( const wchar_t *a, ... )
{
	int len=wcslen(a);
	int pos;
	va_list va, va2;
	wchar_t *arg;

	va_start( va, a );
	va_copy( va2, va );
	while( (arg=va_arg(va, wchar_t *) )!= 0 ) 
	{
		len += wcslen( arg );
	}
	va_end( va );

	wchar_t *res = (wchar_t *)malloc( sizeof(wchar_t)*(len +1 ));
	if( res == 0 )
	{
		DIE_MEM();
	}
	
	wcscpy( res, a );
	pos = wcslen(a);
	while( (arg=va_arg(va2, wchar_t *) )!= 0 ) 
	{
		wcscpy( res+pos, arg );
		pos += wcslen(arg);
	}
	va_end( va2 );
	return res;	

}


wchar_t **strv2wcsv( const char **in )
{
	int count =0;
	int i;

	while( in[count] != 0 )
		count++;
	wchar_t **res = (wchar_t **)malloc( sizeof( wchar_t *)*(count+1));
	if( res == 0 )
	{
		DIE_MEM();
	}

	for( i=0; i<count; i++ )
	{
		res[i]=str2wcs(in[i]);
	}
	res[count]=0;
	return res;

}
wcstring format_string(const wchar_t *format, ...)
{
	va_list va;
	va_start( va, format );
    wcstring result = vformat_string(format, va);
	va_end( va );
    return result;
}

wcstring vformat_string(const wchar_t *format, va_list va_orig)
{
    string_buffer_t buffer;
    sb_init(&buffer);
    sb_vprintf(&buffer, format, va_orig);
    wcstring result = (wchar_t *)buffer.buff;
    sb_destroy(&buffer);
    return result;
}

void append_format(wcstring &str, const wchar_t *format, ...)
{
	va_list va;
	va_start( va, format );
    str.append(vformat_string(format, va));
	va_end( va );
}

wchar_t *wcsvarname( const wchar_t *str )
{
	while( *str )
	{
		if( (!iswalnum(*str)) && (*str != L'_' ) )
		{
			return (wchar_t *)str;
		}
		str++;
	}
	return 0;
}

const wchar_t *wcsfuncname( const wchar_t *str )
{
	return wcschr( str, L'/' );
}


int wcsvarchr( wchar_t chr )
{
	return ( (iswalnum(chr)) || (chr == L'_' ));
}


/** 
	The glibc version of wcswidth seems to hang on some strings. fish uses this replacement.
*/
int my_wcswidth( const wchar_t *c )
{
	int res=0;
	while( *c )
	{
		int w = wcwidth( *c++ );
		if( w < 0 )
			w = 1;
		if( w > 2 )
			w=1;
		
		res += w;		
	}
	return res;
}

wchar_t *quote_end( const wchar_t *pos )
{
	wchar_t c = *pos;
	
	while( 1 )
	{
		pos++;
		
		if( !*pos )
			return 0;
		
		if( *pos == L'\\')
		{
			pos++;
			if( !*pos )
				return 0;
		}
		else
		{
			if( *pos == c )
			{
				return (wchar_t *)pos;
			}
		}
	}
	return 0;
	
}

				
wcstring wsetlocale(int category, const wchar_t *locale)
{

	char *lang = NULL;
	if (locale && wcscmp(locale,L"")){
		lang = wcs2str( locale );
	}
	char * res = setlocale(category,lang);
	free( lang );

	/*
	  Use ellipsis if on known unicode system, otherwise use $
	*/
	char *ctype = setlocale( LC_CTYPE, NULL );
	ellipsis_char = (strstr( ctype, ".UTF")||strstr( ctype, ".utf") )?L'\x2026':L'$';	
		
	if( !res )
		return wcstring();
    else
		return format_string(L"%s", res);
}

bool contains_internal( const wchar_t *a, ... )
{
	const wchar_t *arg;
	va_list va;
	int res = 0;

	CHECK( a, 0 );
	
	va_start( va, a );
	while( (arg=va_arg(va, const wchar_t *) )!= 0 ) 
	{
		if( wcscmp( a,arg) == 0 )
		{
			res=1;
			break;
		}
		
	}
	va_end( va );
	return res;
}

/* wcstring variant of contains_internal. The first parameter is a wcstring, the rest are const wchar_t* */
__sentinel bool contains_internal( const wcstring &needle, ... )
{
	const wchar_t *arg;
	va_list va;
	int res = 0;

	va_start( va, needle );
	while( (arg=va_arg(va, const wchar_t *) )!= 0 ) 
	{
		if( needle == arg)
		{
			res=1;
			break;
		}
		
	}
	va_end( va );
	return res;
}

int read_blocked(int fd, void *buf, size_t count)
{
	int res;	
	sigset_t chldset, oldset; 

	sigemptyset( &chldset );
	sigaddset( &chldset, SIGCHLD );
	sigprocmask(SIG_BLOCK, &chldset, &oldset);
	res = read( fd, buf, count );
	sigprocmask( SIG_SETMASK, &oldset, 0 );
	return res;	
}

ssize_t write_loop(int fd, const char *buff, size_t count)
{
	ssize_t out=0;
	size_t out_cum=0;
	while( 1 ) 
	{
		out = write( fd, 
					 &buff[out_cum],
					 count - out_cum );
		if (out < 0) 
		{
			if(errno != EAGAIN && errno != EINTR) 
			{
				return -1;
			}
		}
        else 
		{
			out_cum += (size_t)out;
		}
		if( out_cum >= count ) 
		{
			break;
		}
	}						
	return out_cum;
}

ssize_t read_loop(int fd, void *buff, size_t count)
{
    ssize_t result;
    do {
        result = read(fd, buff, count);
    } while (result < 0 && (errno == EAGAIN || errno == EINTR));
    return result;
}

void debug( int level, const wchar_t *msg, ... )
{
	va_list va;

	wcstring sb;

	int errno_old = errno;
	
	if( level > debug_level )
		return;

	CHECK( msg, );
		
	sb = format_string(L"%ls: ", program_name);
	va_start(va, msg);
	sb.append(vformat_string(msg, va));
	va_end(va);

	wcstring sb2;
	write_screen( sb, sb2 );
	fwprintf( stderr, L"%ls", sb2.c_str() );	

	errno = errno_old;
}

void debug_safe(int level, const char *msg, const char *param1, const char *param2, const char *param3)
{
    if (! msg)
        return;
        
    /* Can't call printf, that may allocate memory Just call write() over and over. */
    if (level > debug_level)
        return;
    int errno_old = errno;
    
    size_t param_idx = 0;
    const char *cursor = msg;
    while (*cursor != '\0') {
        const char *end = strchr(cursor, '%');
        if (end == NULL)
            end = cursor + strlen(cursor);
        
        write(STDERR_FILENO, cursor, end - cursor);

        if (end[0] == '%' && end[1] == 's') {
            /* Handle a format string */
            const char *format = NULL;
            switch (param_idx++) {
                case 0: format = param1; break;
                case 1: format = param2; break;
                case 2: format = param3; break;
            }
            if (! format) 
                format = "(null)";
            write(STDERR_FILENO, format, strlen(format));
            cursor = end + 2;
        } else if (end[0] == '\0') {
            /* Must be at the end of the string */
            cursor = end;
        } else {
            /* Some other format specifier, just skip it */
            cursor = end + 1;
        }
    }
    
    // We always append a newline
    write(STDERR_FILENO, "\n", 1);
    
    errno = errno_old;
}

void format_int_safe(char buff[128], int val) {
    if (val == 0) {
        strcpy(buff, "0");
    } else {
        /* Generate the string in reverse */
        size_t idx = 0;
        bool negative = (val < 0);
        if (negative)
            val = -val;

        while (val > 0) {
            buff[idx++] = '0' + (val % 10);
            val /= 10;
        }
        if (negative)
            buff[idx++] = '-';
        buff[idx] = 0;
        
        size_t left = 0, right = idx - 1;
        while (left < right)  {
            char tmp = buff[left];
            buff[left++] = buff[right];
            buff[right--] = tmp;
        }
    }
}

void write_screen( const wcstring &msg, wcstring &buff )
{
	const wchar_t *start, *pos;
	int line_width = 0;
	int tok_width = 0;
	int screen_width = common_get_width();
	
	if( screen_width )
	{
		start = pos = msg.c_str();
		while( 1 )
		{
			int overflow = 0;
		
			tok_width=0;

			/*
			  Tokenize on whitespace, and also calculate the width of the token
			*/
			while( *pos && ( !wcschr( L" \n\r\t", *pos ) ) )
			{
				
				/*
				  Check is token is wider than one line.
				  If so we mark it as an overflow and break the token.
				*/
				if((tok_width + wcwidth(*pos)) > (screen_width-1))
				{
					overflow = 1;
					break;				
				}
			
				tok_width += wcwidth( *pos );
				pos++;
			}

			/*
			  If token is zero character long, we don't do anything
			*/
			if( pos == start )
			{
				start = pos = pos+1;
			}
			else if( overflow )
			{
				/*
				  In case of overflow, we print a newline, except if we already are at position 0
				*/
				wchar_t *token = wcsndup( start, pos-start );
				if( line_width != 0 )
                    buff.push_back(L'\n');
                buff.append(format_string(L"%ls-\n", token));
				free( token );
				line_width=0;
			}
			else
			{
				/*
				  Print the token
				*/
				wchar_t *token = wcsndup( start, pos-start );
				if( (line_width + (line_width!=0?1:0) + tok_width) > screen_width )
				{
                    buff.push_back(L'\n');
					line_width=0;
				}
                buff.append(format_string(L"%ls%ls", line_width?L" ":L"", token ));
				free( token );
				line_width += (line_width!=0?1:0) + tok_width;
			}
			
			/*
			  Break on end of string
			*/
			if( !*pos )
			{
				break;
			}
			
			start=pos;
		}
	}
	else
	{
        buff.append(msg);
	}
    buff.push_back(L'\n');
}

void write_screen( const wcstring &msg, string_buffer_t *buff )
{
	const wchar_t *start, *pos;
	int line_width = 0;
	int tok_width = 0;
	int screen_width = common_get_width();
	
	CHECK( buff, );
		
	if( screen_width )
	{
		start = pos = msg.c_str();
		while( 1 )
		{
			int overflow = 0;
		
			tok_width=0;

			/*
			  Tokenize on whitespace, and also calculate the width of the token
			*/
			while( *pos && ( !wcschr( L" \n\r\t", *pos ) ) )
			{
				
				/*
				  Check is token is wider than one line.
				  If so we mark it as an overflow and break the token.
				*/
				if((tok_width + wcwidth(*pos)) > (screen_width-1))
				{
					overflow = 1;
					break;				
				}
			
				tok_width += wcwidth( *pos );
				pos++;
			}

			/*
			  If token is zero character long, we don't do anything
			*/
			if( pos == start )
			{
				start = pos = pos+1;
			}
			else if( overflow )
			{
				/*
				  In case of overflow, we print a newline, except if we already are at position 0
				*/
				wchar_t *token = wcsndup( start, pos-start );
				if( line_width != 0 )
					sb_append_char( buff, L'\n' );
				sb_printf( buff, L"%ls-\n", token );
				free( token );
				line_width=0;
			}
			else
			{
				/*
				  Print the token
				*/
				wchar_t *token = wcsndup( start, pos-start );
				if( (line_width + (line_width!=0?1:0) + tok_width) > screen_width )
				{
					sb_append_char( buff, L'\n' );
					line_width=0;
				}
				sb_printf( buff, L"%ls%ls", line_width?L" ":L"", token );
				free( token );
				line_width += (line_width!=0?1:0) + tok_width;
			}
			
			/*
			  Break on end of string
			*/
			if( !*pos )
			{
				break;
			}
			
			start=pos;
		}
	}
	else
	{
		sb_printf( buff, L"%ls", msg.c_str() );
	}
	sb_append_char( buff, L'\n' );
}

/**
   Perform string escaping of a strinng by only quoting it. Assumes
   the string has already been checked for characters that can not be
   escaped this way.
 */
static wchar_t *escape_simple( const wchar_t *in )
{
	wchar_t *out;
	size_t len = wcslen(in);
	out = (wchar_t *)malloc( sizeof(wchar_t)*(len+3));
	if( !out )
		DIE_MEM();
	
	out[0] = L'\'';
	wcscpy(&out[1], in );
	out[len+1]=L'\'';
	out[len+2]=0;
	return out;
}

wchar_t *escape( const wchar_t *in_orig, 
		 int flags )
{
	const wchar_t *in = in_orig;
	
	int escape_all = flags & ESCAPE_ALL;
	int no_quoted  = flags & ESCAPE_NO_QUOTED;
	
	wchar_t *out;
	wchar_t *pos;

	int need_escape=0;
	int need_complex_escape=0;

	if( !in )
	{
		debug( 0, L"%s called with null input", __func__ );
		FATAL_EXIT();
	}

	if( !no_quoted && (wcslen( in ) == 0) )
	{
		out = wcsdup(L"''");
		if( !out )
			DIE_MEM();
		return out;
	}
	
	
	out = (wchar_t *)malloc( sizeof(wchar_t)*(wcslen(in)*4 + 1));
	pos = out;
	
	if( !out )
		DIE_MEM();
	
	while( *in != 0 )
	{

		if( ( *in >= ENCODE_DIRECT_BASE) &&
			( *in < ENCODE_DIRECT_BASE+256) )
		{
			int val = *in - ENCODE_DIRECT_BASE;
			int tmp;
			
			*(pos++) = L'\\';
			*(pos++) = L'X';
			
			tmp = val/16;			
			*pos++ = tmp > 9? L'a'+(tmp-10):L'0'+tmp;
			
			tmp = val%16;			
			*pos++ = tmp > 9? L'a'+(tmp-10):L'0'+tmp;
			need_escape=need_complex_escape=1;
			
		}
		else
		{
			
			switch( *in )
			{
				case L'\t':
					*(pos++) = L'\\';
					*(pos++) = L't';					
					need_escape=need_complex_escape=1;
					break;
					
				case L'\n':
					*(pos++) = L'\\';
					*(pos++) = L'n';					
					need_escape=need_complex_escape=1;
					break;
					
				case L'\b':
					*(pos++) = L'\\';
					*(pos++) = L'b';					
					need_escape=need_complex_escape=1;
					break;
					
				case L'\r':
					*(pos++) = L'\\';
					*(pos++) = L'r';					
					need_escape=need_complex_escape=1;
					break;
					
				case L'\x1b':
					*(pos++) = L'\\';
					*(pos++) = L'e';					
					need_escape=need_complex_escape=1;
					break;
					

				case L'\\':
				case L'\'':
				{
					need_escape=need_complex_escape=1;
					if( escape_all )
						*pos++ = L'\\';
					*pos++ = *in;
					break;
				}

				case L'&':
				case L'$':
				case L' ':
				case L'#':
				case L'^':
				case L'<':
				case L'>':
				case L'(':
				case L')':
				case L'[':
				case L']':
				case L'{':
				case L'}':
				case L'?':
				case L'*':
				case L'|':
				case L';':
				case L'"':
				case L'%':
				case L'~':
				{
					need_escape=1;
					if( escape_all )
						*pos++ = L'\\';
					*pos++ = *in;
					break;
				}
				
				default:
				{
					if( *in < 32 )
					{
						if( *in <27 && *in > 0 )
						{
							*(pos++) = L'\\';
							*(pos++) = L'c';
							*(pos++) = L'a' + *in -1;
							
							need_escape=need_complex_escape=1;
							break;
								
						}
						

						int tmp = (*in)%16;
						*pos++ = L'\\';
						*pos++ = L'x';
						*pos++ = ((*in>15)? L'1' : L'0');
						*pos++ = tmp > 9? L'a'+(tmp-10):L'0'+tmp;
						need_escape=need_complex_escape=1;
					}
					else
					{
						*pos++ = *in;
					}
					break;
				}
			}
		}
		
		in++;
	}
	*pos = 0;

	/*
	  Use quoted escaping if possible, since most people find it
	  easier to read. 
	 */
	if( !no_quoted && need_escape && !need_complex_escape && escape_all )
	{
		free( out );
		out = escape_simple( in_orig );
	}
	
	return out;
}

wcstring escape_string( const wcstring &in, int escape_all ) {
    wchar_t *tmp = escape(in.c_str(), escape_all);
    wcstring result(tmp);
    free(tmp);
    return result;
}


wchar_t *unescape( const wchar_t * orig, int flags )
{
	
	int mode = 0; 
	int in_pos, out_pos, len;
	int c;
	int bracket_count=0;
	wchar_t prev=0;	
	wchar_t *in;
	int unescape_special = flags & UNESCAPE_SPECIAL;
	int allow_incomplete = flags & UNESCAPE_INCOMPLETE;
	
	CHECK( orig, 0 );
		
	len = wcslen( orig );
	in = wcsdup( orig );

	if( !in )
		DIE_MEM();
	
	for( in_pos=0, out_pos=0; 
		 in_pos<len; 
		 (prev=(out_pos>=0)?in[out_pos]:0), out_pos++, in_pos++ )
	{
		c = in[in_pos];
		switch( mode )
		{

			/*
			  Mode 0 means unquoted string
			*/
			case 0:
			{
				if( c == L'\\' )
				{
					switch( in[++in_pos] )
					{
						
						/*
						  A null character after a backslash is an
						  error, return null
						*/
						case L'\0':
						{
							if( !allow_incomplete )
							{
								free(in);
								return 0;
							}
						}
												
						/*
						  Numeric escape sequences. No prefix means
						  octal escape, otherwise hexadecimal.
						*/
						
						case L'0':
						case L'1':
						case L'2':
						case L'3':
						case L'4':
						case L'5':
						case L'6':
						case L'7':
						case L'u':
						case L'U':
						case L'x':
						case L'X':
						{
							int i;
							long long res=0;
							int chars=2;
							int base=16;
							
							int byte = 0;
							wchar_t max_val = ASCII_MAX;
							
							switch( in[in_pos] )
							{
								case L'u':
								{
									chars=4;
									max_val = UCS2_MAX;
									break;
								}
								
								case L'U':
								{
									chars=8;
									max_val = WCHAR_MAX;
									break;
								}
								
								case L'x':
								{
									break;
								}
								
								case L'X':
								{
									byte=1;
									max_val = BYTE_MAX;
									break;
								}
								
								default:
								{
									base=8;
									chars=3;
									in_pos--;
									break;
								}								
							}
					
							for( i=0; i<chars; i++ )
							{
								int d = convert_digit( in[++in_pos],base);
								
								if( d < 0 )
								{
									in_pos--;
									break;
								}
								
								res=(res*base)|d;
							}

							if( (res <= max_val) )
							{
								in[out_pos] = (byte?ENCODE_DIRECT_BASE:0)+res;
							}
							else
							{	
								free(in);	
								return 0;
							}
							
							break;
						}

						/*
						  \a means bell (alert)
						*/
						case L'a':
						{
							in[out_pos]=L'\a';
							break;
						}
						
						/*
						  \b means backspace
						*/
						case L'b':
						{
							in[out_pos]=L'\b';
							break;
						}
						
						/*
						  \cX means control sequence X
						*/
						case L'c':
						{
							in_pos++;
							if( in[in_pos] >= L'a' &&
								in[in_pos] <= (L'a'+32) )
							{
								in[out_pos]=in[in_pos]-L'a'+1;
							}
							else if( in[in_pos] >= L'A' &&
									 in[in_pos] <= (L'A'+32) )
							{
								in[out_pos]=in[in_pos]-L'A'+1;
							}
							else
							{
								free(in);	
								return 0;
							}
							break;
							
						}
						
						/*
						  \x1b means escape
						*/
						case L'e':
						{
							in[out_pos]=L'\x1b';
							break;
						}
						
						/*
						  \f means form feed
						*/
						case L'f':
						{
							in[out_pos]=L'\f';
							break;
						}

						/*
						  \n means newline
						*/
						case L'n':
						{
							in[out_pos]=L'\n';
							break;
						}
						
						/*
						  \r means carriage return
						*/
						case L'r':
						{
							in[out_pos]=L'\r';
							break;
						}
						
						/*
						  \t means tab
						 */
						case L't':
						{
							in[out_pos]=L'\t';
							break;
						}

						/*
						  \v means vertical tab
						*/
						case L'v':
						{
							in[out_pos]=L'\v';
							break;
						}
						
						default:
						{
							if( unescape_special )
								in[out_pos++] = INTERNAL_SEPARATOR;							
							in[out_pos]=in[in_pos];
							break;
						}
					}
				}
				else 
				{
					switch( in[in_pos])
					{
						case L'~':
						{
							if( unescape_special && (in_pos == 0) )
							{
								in[out_pos]=HOME_DIRECTORY;
							}
							else
							{
								in[out_pos] = L'~';
							}
							break;
						}

						case L'%':
						{
							if( unescape_special && (in_pos == 0) )
							{
								in[out_pos]=PROCESS_EXPAND;
							}
							else
							{
								in[out_pos]=in[in_pos];						
							}
							break;
						}

						case L'*':
						{
							if( unescape_special )
							{
								if( out_pos > 0 && in[out_pos-1]==ANY_STRING )
								{
									out_pos--;
									in[out_pos] = ANY_STRING_RECURSIVE;
								}
								else
									in[out_pos]=ANY_STRING;
							}
							else
							{
								in[out_pos]=in[in_pos];						
							}
							break;
						}

						case L'?':
						{
							if( unescape_special )
							{
								in[out_pos]=ANY_CHAR;
							}
							else
							{
								in[out_pos]=in[in_pos];						
							}
							break;					
						}

						case L'$':
						{
							if( unescape_special )
							{
								in[out_pos]=VARIABLE_EXPAND;
							}
							else
							{
								in[out_pos]=in[in_pos];											
							}
							break;					
						}

						case L'{':
						{
							if( unescape_special )
							{
								bracket_count++;
								in[out_pos]=BRACKET_BEGIN;
							}
							else
							{
								in[out_pos]=in[in_pos];						
							}
							break;					
						}
						
						case L'}':
						{
							if( unescape_special )
							{
								bracket_count--;
								in[out_pos]=BRACKET_END;
							}
							else
							{
								in[out_pos]=in[in_pos];						
							}
							break;					
						}
						
						case L',':
						{
							if( unescape_special && bracket_count && prev!=BRACKET_SEP)
							{
								in[out_pos]=BRACKET_SEP;
							}
							else
							{
								in[out_pos]=in[in_pos];						
							}
							break;					
						}
						
						case L'\'':
						{
							mode = 1;
							if( unescape_special )
								in[out_pos] = INTERNAL_SEPARATOR;							
							else
								out_pos--;						
							break;					
						}
				
						case L'\"':
						{
							mode = 2;
							if( unescape_special )
								in[out_pos] = INTERNAL_SEPARATOR;							
							else
								out_pos--;						
							break;
						}

						default:
						{
							in[out_pos] = in[in_pos];
							break;
						}
					}
				}		
				break;
			}

			/*
			  Mode 1 means single quoted string, i.e 'foo'
			*/
			case 1:
			{
				if( c == L'\\' )
				{
					switch( in[++in_pos] )
					{
						case '\\':
						case L'\'':
						case L'\n':
						{
							in[out_pos]=in[in_pos];
							break;
						}
						
						case 0:
						{
							if( !allow_incomplete )
							{
								free(in);
								return 0;
							}
							else
								out_pos--;						
						}
						
						default:
						{
							in[out_pos++] = L'\\';
							in[out_pos]= in[in_pos];
						}
					}
					
				}
				if( c == L'\'' )
				{
					if( unescape_special )
						in[out_pos] = INTERNAL_SEPARATOR;							
					else
						out_pos--;						
					mode = 0;
				}
				else
				{
					in[out_pos] = in[in_pos];
				}
				
				break;
			}

			/*
			  Mode 2 means double quoted string, i.e. "foo"
			*/
			case 2:
			{
				switch( c )
				{
					case '"':
					{
						mode = 0;
						if( unescape_special )
							in[out_pos] = INTERNAL_SEPARATOR;							
						else
							out_pos--;						
						break;
					}
				
					case '\\':
					{
						switch( in[++in_pos] )
						{
							case L'\0':
							{
								if( !allow_incomplete )
								{
									free(in);
									return 0;
								}
								else
									out_pos--;						
							}
							
							case '\\':
							case L'$':
							case '"':
							case '\n':
							{
								in[out_pos]=in[in_pos];
								break;
							}

							default:
							{
								in[out_pos++] = L'\\';
								in[out_pos] = in[in_pos];
								break;
							}
						}
						break;
					}
					
					case '$':
					{
						if( unescape_special )
						{
							in[out_pos]=VARIABLE_EXPAND_SINGLE;
						}
						else
						{
							in[out_pos]=in[in_pos];
						}
						break;
					}
			
					default:
					{
						in[out_pos] = in[in_pos];
						break;
					}
				
				}						
				break;
			}
		}
	}

	if( !allow_incomplete && mode )
	{
		free( in );
		return 0;
	}

	in[out_pos]=L'\0';
	return in;	
}

bool unescape_string(wcstring &str, int escape_special)
{
    bool success = false;
    wchar_t *result = unescape(str.c_str(), escape_special);
    if (result) {
        str.replace(str.begin(), str.end(), result);
        free(result);
        success = true;
    }
    return success;
}



void common_handle_winch( int signal )
{
#ifdef HAVE_WINSIZE
	if (ioctl(1,TIOCGWINSZ,&termsize)!=0)
	{
		return;
	}
#else
	termsize.ws_col = 80;
	termsize.ws_row = 24;
#endif
}

int common_get_width()
{
	return termsize.ws_col;
}


int common_get_height()
{
	return termsize.ws_row;
}

void tokenize_variable_array( const wcstring &val, std::vector<wcstring> &out)
{
    size_t pos = 0, end = val.size();
    while (pos < end) {
        size_t next_pos = val.find(ARRAY_SEP, pos);
        if (next_pos == wcstring::npos) break;
        out.push_back(val.substr(pos, next_pos - pos));
        pos = next_pos + 1; //skip the separator
    }
    out.push_back(val.substr(pos, end - pos));
}

bool string_prefixes_string(const wcstring &proposed_prefix, const wcstring &value)
{
    size_t prefix_size = proposed_prefix.size();
    return prefix_size <= value.size() && value.compare(0, prefix_size, proposed_prefix) == 0;
}

bool string_prefixes_string_case_insensitive(const wcstring &proposed_prefix, const wcstring &value) {
    size_t prefix_size = proposed_prefix.size();
    return prefix_size <= value.size() && wcsncasecmp(proposed_prefix.c_str(), value.c_str(), prefix_size) == 0;
}

bool list_contains_string(const wcstring_list_t &list, const wcstring &str)
{
    return std::find(list.begin(), list.end(), str) != list.end();
}

int create_directory( const wcstring &d )
{
	int ok = 0;
	struct stat buf;
	int stat_res = 0;
	
	while( (stat_res = wstat(d, &buf ) ) != 0 )
	{
		if( errno != EAGAIN )
			break;
	}
	
	if( stat_res == 0 )
	{
		if( S_ISDIR( buf.st_mode ) )
		{
			ok = 1;
		}
	}
	else
	{
		if( errno == ENOENT )
		{
            wcstring dir = wdirname(d);
			if( !create_directory( dir ) )
			{
				if( !wmkdir( d, 0700 ) )
				{
					ok = 1;
				}
			}
		}
	}
	
	return ok?0:-1;
}

__attribute__((noinline))
void bugreport()
{
	debug( 1,
	       _( L"This is a bug. Break on bugreport to debug."
		  L"If you can reproduce it, please send a bug report to %s." ),
		PACKAGE_BUGREPORT );
}


void sb_format_size( string_buffer_t *sb,
		     long long sz )
{
	const wchar_t *sz_name[]=
		{
			L"kB", L"MB", L"GB", L"TB", L"PB", L"EB", L"ZB", L"YB", 0
		}
	;

	if( sz < 0 )
	{
		sb_append( sb, L"unknown" );
	}
	else if( sz < 1 )
	{
		sb_append( sb, _( L"empty" ) );
	}
	else if( sz < 1024 )
	{
		sb_printf( sb, L"%lldB", sz );
	}
	else
	{
		int i;
		
		for( i=0; sz_name[i]; i++ )
		{
			if( sz < (1024*1024) || !sz_name[i+1] )
			{
				int isz = sz/1024;
				if( isz > 9 )
					sb_printf( sb, L"%d%ls", isz, sz_name[i] );
				else
					sb_printf( sb, L"%.1f%ls", (double)sz/1024, sz_name[i] );
				break;
			}
			sz /= 1024;
			
		}
	}	
}

wcstring format_size(long long sz)
{
    wcstring result;
	const wchar_t *sz_name[]= {
			L"kB", L"MB", L"GB", L"TB", L"PB", L"EB", L"ZB", L"YB", 0
    };

	if( sz < 0 )
	{
		result.append( L"unknown" );
	}
	else if( sz < 1 )
	{
		result.append( _( L"empty" ) );
	}
	else if( sz < 1024 )
	{
		result.append(format_string( L"%lldB", sz ));
	}
	else
	{
		int i;
		
		for( i=0; sz_name[i]; i++ )
		{
			if( sz < (1024*1024) || !sz_name[i+1] )
			{
				int isz = sz/1024;
				if( isz > 9 )
					result.append( format_string( L"%d%ls", isz, sz_name[i] ));
				else
					result.append( format_string( L"%.1f%ls", (double)sz/1024, sz_name[i] ));
				break;
			}
			sz /= 1024;
			
		}
	}
    return result;
}

/* Crappy function to extract the most significant digit of an unsigned long long value */
static char extract_most_significant_digit(unsigned long long *xp) {
    unsigned long long place_value = 1;
    unsigned long long x = *xp;
    while (x >= 10) {
        x /= 10;
        place_value *= 10;
    }
    *xp -= (place_value * x);
    return x + '0';
}

void append_ull(char *buff, unsigned long long val, size_t *inout_idx, size_t max_len) {
    size_t idx = *inout_idx;
    while (val > 0 && idx < max_len)
        buff[idx++] = extract_most_significant_digit(&val);
    *inout_idx = idx;
}

void append_str(char *buff, const char *str, size_t *inout_idx, size_t max_len) {
    size_t idx = *inout_idx;
    while (*str && idx < max_len)
        buff[idx++] = *str++;
    *inout_idx = idx;
}

void format_size_safe(char buff[128], unsigned long long sz) {
    const size_t buff_size = 128;
    const size_t max_len = buff_size - 1; //need to leave room for a null terminator
    bzero(buff, buff_size);
    size_t idx = 0;
	const char * const sz_name[]= {
        "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB", NULL
    };
	if (sz < 1)
    {
        strncpy(buff, "empty", buff_size);
    }
    else if (sz < 1024)
    {
        append_ull(buff, sz, &idx, max_len);
        append_str(buff, "B", &idx, max_len);
    }
    else
    {
		for( size_t i=0; sz_name[i]; i++ )
		{
			if( sz < (1024*1024) || !sz_name[i+1] )
			{
				unsigned long long isz = sz/1024;
				if( isz > 9 )
                {
                    append_ull(buff, isz, &idx, max_len);
                }
				else
                {
                    if (isz == 0)
                    {
                        append_str(buff, "0", &idx, max_len);
                    }
                    else
                    {
                        append_ull(buff, isz, &idx, max_len);
                    }
                    
                    // Maybe append a single fraction digit
                    unsigned long long remainder = sz % 1024;
                    if (remainder > 0)
                    {
                        char tmp[3] = {'.', extract_most_significant_digit(&remainder), 0};
                        append_str(buff, tmp, &idx, max_len);
                    }
                }
                append_str(buff, sz_name[i], &idx, max_len);
				break;
			}
			sz /= 1024;
		}
    }
}

double timef()
{
	int time_res;
	struct timeval tv;
	
	time_res = gettimeofday(&tv, 0);
	
	if( time_res ) 
	{
		/*
		  Fixme: What on earth is the correct parameter value for NaN?
		  The man pages and the standard helpfully state that this
		  parameter is implementation defined. Gcc gives a warning if
		  a null pointer is used. But not even all mighty Google gives
		  a hint to what value should actually be returned.
		 */
		return nan("");
	}
	
	return (double)tv.tv_sec + 0.000001*tv.tv_usec;
}

void exit_without_destructors(int code) {
    _exit(code);
}

/* Helper function to convert from a null_terminated_array_t<wchar_t> to a null_terminated_array_t<char_t> */
null_terminated_array_t<char> convert_wide_array_to_narrow(const null_terminated_array_t<wchar_t> &wide_arr) {
    const wchar_t *const *arr = wide_arr.get();
    if (! arr)
        return null_terminated_array_t<char>();
        
    std::vector<std::string> list;
    for (size_t i=0; arr[i]; i++) {
        list.push_back(wcs2string(arr[i]));
    }
    return null_terminated_array_t<char>(list);
}

void append_path_component(wcstring &path, const wcstring &component)
{
    size_t len = path.size();
    if (len == 0)
    {
        path = component;
    }
    else
    {
        if (path[len-1] != L'/') path.push_back(L'/');
        path.append(component);
    }
}

extern "C" {
__attribute__((noinline)) void debug_thread_error(void) {}
}

 
void set_main_thread() {
 	main_thread_id = pthread_self();
}


/* Notice when we've forked */
static bool is_child_of_fork = false;
static void child_note_forked(void) {
    is_child_of_fork = true;
}

bool is_forked_child(void) {
    if (is_child_of_fork) {
        printf("Uh-oh: %d\n", getpid());
        while (1) sleep(10000);
    }
    return is_child_of_fork;
}

void setup_fork_guards(void) {
    /* Notice when we fork */
    pthread_atfork(NULL /* prepare */, NULL /* parent */, child_note_forked);
}

static bool is_main_thread() {
 	assert (main_thread_id != 0);
 	return main_thread_id == pthread_self();
}

void assert_is_main_thread(const char *who)
{
    if (! is_main_thread()) {
        fprintf(stderr, "Warning: %s called off of main thread. Break on debug_thread_error to debug.\n", who);
        debug_thread_error();
    }
}

void assert_is_not_forked_child(const char *who)
{
    if (is_forked_child()) {
        fprintf(stderr, "Warning: %s called in a forked child. Break on debug_thread_error to debug.\n", who);
        debug_thread_error();
    }
}

void assert_is_background_thread(const char *who)
{
    if (is_main_thread()) {
        fprintf(stderr, "Warning: %s called on the main thread (may block!). Break on debug_thread_error to debug.\n", who);
        debug_thread_error();
    }
}

void assert_is_locked(void *vmutex, const char *who)
{
    pthread_mutex_t *mutex = static_cast<pthread_mutex_t*>(vmutex);
    if (0 == pthread_mutex_trylock(mutex)) {
        fprintf(stderr, "Warning: %s is not locked when it should be. Break on debug_thread_error to debug.\n", who);
        debug_thread_error();
        pthread_mutex_unlock(mutex);
    }
}

void scoped_lock::lock(void) {
    assert(! locked);
    assert(! is_forked_child());
    VOMIT_ON_FAILURE(pthread_mutex_lock(lock_obj));
    locked = true;
}

void scoped_lock::unlock(void) {
    assert(locked);
    assert(! is_forked_child());
    VOMIT_ON_FAILURE(pthread_mutex_unlock(lock_obj));
    locked = false;
}

scoped_lock::scoped_lock(pthread_mutex_t &mutex) : lock_obj(&mutex), locked(false) {
    this->lock();
}

scoped_lock::~scoped_lock() {
    if (locked) this->unlock();
}