fish-shell/tokenizer.cpp

/** \file tokenizer.c

A specialized tokenizer for tokenizing the fish language. In the
future, the tokenizer should be extended to support marks,
tokenizing multiple strings and disposing of unused string
segments.
*/

#include "config.h"

#include <stdlib.h>
#include <stdio.h>
#include <wchar.h>
#include <wctype.h>
#include <string.h>
#include <unistd.h>


#include "fallback.h"
#include "util.h"

#include "wutil.h"
#include "tokenizer.h"
#include "common.h"

/* Wow what a hack */
#define TOK_CALL_ERROR(t, e, x) do { tok_call_error((t), (e), (t)->squash_errors ? L"" : (x)); } while (0)

/**
   Error string for unexpected end of string
*/
#define QUOTE_ERROR _( L"Unexpected end of string, quotes are not balanced" )

/**
   Error string for mismatched parenthesis
*/
#define PARAN_ERROR _( L"Unexpected end of string, parenthesis do not match" )

/**
   Error string for invalid redirections
*/
#define REDIRECT_ERROR _( L"Invalid input/output redirection" )

/**
   Error string for when trying to pipe from fd 0
*/
#define PIPE_ERROR _( L"Can not use fd 0 as pipe output" )

/**
   Characters that separate tokens. They are ordered by frequency of occurrence to increase parsing speed.
*/
#define SEP L" \n|\t;#\r<>^&"

/**
   Maximum length of a string containing a file descriptor number
*/
#define FD_STR_MAX_LEN 16

/**
   Descriptions of all tokenizer errors
*/
static const wchar_t *tok_desc[] =
{
    N_(L"Tokenizer not yet initialized"),
    N_(L"Tokenizer error"),
    N_(L"Invalid token"),
    N_(L"String"),
    N_(L"Pipe"),
    N_(L"End of command"),
    N_(L"Redirect output to file"),
    N_(L"Append output to file"),
    N_(L"Redirect input to file"),
    N_(L"Redirect to file descriptor"),
    N_(L"Redirect output to file if file does not exist"),
    N_(L"Run job in background"),
    N_(L"Comment")
}
;

/**
   Tests if the tokenizer buffer is large enough to hold contents of
   the specified length, and if not, reallocates the tokenizer buffer.

   \return 0 if the system could not provide the memory needed, and 1 otherwise.
*/
static int check_size(tokenizer *tok, size_t len)
{
    if (tok->last_len <= len)
    {
        wchar_t *tmp;
        tok->last_len = len +1;
        tmp = (wchar_t *)realloc(tok->last, sizeof(wchar_t)*tok->last_len);
        if (tmp == 0)
        {
            wperror(L"realloc");
            return 0;
        }
        tok->last = tmp;
    }
    return 1;
}

/**
   Set the latest tokens string to be the specified error message
*/
static void tok_call_error(tokenizer *tok, int error_type, const wchar_t *error_message)
{
    tok->last_type = TOK_ERROR;
    tok->error = error_type;
    if (!check_size(tok, wcslen(error_message)+1))
    {
        if (tok->last != 0)
            *tok->last=0;
        return;
    }

    wcscpy(tok->last, error_message);
}

int tok_get_error(tokenizer *tok)
{
    return tok->error;
}


void tok_init(tokenizer *tok, const wchar_t *b, int flags)
{

    /* We can only generate error messages on the main thread due to wgettext() thread safety issues. */
    if (!(flags & TOK_SQUASH_ERRORS))
    {
        ASSERT_IS_MAIN_THREAD();
    }

    CHECK(tok,);

    memset(tok, 0, sizeof(tokenizer));

    CHECK(b,);


    tok->accept_unfinished = !!(flags & TOK_ACCEPT_UNFINISHED);
    tok->show_comments = !!(flags & TOK_SHOW_COMMENTS);
    tok->squash_errors = !!(flags & TOK_SQUASH_ERRORS);
    tok->has_next=true;

    tok->has_next = (*b != L'\0');
    tok->orig_buff = tok->buff = b;
    tok->cached_lineno_offset = 0;
    tok->cached_lineno_count = 0;
    tok_next(tok);
}

void tok_destroy(tokenizer *tok)
{
    CHECK(tok,);

    free(tok->last);
}

int tok_last_type(tokenizer *tok)
{
    CHECK(tok, TOK_ERROR);
    CHECK(tok->buff, TOK_ERROR);

    return tok->last_type;
}

wchar_t *tok_last(tokenizer *tok)
{
    CHECK(tok, 0);

    return tok->last;
}

int tok_has_next(tokenizer *tok)
{
    /*
      Return 1 on broken tokenizer
    */
    CHECK(tok, 1);
    CHECK(tok->buff, 1);

    /*  fwprintf( stderr, L"has_next is %ls \n", tok->has_next?L"true":L"false" );*/
    return   tok->has_next;
}

int tokenizer::line_number_of_character_at_offset(size_t offset)
{
    // we want to return (one plus) the number of newlines at offsets less than the given offset
    // cached_lineno_count is the number of newlines at indexes less than cached_lineno_offset
    const wchar_t *str = orig_buff;
    if (! str)
        return 0;

    // easy hack to handle 0
    if (offset == 0)
        return 1;

    size_t i;
    if (offset > cached_lineno_offset)
    {
        for (i = cached_lineno_offset; str[i] && i<offset; i++)
        {
            /* Add one for every newline we find in the range [cached_lineno_offset, offset) */
            if (str[i] == L'\n')
                cached_lineno_count++;
        }
        cached_lineno_offset = i; //note: i, not offset, in case offset is beyond the length of the string
    }
    else if (offset < cached_lineno_offset)
    {
        /* Subtract one for every newline we find in the range [offset, cached_lineno_offset) */
        for (i = offset; i < cached_lineno_offset; i++)
        {
            if (str[i] == L'\n')
                cached_lineno_count--;
        }
        cached_lineno_offset = offset;
    }
    return cached_lineno_count + 1;
}

/**
   Tests if this character can be a part of a string. The redirect ^ is allowed unless it's the first character.
*/
bool tok_is_string_character(wchar_t c, bool is_first)
{
    switch (c)
    {
            /* Unconditional separators */
        case L'\0':
        case L' ':
        case L'\n':
        case L'|':
        case L'\t':
        case L';':
        case L'#':
        case L'\r':
        case L'<':
        case L'>':
        case L'&':
            return false;

            /* Conditional separator */
        case L'^':
            return ! is_first;

        default:
            return true;
    }
}

/**
   Quick test to catch the most common 'non-magical' characters, makes
   read_string slightly faster by adding a fast path for the most
   common characters. This is obviously not a suitable replacement for
   iswalpha.
*/
static int myal(wchar_t c)
{
    return (c>=L'a' && c<=L'z') || (c>=L'A'&&c<=L'Z');
}

/**
   Read the next token as a string
*/
static void read_string(tokenizer *tok)
{
    const wchar_t *start;
    long len;
    int mode=0;
    int do_loop=1;
    int paran_count=0;

    start = tok->buff;
    bool is_first = true;

    while (1)
    {

        if (!myal(*tok->buff))
        {
//      debug(1, L"%lc", *tok->buff );

            if (*tok->buff == L'\\')
            {
                tok->buff++;
                if (*tok->buff == L'\0')
                {
                    if ((!tok->accept_unfinished))
                    {
                        TOK_CALL_ERROR(tok, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR);
                        return;
                    }
                    else
                    {
                        do_loop = 0;
                    }


                }
                else if (*tok->buff == L'\n' && mode == 0)
                {
                    tok->buff--;
                    do_loop = 0;
                    break;
                }

                tok->buff++;
                continue;
            }


            /*
              The modes are as follows:

              0: regular text
              1: inside of subshell
              2: inside of array brackets
              3: inside of array brackets and subshell, like in '$foo[(ech'
            */
            switch (mode)
            {
                case 0:
                {
                    switch (*tok->buff)
                    {
                        case L'(':
                        {
                            paran_count=1;
                            mode = 1;
                            break;
                        }

                        case L'[':
                        {
                            if (tok->buff != start)
                                mode=2;
                            break;
                        }

                        case L'\'':
                        case L'"':
                        {

                            const wchar_t *end = quote_end(tok->buff);
                            tok->last_quote = *tok->buff;
                            if (end)
                            {
                                tok->buff=(wchar_t *)end;
                            }
                            else
                            {
                                tok->buff += wcslen(tok->buff);

                                if ((!tok->accept_unfinished))
                                {
                                    TOK_CALL_ERROR(tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
                                    return;
                                }
                                do_loop = 0;

                            }
                            break;
                        }

                        default:
                        {
                            if (!tok_is_string_character(*(tok->buff), is_first))
                            {
                                do_loop=0;
                            }
                        }
                    }
                    break;
                }

                case 3:
                case 1:
                    switch (*tok->buff)
                    {
                        case L'\'':
                        case L'\"':
                        {
                            const wchar_t *end = quote_end(tok->buff);
                            if (end)
                            {
                                tok->buff=(wchar_t *)end;
                            }
                            else
                            {
                                tok->buff += wcslen(tok->buff);
                                if ((!tok->accept_unfinished))
                                {
                                    TOK_CALL_ERROR(tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR);
                                    return;
                                }
                                do_loop = 0;
                            }

                            break;
                        }

                        case L'(':
                            paran_count++;
                            break;
                        case L')':
                            paran_count--;
                            if (paran_count == 0)
                            {
                                mode--;
                            }
                            break;
                        case L'\0':
                            do_loop = 0;
                            break;
                    }
                    break;
                case 2:
                    switch (*tok->buff)
                    {
                        case L'(':
                            paran_count=1;
                            mode = 3;
                            break;

                        case L']':
                            mode=0;
                            break;

                        case L'\0':
                            do_loop = 0;
                            break;
                    }
                    break;
            }
        }


        if (!do_loop)
            break;

        tok->buff++;
        is_first = false;
    }

    if ((!tok->accept_unfinished) && (mode!=0))
    {
        TOK_CALL_ERROR(tok, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR);
        return;
    }


    len = tok->buff - start;

    if (!check_size(tok, len))
        return;

    memcpy(tok->last, start, sizeof(wchar_t)*len);
    tok->last[len] = L'\0';
    tok->last_type = TOK_STRING;
}

/**
   Read the next token as a comment.
*/
static void read_comment(tokenizer *tok)
{
    const wchar_t *start;

    start = tok->buff;
    while (*(tok->buff)!= L'\n' && *(tok->buff)!= L'\0')
        tok->buff++;

    size_t len = tok->buff - start;
    if (!check_size(tok, len))
        return;

    memcpy(tok->last, start, sizeof(wchar_t)*len);
    tok->last[len] = L'\0';
    tok->last_type = TOK_COMMENT;
}

/**
   Read a FD redirection.
*/
static void read_redirect(tokenizer *tok, int fd)
{
    int mode = -1;

    if ((*tok->buff == L'>') ||
            (*tok->buff == L'^'))
    {
        tok->buff++;
        if (*tok->buff == *(tok->buff-1))
        {
            tok->buff++;
            mode = 1;
        }
        else
        {
            mode = 0;
        }

        if (*tok->buff == L'|')
        {
            if (fd == 0)
            {
                TOK_CALL_ERROR(tok, TOK_OTHER, PIPE_ERROR);
                return;
            }
            check_size(tok, FD_STR_MAX_LEN);
            tok->buff++;
            swprintf(tok->last, FD_STR_MAX_LEN, L"%d", fd);
            tok->last_type = TOK_PIPE;
            return;
        }
    }
    else if (*tok->buff == L'<')
    {
        tok->buff++;
        mode = 2;
    }
    else
    {
        TOK_CALL_ERROR(tok, TOK_OTHER, REDIRECT_ERROR);
    }

    if (!check_size(tok, 2))
    {
        return;
    }

    swprintf(tok->last, tok->last_len, L"%d", fd);

    if (*tok->buff == L'&')
    {
        tok->buff++;
        tok->last_type = TOK_REDIRECT_FD;
    }
    else if (*tok->buff == L'?')
    {
        tok->buff++;
        tok->last_type = TOK_REDIRECT_NOCLOB;
    }
    else
    {
        tok->last_type = TOK_REDIRECT_OUT + mode;
    }
}

wchar_t tok_last_quote(tokenizer *tok)
{
    CHECK(tok, 0);

    return tok->last_quote;
}

/**
   Test if a character is whitespace. Differs from iswspace in that it
   does not consider a newline to be whitespace.
*/
static int my_iswspace(wchar_t c)
{
    if (c == L'\n')
        return 0;
    else
        return iswspace(c);
}


const wchar_t *tok_get_desc(int type)
{
    if (type < 0 || (size_t)type >= sizeof(tok_desc))
    {
        return _(L"Invalid token type");
    }
    return _(tok_desc[type]);
}


void tok_next(tokenizer *tok)
{

    CHECK(tok,);
    CHECK(tok->buff,);

    if (tok_last_type(tok) == TOK_ERROR)
    {
        tok->has_next=false;
        return;
    }

    if (!tok->has_next)
    {
        /*    wprintf( L"EOL\n" );*/
        tok->last_type = TOK_END;
        return;
    }

    while (1)
    {
        if (my_iswspace(*(tok->buff)))
        {
            tok->buff++;
        }
        else
        {
            if ((*(tok->buff) == L'\\') &&(*(tok->buff+1) == L'\n'))
            {
                tok->last_pos = tok->buff - tok->orig_buff;
                tok->buff+=2;
                tok->last_type = TOK_END;
                return;
            }
            break;
        }
    }


    if (*tok->buff == L'#')
    {
        if (tok->show_comments)
        {
            tok->last_pos = tok->buff - tok->orig_buff;
            read_comment(tok);
            return;
        }
        else
        {
            while (*(tok->buff)!= L'\n' && *(tok->buff)!= L'\0')
                tok->buff++;
        }

        while (my_iswspace(*(tok->buff)))
            tok->buff++;
    }

    tok->last_pos = tok->buff - tok->orig_buff;

    switch (*tok->buff)
    {

        case L'\0':
            tok->last_type = TOK_END;
            /*fwprintf( stderr, L"End of string\n" );*/
            tok->has_next = false;
            break;
        case 13:
        case L'\n':
        case L';':
            tok->last_type = TOK_END;
            tok->buff++;
            break;
        case L'&':
            tok->last_type = TOK_BACKGROUND;
            tok->buff++;
            break;

        case L'|':
            check_size(tok, 2);

            tok->last[0]=L'1';
            tok->last[1]=L'\0';
            tok->last_type = TOK_PIPE;
            tok->buff++;
            break;

        case L'>':
            read_redirect(tok, 1);
            return;
        case L'<':
            read_redirect(tok, 0);
            return;
        case L'^':
            read_redirect(tok, 2);
            return;

        default:
        {
            if (iswdigit(*tok->buff))
            {
                const wchar_t *orig = tok->buff;
                int fd = 0;
                while (iswdigit(*tok->buff))
                    fd = (fd*10) + (*(tok->buff++) - L'0');

                switch (*(tok->buff))
                {
                    case L'^':
                    case L'>':
                    case L'<':
                        read_redirect(tok, fd);
                        return;
                }
                tok->buff = orig;
            }
            read_string(tok);
        }

    }

}

const wchar_t *tok_string(tokenizer *tok)
{
    return tok?tok->orig_buff:0;
}

wchar_t *tok_first(const wchar_t *str)
{
    tokenizer t;
    wchar_t *res=0;

    CHECK(str, 0);

    tok_init(&t, str, TOK_SQUASH_ERRORS);

    switch (tok_last_type(&t))
    {
        case TOK_STRING:
//        fwprintf( stderr, L"Got token %ls\n", tok_last( &t ));
            res = wcsdup(tok_last(&t));
            break;
        default:
            break;
    }

    tok_destroy(&t);
    return res;
}

int tok_get_pos(tokenizer *tok)
{
    CHECK(tok, 0);

    return (int)tok->last_pos;
}


void tok_set_pos(tokenizer *tok, int pos)
{
    CHECK(tok,);

    tok->buff = tok->orig_buff + pos;
    tok->has_next = true;
    tok_next(tok);
}


#ifdef TOKENIZER_TEST

/**
   This main function is used for compiling the tokenizer_test command, used for testing the tokenizer.
*/
int main(int argc, char **argv)
{
    tokenizer tok;
    int i;
    for (i=1; i<argc; i++)
    {
        wprintf(L"Tokenizing string %s\n", argv[i]);
        for (tok_init(&tok, str2wcs(argv[i]), 0); tok_has_next(&tok); tok_next(&tok))
        {
            switch (tok_last_type(&tok))
            {
                case TOK_INVALID:
                    wprintf(L"Type: INVALID\n");
                    break;
                case TOK_STRING:
                    wprintf(L"Type: STRING\t Value: %ls\n", tok_last(&tok));
                    break;
                case TOK_PIPE:
                    wprintf(L"Type: PIPE\n");
                    break;
                case TOK_END:
                    wprintf(L"Type: END\n");
                    break;
                case TOK_ERROR:
                    wprintf(L"Type: ERROR\n");
                    break;
                default:
                    wprintf(L"Type: Unknown\n");
                    break;
            }
        }
        tok_destroy(&tok);

    }
}

#endif